Disk Cache initial implementation

Serializes code blocks to disk - only blocks coming from known regions, for now

Disabled by default, key and versioning still needs work, but works for testing

FEXCore/Source/CMakeLists.txt

@@ -18,6 +18,7 @@ set(SRCS
   Common/JitSymbols.cpp
   Interface/Context/Context.cpp
   Interface/Core/LookupCache.cpp
+  Interface/Core/DiskCache.cpp
   Interface/Core/CodeCache.cpp
   Interface/Core/Core.cpp
   Interface/Core/CPUBackend.cpp

FEXCore/Source/Interface/Config/Config.cpp

@@ -48,6 +48,7 @@ enum Paths {
   PATH_CONFIG_FILE_LOCAL,
   PATH_CONFIG_FILE_GLOBAL,
   PATH_CONFIG_TELEMETRY_FOLDER,
+  PATH_CACHE_DIR,
   PATH_LAST,
 };
 static std::array<fextl::string, Paths::PATH_LAST> Paths;
@@ -64,6 +65,10 @@ void SetConfigFileLocation(const std::string_view Path, bool Global) {
   Paths[PATH_CONFIG_FILE_LOCAL + Global] = Path;
 }
 
+void SetCacheDirectory(const std::string_view Path) {
+  Paths[PATH_CACHE_DIR] = Path;
+}
+
 const fextl::string& GetTelemetryDirectory() {
   auto& Path = Paths[PATH_CONFIG_TELEMETRY_FOLDER];
   if (Path.empty()) {
@@ -91,6 +96,10 @@ const fextl::string& GetConfigFileLocation(bool Global) {
   return Paths[PATH_CONFIG_FILE_LOCAL + Global];
 }
 
+const fextl::string& GetCacheDirectory() {
+  return Paths[PATH_CACHE_DIR];
+}
+
 fextl::string GetApplicationConfig(const std::string_view Program, bool Global) {
   fextl::string ConfigFile = GetConfigDirectory(Global);
 

FEXCore/Source/Interface/Config/Config.json.in

@@ -132,6 +132,34 @@
         "Desc": [
           "Allows overriding cpu feature flags for manual testing"
         ]
+      },
+      "DiskCache": {
+        "Type": "bool",
+        "Default": "false",
+        "Desc": [
+          "Enables disk caching for code blocks"
+        ]
+      },
+      "DiskCacheRelocationFilter": {
+        "Type": "bool",
+        "Default": "true",
+        "Desc": [
+          "Don't cache blocks with relocations pointing outside of any known region"
+        ]
+      },
+      "DiskCachePath": {
+        "Type": "str",
+        "Default": "",
+        "Desc": [
+          "Optional base directory override for disk cache"
+        ]
+      },
+      "DiskCacheRODBNames": {
+        "Type": "str",
+        "Default": "",
+        "Desc": [
+          "Optional list of extra read-only disk cache DBs to consider"
+        ]
       }
     },
     "Emulation": {

FEXCore/Source/Interface/Context/Context.cpp

@@ -58,7 +58,7 @@ bool FEXCore::Context::ContextImpl::IsAddressInCodeBuffer(FEXCore::Core::Interna
 
 bool FEXCore::Context::ContextImpl::RequiresRelocatableConstants() const {
   // Support relocation when generating a cache or when generating reference code for validation
-  return CodeCache.IsGeneratingCache || FEXCore::Config::Get_ENABLECODECACHEVALIDATION();
+  return CodeCache.IsGeneratingCache || FEXCore::Config::Get_ENABLECODECACHEVALIDATION() || DiskCache.IsWritingDiskCache();
 }
 
 } // namespace FEXCore::Context

FEXCore/Source/Interface/Context/Context.h

@@ -10,6 +10,7 @@
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/HostFeatures.h>
+#include <FEXCore/Core/DiskCache.h>
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/SignalScopeGuards.h>
@@ -202,6 +203,8 @@ public:
   FEXCore::CPUID::XCRResults RunXCRFunction(uint32_t Function) override;
   FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) override;
 
+  virtual void InitDiskCache() override {}
+
   CodeCache& GetCodeCache() override {
     return CodeCache;
   }
@@ -375,6 +378,7 @@ public:
   FEXCore::HLE::SourcecodeResolver* SourcecodeResolver {};
   FEXCore::ThunkHandler* ThunkHandler {};
   fextl::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
+  DiskCache::DiskCache DiskCache;
   CodeCache CodeCache;
   fextl::unique_ptr<CodeMapWriter> CodeMapWriter;
 

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -18,6 +18,11 @@ $end_info$
 #include <FEXCore/fextl/map.h>
 
 #include <cstdint>
+#include <span>
+
+namespace FEXCore::DiskCache {
+struct BlobEntryPoint;
+}
 
 namespace FEXCore::CPU {
 union Relocation;
@@ -56,6 +61,8 @@ namespace CPU {
       fextl::map<uint64_t, uint8_t*> EntryPoints;
       // The total size of the codeblock from [BlockBegin, BlockBegin+Size).
       size_t Size;
+      // Offset of BlockBegin from the start of the CodeBuffer it lives in
+      uint64_t HostCodeOffset;
     };
 
     // Header that can live at the start of a JIT block.
@@ -115,6 +122,10 @@ namespace CPU {
     virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                      FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0;
 
+    virtual CompiledCode LoadCachedCode(std::span<const uint8_t> HostBytes, std::span<const DiskCache::BlobEntryPoint> EntryPoints) {
+      return {};
+    }
+
     virtual fextl::vector<FEXCore::CPU::Relocation> TakeRelocations(uint64_t GuestBaseAddress) = 0;
 
     virtual void ClearCache() {}

FEXCore/Source/Interface/Core/Core.cpp

@@ -103,6 +103,8 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
 
   // Track atomic TSO emulation configuration.
   UpdateAtomicTSOEmulationConfig();
+
+  DiskCache.Init(this);
 }
 
 struct GetFrameBlockInfoResult {
@@ -858,6 +860,40 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     return HostCode;
   }
 
+  std::optional<ExecutableFileSectionInfo> Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);
+  std::optional<DiskCache::CodeHitData> Hit;
+  bool DiskCacheHitRelocationsApplied = false;
+  bool LoadDiskCacheCode = true;
+  if (Region && Region->FileStartVA != 0) {
+    Hit = DiskCache.Lookup(Thread, *Region, GuestRIP);
+    if (Hit) {
+      DiskCacheHitRelocationsApplied = CodeCache.ApplyCodeRelocations(GuestRIP, std::as_writable_bytes(Hit->HostCode), Hit->Relocations, 0, false);
+
+      if (DiskCacheHitRelocationsApplied && LoadDiskCacheCode) {
+        auto LoadedCode = Thread->CPUBackend->LoadCachedCode(Hit->HostCode, Hit->EntryPoints);
+        if (LoadedCode.BlockBegin) {
+
+          // annoying to unpack a different copy here, maybe better way to do this
+          fextl::set<uint64_t> EntryPoints;
+          for (auto [GuestOffset, HostAddr] : LoadedCode.EntryPoints) {
+            EntryPoints.insert(GuestOffset + Region->FileStartVA);
+          }
+          for (auto CodePage : Hit->GuestPages) {
+            if (Thread->LookupCache->AddBlockExecutableRange(Thread, EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
+              SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
+            }
+          }
+          for (auto [GuestOffset, HostAddr] : LoadedCode.EntryPoints) {
+            Thread->LookupCache->AddBlockMapping(Thread, GuestOffset + Region->FileStartVA, Hit->GuestPages, HostAddr);
+          }
+
+          uint64_t ModuleOffset = GuestRIP - Region->FileStartVA;
+          return reinterpret_cast<uintptr_t>(LoadedCode.EntryPoints[ModuleOffset]);
+        }
+      }
+    }
+  }
+
   // Accumulate a JIT count now, as even if another thread raced us, it should count as a compile.
   FEXCORE_PROFILE_INSTANT_INCREMENT(Thread, AccumulatedJITCount, 1);
 
@@ -870,6 +906,9 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     return reinterpret_cast<uintptr_t>(CodePtr);
   }
 
+  // if this ever fires, we need to serialize the offset into disk cache
+  LOGMAN_THROW_A_FMT(StartAddr == GuestRIP, "StartAddr offset from GuestRIP");
+
   // The core managed to compile the code.
   if (Config.BlockJITNaming()) {
     auto FragmentBasePtr = CompiledCode.BlockBegin;
@@ -909,11 +948,6 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     }
   }
 
-  // Clear any relocations that might have been generated
-  if (!CodeCache.IsGeneratingCache) {
-    Thread->CPUBackend->ClearRelocations();
-  }
-
   fextl::vector<uint64_t> CodePages;
 
   if (NeedsAddGuestCodeRanges) {
@@ -929,17 +963,29 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     }
   }
 
-  // Insert to lookup cache
+  // Disk Cache
+  if (Region && Region->FileStartVA != 0) {
+    std::span<const FEXCore::CPU::Relocation> Relocations;
+    if (DebugData && DebugData->Relocations) {
+      Relocations = *DebugData->Relocations;
+    }
+    std::span<const uint8_t> GuestCode = {reinterpret_cast<const uint8_t*>(StartAddr), Length};
+    const Frontend::Decoder::DecodedBlockInformation* BlockInfo = NeedsAddGuestCodeRanges ? Thread->FrontendDecoder->GetDecodedBlockInfo() : nullptr;
+    DiskCache.Store(Thread, *Region, GuestRIP, GuestCode, CompiledCode, Relocations, BlockInfo);
 
+    if (CodeMapWriter) {
+      CodeMapWriter->AppendBlock(*Region, GuestRIP);
+    }
+  }
+
+  // Insert to lookup cache
   for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
     Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr);
   }
 
-  if (CodeMapWriter) {
-    auto Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);
-    if (Region && Region->FileStartVA != 0) {
-      CodeMapWriter->AppendBlock(*Region, GuestRIP);
-    }
+  // Clear any relocations that might have been generated
+  if (!CodeCache.IsGeneratingCache) {
+    Thread->CPUBackend->ClearRelocations();
   }
 
   return (uintptr_t)CodePtr;

FEXCore/Source/Interface/Core/DiskCache.cpp

@@ -0,0 +1,544 @@
+// SPDX-License-Identifier: MIT
+
+#include "FEXHeaderUtils/Filesystem.h"
+#include "FEXCore/Core/DiskCache.h"
+#include "FEXCore/Utils/LogManager.h"
+#include "Interface/Context/Context.h"
+#include "FEXCore/HLE/SyscallHandler.h"
+#include "FEXCore/Utils/File.h"
+#include "FEXCore/fextl/memory.h"
+#include <cstdint>
+#include <cstring>
+
+namespace FEXCore {
+
+namespace DiskCache {
+
+  namespace MesaFOZ {
+
+    enum { FOSSILIZE_COMPRESSION_NONE = 1, FOSSILIZE_COMPRESSION_DEFLATE = 2 };
+
+    enum { FOSSILIZE_FORMAT_VERSION = 6, FOSSILIZE_FORMAT_MIN_COMPAT_VERSION = 5 };
+
+#define FOZ_REF_MAGIC_SIZE 16
+
+    static const uint8_t stream_reference_magic_and_version[FOZ_REF_MAGIC_SIZE] = {
+      0x81, 'F', 'O', 'S', 'S', 'I', 'L', 'I', 'Z', 'E', 'D', 'B', 0, 0, 0, FOSSILIZE_FORMAT_VERSION, /* 4 bytes to use for versioning. */
+    };
+
+    struct __attribute__((packed)) mesa_index_db_file_entry {
+      uint64_t hash;
+      uint32_t size;
+      uint64_t last_access_time;
+      uint64_t cache_db_file_offset;
+    };
+
+  } // namespace MesaFOZ
+
+  bool FOZFile::Open(const fextl::string& FOZFileName, bool ReadOnly) {
+    FileName = FOZFileName;
+    this->ReadOnly = ReadOnly;
+
+    File::FileModes Modes = File::FileModes::READ;
+    if (!ReadOnly) {
+      Modes = Modes | File::FileModes::WRITE | File::FileModes::CREATE;
+    }
+    FD = fextl::make_unique<File::File>(FileName.c_str(), Modes);
+    if (!FD->IsValid()) {
+      FD.reset();
+      return false;
+    }
+
+    bool Valid = false;
+    bool TookLock = false;
+    ssize_t Size = FD->Seek(0, File::SeekOp::END);
+
+    if (Size < FOZ_REF_MAGIC_SIZE && !ReadOnly) {
+      if (!FD->Lock(OPEN_LOCK_TIMEOUT_MS)) {
+        FD.reset();
+        return false;
+      }
+      TookLock = true;
+      // seek in case someone else made it while we waited above
+      Size = FD->Seek(0, File::SeekOp::END);
+    }
+
+    if (Size == 0 && !ReadOnly) {
+      Valid = FD->Write(MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE) == FOZ_REF_MAGIC_SIZE;
+    } else {
+      FD->Seek(0, File::SeekOp::BEGIN);
+      uint8_t magic[FOZ_REF_MAGIC_SIZE];
+      if (FD->Read(magic, FOZ_REF_MAGIC_SIZE) == FOZ_REF_MAGIC_SIZE &&
+          memcmp(magic, MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE - 1) == 0) {
+        int version = magic[FOZ_REF_MAGIC_SIZE - 1];
+        Valid = version <= MesaFOZ::FOSSILIZE_FORMAT_VERSION && version >= MesaFOZ::FOSSILIZE_FORMAT_MIN_COMPAT_VERSION;
+      }
+    }
+
+    if (TookLock) {
+      FD->Unlock();
+    }
+
+    if (!Valid) {
+      FD.reset();
+    }
+    return Valid;
+  }
+
+  bool FOZFile::ReadNextBlob(MesaFOZ::foz_payload_key& OutKey, MesaFOZ::foz_payload_header& OutHeader, fextl::vector<uint8_t>& OutBlob) {
+    if (FD->Read(OutKey.bytes, sizeof(OutKey.bytes)) != sizeof(OutKey.bytes)) {
+      return false;
+    }
+    if (FD->Read(&OutHeader, sizeof(OutHeader)) != sizeof(OutHeader)) {
+      return false;
+    }
+    OutBlob.resize(OutHeader.payload_size);
+    if (FD->Read(OutBlob.data(), OutBlob.size()) != (ssize_t)OutBlob.size()) {
+      return false;
+    }
+    return true;
+  }
+
+  bool FOZFile::ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
+    ssize_t SeekRet = FD->Seek(Offset, File::SeekOp::BEGIN);
+    if (SeekRet < 0) {
+      return false;
+    }
+    if (FD->Read(OutBlob.data(), OutBlob.size()) != (ssize_t)OutBlob.size()) {
+      return false;
+    }
+
+    return true;
+  }
+
+  bool FOZFile::WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset) {
+    uint64_t WriteOffset = 0;
+    ssize_t SeekRet = FD->Seek(0, File::SeekOp::END);
+    if (SeekRet < 0) {
+      return false;
+    }
+    WriteOffset = (uint64_t)SeekRet;
+
+    if (FD->Write(Key.bytes, sizeof(Key.bytes)) != sizeof(Key.bytes)) {
+      return false;
+    }
+    WriteOffset += sizeof(Key.bytes);
+
+    uint64_t TotalBlobSize = 0;
+    for (const std::span<const uint8_t>& Chunk : BlobChunks) {
+      TotalBlobSize += Chunk.size();
+    }
+
+    MesaFOZ::foz_payload_header ScratchHeader {.payload_size = (uint32_t)TotalBlobSize,
+                                               .format = MesaFOZ::FOSSILIZE_COMPRESSION_NONE,
+                                               .crc = 0, // todo? maybe
+                                               .uncompressed_size = (uint32_t)TotalBlobSize};
+
+    if (FD->Write(&ScratchHeader, sizeof(ScratchHeader)) != sizeof(ScratchHeader)) {
+      return false;
+    }
+    WriteOffset += sizeof(ScratchHeader);
+
+    OutBlobOffset = WriteOffset;
+
+    for (const std::span<const uint8_t>& Chunk : BlobChunks) {
+      if (Chunk.size() == 0) {
+        continue;
+      }
+      if (FD->Write(Chunk.data(), Chunk.size()) != (ssize_t)Chunk.size()) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  bool IndexedDB::Open(const fextl::string& CacheDBName, bool ReadOnly) {
+    if (!CacheFOZ.Open(CacheDBName + ".foz", ReadOnly)) {
+      return false;
+    }
+    if (!IndexFOZ.Open(CacheDBName + "_idx.foz", ReadOnly)) {
+      return false;
+    }
+
+    this->ReadOnly = ReadOnly;
+    return true;
+  }
+
+  void IndexedDB::PopulateIndex(Index& CacheIndex) {
+    MesaFOZ::foz_payload_key Key;
+    MesaFOZ::foz_payload_header Header;
+    fextl::vector<uint8_t> Blob;
+
+    while (IndexFOZ.ReadNextBlob(Key, Header, Blob)) {
+      if (Blob.size() != sizeof(MesaFOZ::mesa_index_db_file_entry)) {
+        break;
+      }
+      MesaFOZ::mesa_index_db_file_entry* IndexEntry = (MesaFOZ::mesa_index_db_file_entry*)Blob.data();
+      if (IndexEntry->hash != XXH3_64bits(Key.bytes, FOSSILIZE_BLOB_HASH_LENGTH)) {
+        break;
+      }
+      CacheIndex.insert({IndexEntry->hash, {this, IndexEntry->cache_db_file_offset, IndexEntry->size}});
+    }
+    // could truncate/delete index if we don't end up perfectly at end here
+  }
+
+  bool IndexedDB::ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
+    return CacheFOZ.ReadBlob(Offset, OutBlob);
+  }
+
+  bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, Index& Index) {
+    if (ReadOnly) {
+      // shouldn't happen
+      return false;
+    }
+    uint64_t Hash = XXH3_64bits(Key.bytes, FOSSILIZE_BLOB_HASH_LENGTH);
+    if (Index.contains(Hash)) {
+      // shouldn't really happen.. assert or something?
+      return true;
+    }
+
+    if (!CacheFOZ.Lock(STORE_LOCK_TIMEOUT_MS) || !IndexFOZ.Lock(STORE_LOCK_TIMEOUT_MS)) {
+      CacheFOZ.Unlock();
+      IndexFOZ.Unlock();
+      return false;
+    }
+
+    // write cache side first so we get offset for index
+    uint64_t BlobOffset = 0;
+    if (!CacheFOZ.WriteBlob(Key, BlobChunks, BlobOffset)) {
+      CacheFOZ.Unlock();
+      IndexFOZ.Unlock();
+      return false;
+    }
+
+    uint64_t TotalBlobSize = 0;
+    for (const std::span<const uint8_t>& Chunk : BlobChunks) {
+      TotalBlobSize += Chunk.size();
+    }
+
+    MesaFOZ::mesa_index_db_file_entry IndexEntry {.hash = Hash,
+                                                  .size = (uint32_t)TotalBlobSize,
+                                                  .last_access_time = 0, // todo..
+                                                  .cache_db_file_offset = BlobOffset};
+
+    std::span<const uint8_t> IndexBlobChunks[] = {{(const uint8_t*)&IndexEntry, sizeof(IndexEntry)}};
+    uint64_t UnusedIndexBlobOffset = 0;
+    if (!IndexFOZ.WriteBlob(Key, IndexBlobChunks, UnusedIndexBlobOffset)) {
+      CacheFOZ.Unlock();
+      IndexFOZ.Unlock();
+      return false;
+    }
+
+    CacheFOZ.Unlock();
+    IndexFOZ.Unlock();
+
+    Index[Hash] = {this, BlobOffset, (uint32_t)TotalBlobSize};
+    return true;
+  }
+
+  bool DiskCache::OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly) {
+    fextl::unique_ptr<IndexedDB> CurDB;
+
+    if (!ReadOnly && RWCacheDB) {
+      // rw already opened, just support one
+      return false;
+    }
+
+    CurDB = fextl::make_unique<IndexedDB>();
+    if (!CurDB) {
+      return false;
+    }
+
+    if (!CurDB->Open(CacheDBName, ReadOnly)) {
+      CurDB.reset();
+      return false;
+    }
+
+    CurDB->PopulateIndex(Index);
+
+    if (ReadOnly) {
+      ROCacheDBs.push_back(std::move(CurDB));
+    } else {
+      RWCacheDB = std::move(CurDB);
+    }
+
+    return true;
+  }
+
+  void DiskCache::Init(FEXCore::Context::ContextImpl* CTX) {
+    this->CTX = CTX;
+
+    if (!EnableDiskCache) {
+      return;
+    }
+
+    // todo grab all CTX options that can change compilation here + any environmental/hw things and hash into a bucket key
+
+    fextl::string BasePath = BasePathOverride();
+    if (BasePath.empty()) {
+      // todo put bucket hash in that path
+      BasePath = FEXCore::Config::GetCacheDirectory() + "DiskCache/";
+    }
+    FHU::Filesystem::CreateDirectories(BasePath);
+
+    fextl::string RWDBBasePath = BasePath + "RWCacheDB";
+    OpenCacheDB(RWDBBasePath, false);
+
+    std::string_view RONames = RODBNames();
+    while (!RONames.empty()) {
+      const auto Delim = RONames.find(',');
+      const std::string_view ROName = RONames.substr(0, Delim);
+      if (!ROName.empty()) {
+        fextl::string RODBBasePath = BasePath;
+        RODBBasePath += ROName;
+        OpenCacheDB(RODBBasePath, true);
+      }
+      if (Delim == std::string_view::npos) {
+        break;
+      }
+      // advance to next
+      RONames.remove_prefix(Delim + 1);
+    }
+  }
+
+  std::optional<CodeHitData> DiskCache::Lookup(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP) {
+    if (!IsReadingDiskCache()) {
+      return std::nullopt;
+    }
+    std::lock_guard Guard(Lock);
+    uint64_t ModuleOffset = GuestRIP - Region.FileStartVA;
+
+    // todo move key making to a helper once we have options and stuff (see Store)
+    MesaFOZ::foz_payload_key Key = {};
+    memcpy(Key.bytes, &ModuleOffset, sizeof(ModuleOffset));
+
+    uint64_t Hash = XXH3_64bits(Key.bytes, FOSSILIZE_BLOB_HASH_LENGTH);
+    auto It = Index.find(Hash);
+    if (It == Index.end()) {
+      // definite miss
+      return std::nullopt;
+    }
+    const IndexEntry& Entry = It->second;
+    // found a key hash match, could still be a miss, read the blob and verify more
+    CodeHitData HitData;
+    HitData.Blob.resize(Entry.Size);
+    if (!Entry.DB->ReadCacheBlob(Entry.Offset, HitData.Blob)) {
+      return std::nullopt;
+    }
+
+    if (Entry.Size < sizeof(BlobFixedHeader)) {
+      return std::nullopt;
+    }
+    BlobFixedHeader Header;
+    memcpy(&Header, HitData.Blob.data(), sizeof(Header));
+
+    // do we have enough room in our live code to even hash GuestSize worth?
+    auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
+    if (RangeInfo.Size == 0 || RangeInfo.Base > GuestRIP) {
+      return std::nullopt;
+    }
+    uint64_t Available = RangeInfo.Base + RangeInfo.Size - GuestRIP;
+    if (Available < Header.GuestSize) {
+      return std::nullopt;
+    }
+
+    XXH128_hash_t LiveGuestHash = XXH3_128bits(reinterpret_cast<void*>(GuestRIP), Header.GuestSize);
+    if (std::memcmp(&LiveGuestHash, &Header.GuestHash, sizeof(Header.GuestHash)) != 0) {
+      // LogMan::Msg::IFmt("hash mismatch! length {:d}", Header.GuestSize);
+      return std::nullopt;
+    }
+    // LogMan::Msg::IFmt("hash ok! length {:d}", Header.GuestSize);
+
+    // this seems to be a full hit, lastly, check the entry is big enough to have everything (except maybe GuestCode)
+    uint32_t SizeNeeded = sizeof(Header) + Header.HostSize + Header.EntryPointCount * sizeof(BlobEntryPoint);
+    SizeNeeded += Header.SmallRelocCount * sizeof(BlobSmallRelocation) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation) +
+                  Header.TouchedGuestPagesCount * sizeof(int64_t);
+    if (Entry.Size < SizeNeeded) {
+      return std::nullopt;
+    }
+
+    HitData.HostCode = {HitData.Blob.data() + sizeof(Header), Header.HostSize};
+    HitData.EntryPoints = {reinterpret_cast<const BlobEntryPoint*>(HitData.Blob.data() + sizeof(Header) + Header.HostSize), Header.EntryPointCount};
+
+    auto* SmallRelocs = reinterpret_cast<const BlobSmallRelocation*>(
+      HitData.Blob.data() + sizeof(Header) + Header.HostSize + Header.EntryPointCount * sizeof(BlobEntryPoint));
+    auto* ThunkRelocs = reinterpret_cast<const BlobThunkRelocation*>(
+      reinterpret_cast<const uint8_t*>(SmallRelocs) + Header.SmallRelocCount * sizeof(BlobSmallRelocation));
+
+    HitData.Relocations.reserve(Header.SmallRelocCount + Header.ThunkRelocCount);
+    for (uint32_t i = 0; i < Header.SmallRelocCount; ++i) {
+      const auto& SmallReloc = SmallRelocs[i];
+      FEXCore::CPU::Relocation Reloc = FEXCore::CPU::Relocation::Default();
+      Reloc.Header.Type = (CPU::RelocationTypes)SmallReloc.Type;
+      Reloc.Header.Offset = SmallReloc.Offset;
+      switch (SmallReloc.Type) {
+      case uint8_t(CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL):
+        Reloc.NamedSymbolLiteral.Symbol = CPU::RelocNamedSymbolLiteral::NamedSymbol(SmallReloc.Named.Symbol);
+        break;
+      case uint8_t(CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL): Reloc.GuestRIP.GuestRIP = SmallReloc.RIPLiteral.GuestRIP; break;
+      case uint8_t(CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE):
+        Reloc.GuestRIP.RegisterIndex = SmallReloc.RIPMove.RegisterIndex;
+        Reloc.GuestRIP.GuestRIP = SmallReloc.RIPMove.GuestRIP;
+        break;
+      default: return std::nullopt;
+      }
+      HitData.Relocations.push_back(Reloc);
+    }
+    for (uint32_t i = 0; i < Header.ThunkRelocCount; ++i) {
+      const auto& BigReloc = ThunkRelocs[i];
+      FEXCore::CPU::Relocation Reloc = FEXCore::CPU::Relocation::Default();
+      Reloc.NamedThunkMove.Header.Offset = BigReloc.Offset;
+      Reloc.NamedThunkMove.Header.Type = CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE;
+      Reloc.NamedThunkMove.RegisterIndex = BigReloc.RegisterIndex;
+      memcpy(&Reloc.NamedThunkMove.Symbol, BigReloc.SymbolHash, sizeof(BigReloc.SymbolHash));
+      HitData.Relocations.push_back(Reloc);
+    }
+
+    auto* PageOffsets =
+      reinterpret_cast<const int64_t*>(reinterpret_cast<const uint8_t*>(ThunkRelocs) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation));
+    HitData.GuestPages.reserve(Header.TouchedGuestPagesCount);
+    for (uint32_t i = 0; i < Header.TouchedGuestPagesCount; ++i) {
+      HitData.GuestPages.push_back(GuestRIP + PageOffsets[i]);
+    }
+
+    return HitData;
+  }
+
+  bool DiskCache::Store(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP,
+                        std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
+                        std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo) {
+    if (!IsWritingDiskCache()) {
+      return false;
+    }
+    if (!DecodedBlockInfo) {
+      return false;
+    }
+    std::lock_guard Guard(Lock);
+
+    // check for any reloc targets outside of our jurisdiction
+    // todo what are they exactly? caching those blocks is great when it works, so need to figure this out and make finer-grained if we can
+    if (RelocationFilter) {
+      for (const auto& Reloc : Relocations) {
+        if (Reloc.Header.Offset < CompiledCode.HostCodeOffset || Reloc.Header.Offset >= CompiledCode.HostCodeOffset + CompiledCode.Size) {
+          continue;
+        }
+        if (Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE) {
+          continue;
+        }
+        uint64_t Target = Reloc.GuestRIP.GuestRIP;
+        if (Target >= Region.BeginVA && Target < Region.EndVA) {
+          continue;
+        }
+        auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target);
+        if (!TargetSection || TargetSection->FileInfo.FileId != Region.FileInfo.FileId) {
+          // we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm
+          return false;
+        }
+      }
+    }
+
+    // pack entrypoints to disk format
+    fextl::vector<BlobEntryPoint> CacheEntryPoints;
+    CacheEntryPoints.reserve(CompiledCode.EntryPoints.size());
+
+    for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
+      CacheEntryPoints.push_back({GuestAddr - Region.FileStartVA, uint32_t(HostAddr - CompiledCode.BlockBegin)});
+    }
+
+    // pack relocations to disk format
+    fextl::vector<BlobSmallRelocation> SmallRelocs;
+    fextl::vector<BlobThunkRelocation> ThunkRelocs;
+
+    // todo discover sizes first and reserve vecs?
+
+    for (const auto& Reloc : Relocations) {
+      // relocs aren't cleared every time if IsGeneratingCache, so filter just in case
+      if (Reloc.Header.Offset < CompiledCode.HostCodeOffset || Reloc.Header.Offset >= CompiledCode.HostCodeOffset + CompiledCode.Size) {
+        continue;
+      }
+      // re-relocate :harold:
+      uint32_t LocalOffset = uint32_t(Reloc.Header.Offset - CompiledCode.HostCodeOffset);
+
+      switch (Reloc.Header.Type) {
+      // it's important to zero-init the element completely so we don't have garbage in unused fields
+      // this way, the caches stay deterministic across machines
+      case CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
+        BlobSmallRelocation SmallReloc = {};
+        SmallReloc.Offset = LocalOffset;
+        SmallReloc.Type = uint8_t(Reloc.Header.Type);
+        SmallReloc.Named.Symbol = uint32_t(Reloc.NamedSymbolLiteral.Symbol);
+        SmallRelocs.push_back(SmallReloc);
+        break;
+      }
+      case CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {
+        BlobSmallRelocation SmallReloc = {};
+        SmallReloc.Offset = LocalOffset;
+        SmallReloc.Type = uint8_t(Reloc.Header.Type);
+        SmallReloc.RIPLiteral.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
+        SmallRelocs.push_back(SmallReloc);
+        break;
+      }
+      case CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {
+        BlobSmallRelocation SmallReloc = {};
+        SmallReloc.Offset = LocalOffset;
+        SmallReloc.Type = uint8_t(Reloc.Header.Type);
+        SmallReloc.RIPMove.RegisterIndex = Reloc.GuestRIP.RegisterIndex;
+        SmallReloc.RIPMove.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
+        SmallRelocs.push_back(SmallReloc);
+        break;
+      }
+      case CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
+        BlobThunkRelocation BigReloc = {};
+        BigReloc.Offset = LocalOffset;
+        BigReloc.RegisterIndex = Reloc.NamedThunkMove.RegisterIndex;
+        memcpy(BigReloc.SymbolHash, &Reloc.NamedThunkMove.Symbol, sizeof(BigReloc.SymbolHash));
+        ThunkRelocs.push_back(BigReloc);
+        break;
+      }
+      }
+    }
+
+    // pack touched pages, relative to GuestRIP
+    // in theory we could save some size here, unlikely we need all 64bits
+    fextl::vector<int64_t> GuestPageOffsets;
+    if (DecodedBlockInfo) {
+      GuestPageOffsets.reserve(DecodedBlockInfo->CodePages.size());
+      for (auto& GuestPage : DecodedBlockInfo->CodePages) {
+        GuestPageOffsets.push_back(GuestPage - GuestRIP);
+      }
+    }
+
+    uint64_t ModuleOffset = GuestRIP - Region.FileStartVA;
+
+    // todo also copy/hash options that affect codegen into the key
+    // todo should try to keep the key ascii i think?
+    MesaFOZ::foz_payload_key Key = {};
+    memcpy(Key.bytes, &ModuleOffset, sizeof(ModuleOffset));
+
+    BlobFixedHeader Header {
+      .GuestSize = (uint32_t)GuestCode.size(),
+      .HostSize = (uint32_t)CompiledCode.Size,
+      .EntryPointCount = (uint32_t)CacheEntryPoints.size(),
+      .SmallRelocCount = (uint32_t)SmallRelocs.size(),
+      .ThunkRelocCount = (uint32_t)ThunkRelocs.size(),
+      .TouchedGuestPagesCount = (uint32_t)GuestPageOffsets.size(),
+      .GuestHash = XXH3_128bits(GuestCode.data(), GuestCode.size()),
+    };
+
+    std::span<const uint8_t> BlobChunks[] = {
+      {(const uint8_t*)&Header, sizeof(Header)},
+      {(const uint8_t*)CompiledCode.BlockBegin, CompiledCode.Size},
+      {(const uint8_t*)CacheEntryPoints.data(), CacheEntryPoints.size() * sizeof(BlobEntryPoint)},
+      {(const uint8_t*)SmallRelocs.data(), SmallRelocs.size() * sizeof(BlobSmallRelocation)},
+      {(const uint8_t*)ThunkRelocs.data(), ThunkRelocs.size() * sizeof(BlobThunkRelocation)},
+      {(const uint8_t*)GuestPageOffsets.data(), GuestPageOffsets.size() * sizeof(int64_t)},
+      GuestCode,
+    };
+
+    return RWCacheDB->StoreCacheBlob(Key, BlobChunks, Index);
+  }
+
+} // namespace DiskCache
+
+} // namespace FEXCore

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -1117,6 +1117,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
       EntryPoint.second += Delta;
     }
     CodeBegin += Delta;
+    CodeData.HostCodeOffset = CodeData.BlockBegin - CurrentCodeBuffer->Ptr;
 
     // Offset the relocations based on how far forward they moved from the temp buffer to the new buffer.
     // TODO: Relocations should instead be relocated based on the block entrypoint instead of the codebuffer base.
@@ -1166,6 +1167,25 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   return std::move(CodeData);
 }
 
+CPUBackend::CompiledCode Arm64JITCore::LoadCachedCode(std::span<const uint8_t> HostBytes, std::span<const DiskCache::BlobEntryPoint> EntryPoints) {
+  // we stored it aligned, better still be?
+  LOGMAN_THROW_A_FMT(HostBytes.size() % 16 == 0, "Needs to be 16B aligned!");
+  auto AllocatedInfo = AllocateCodeBufferInSharedCache(HostBytes.size());
+
+  uint8_t* Dest = AllocatedInfo.BufferAllocationOffset;
+  memcpy(Dest, HostBytes.data(), HostBytes.size());
+  ClearICache(Dest, HostBytes.size());
+
+  CPUBackend::CompiledCode Result;
+  Result.BlockBegin = Dest;
+  Result.Size = HostBytes.size();
+  Result.HostCodeOffset = Dest - CurrentCodeBuffer->Ptr;
+  for (const auto& Ep : EntryPoints) {
+    Result.EntryPoints[Ep.GuestRIP] = Dest + Ep.HostOffset;
+  }
+  return Result;
+}
+
 void Arm64JITCore::ResetStack() {
   if (SpillSlots == 0) {
     return;

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -54,6 +54,9 @@ public:
   CPUBackend::CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                        FEXCore::Core::DebugData* DebugData, bool CheckTF) override;
 
+  [[nodiscard]]
+  CPUBackend::CompiledCode LoadCachedCode(std::span<const uint8_t> HostBytes, std::span<const DiskCache::BlobEntryPoint> EntryPoints) override;
+
   void ClearCache() override;
 
   void ClearRelocations() override {

FEXCore/include/FEXCore/Config/Config.h

@@ -116,11 +116,13 @@ namespace detail {
 FEX_DEFAULT_VISIBILITY void SetDataDirectory(std::string_view Path, bool Global);
 FEX_DEFAULT_VISIBILITY void SetConfigDirectory(const std::string_view Path, bool Global);
 FEX_DEFAULT_VISIBILITY void SetConfigFileLocation(std::string_view Path, bool Global);
+FEX_DEFAULT_VISIBILITY void SetCacheDirectory(const std::string_view Path);
 
 FEX_DEFAULT_VISIBILITY const fextl::string& GetDataDirectory(bool Global = false);
 FEX_DEFAULT_VISIBILITY const fextl::string& GetConfigDirectory(bool Global);
 FEX_DEFAULT_VISIBILITY const fextl::string& GetConfigFileLocation(bool Global = false);
 FEX_DEFAULT_VISIBILITY fextl::string GetApplicationConfig(const std::string_view Program, bool Global);
+FEX_DEFAULT_VISIBILITY const fextl::string& GetCacheDirectory();
 
 using LayerValue = std::variant< fextl::string, StringArrayType, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t, bool >;
 

FEXCore/include/FEXCore/Core/Context.h

@@ -136,6 +136,8 @@ public:
   FEX_DEFAULT_VISIBILITY virtual FEXCore::CPUID::XCRResults RunXCRFunction(uint32_t Function) = 0;
   FEX_DEFAULT_VISIBILITY virtual FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) = 0;
 
+  virtual void InitDiskCache() = 0;
+
   virtual AbstractCodeCache& GetCodeCache() = 0;
   virtual void SetCodeMapWriter(fextl::unique_ptr<CodeMapWriter>) = 0;
   virtual void FlushAndCloseCodeMap() = 0;

FEXCore/include/FEXCore/Core/DiskCache.h

@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include "FEXCore/Core/CodeCache.h"
+#include "FEXCore/Core/Context.h"
+#include "Interface/Core/JIT/Relocations.h"
+#include "Interface/Core/Frontend.h"
+#include "Interface/Core/CPUBackend.h"
+#include "FEXCore/Config/Config.h"
+#include "FEXCore/Utils/File.h"
+#include "FEXCore/fextl/memory.h"
+#include <FEXCore/fextl/string.h>
+#include <FEXCore/fextl/unordered_set.h>
+#include <FEXCore/fextl/robin_map.h>
+#include <FEXCore/fextl/vector.h>
+#include <stdint.h>
+#include <mutex>
+#include <optional>
+#include <span>
+#include <xxhash.h>
+
+namespace FEXCore {
+
+namespace Context {
+  class ContextImpl;
+}
+
+namespace DiskCache {
+
+  namespace MesaFOZ {
+
+#define FOSSILIZE_BLOB_HASH_LENGTH 40 /* SHA1 hexadecimal string length */
+
+    struct __attribute__((packed)) foz_payload_key {
+      uint8_t bytes[FOSSILIZE_BLOB_HASH_LENGTH];
+    };
+
+    struct __attribute__((packed)) foz_payload_header {
+      uint32_t payload_size;
+      uint32_t format;
+      uint32_t crc;
+      uint32_t uncompressed_size;
+    };
+
+  } // namespace MesaFOZ
+
+  class IndexedDB;
+
+  struct IndexEntry {
+    IndexedDB* DB;
+    uint64_t Offset;
+    uint32_t Size;
+  };
+
+  struct __attribute__((packed)) BlobFixedHeader {
+    uint32_t GuestSize;
+    uint32_t HostSize;
+    uint32_t EntryPointCount;
+    uint32_t SmallRelocCount;
+    uint32_t ThunkRelocCount;
+    uint32_t TouchedGuestPagesCount;
+    XXH128_hash_t GuestHash;
+  };
+
+  struct __attribute__((packed)) BlobEntryPoint {
+    uint64_t GuestRIP; // todo those have been made relative since i wrote this, can we get away with less size here?
+    uint32_t HostOffset;
+  };
+
+  // packed struct for types 0, 2 and 3. type 1 is bigger and separate below
+  struct __attribute__((packed)) BlobSmallRelocation {
+    uint32_t Offset;
+    uint8_t Type;
+    union {
+      struct __attribute__((packed)) {
+        uint32_t Symbol;
+      } Named;
+      struct __attribute__((packed)) {
+        uint64_t GuestRIP;
+      } RIPLiteral;
+      struct __attribute__((packed)) {
+        uint8_t RegisterIndex;
+        uint64_t GuestRIP;
+      } RIPMove;
+    };
+  };
+
+  // type 1, implicit
+  struct __attribute__((packed)) BlobThunkRelocation {
+    uint32_t Offset;
+    uint8_t RegisterIndex;
+    uint8_t SymbolHash[32]; // sha256sum in the real RelocNamedThunkMove
+  };
+
+  struct CodeHitData {
+    fextl::vector<uint8_t> Blob;
+    std::span<uint8_t> HostCode;
+    std::span<const BlobEntryPoint> EntryPoints;
+    fextl::vector<FEXCore::CPU::Relocation> Relocations;
+    fextl::vector<uint64_t> GuestPages;
+
+    // the spans above point to memory owned by the Blob vec, so it's important this can't be copied
+    CodeHitData() = default;
+    CodeHitData(CodeHitData&&) = default;
+    CodeHitData& operator=(CodeHitData&&) = default;
+    CodeHitData(const CodeHitData&) = delete;
+    CodeHitData& operator=(const CodeHitData&) = delete;
+  };
+
+  using Index = fextl::robin_map<uint64_t, IndexEntry>;
+
+  class FOZFile {
+  public:
+    bool Open(const fextl::string& CacheFileName, bool ReadOnly);
+    bool Lock(uint32_t TimeoutMS) {
+      if (!FD) {
+        return false;
+      }
+      return FD->Lock(TimeoutMS);
+    }
+    bool Unlock() {
+      if (!FD) {
+        return false;
+      }
+      return FD->Unlock();
+    }
+    bool ReadNextBlob(MesaFOZ::foz_payload_key& OutKey, MesaFOZ::foz_payload_header& OutHeader, fextl::vector<uint8_t>& OutBlob);
+    bool ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
+    bool WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset);
+
+  private:
+    static constexpr uint32_t OPEN_LOCK_TIMEOUT_MS = 100;
+
+    fextl::string FileName;
+    fextl::unique_ptr<File::File> FD;
+    bool ReadOnly = false;
+  };
+
+  class IndexedDB {
+  public:
+    bool Open(const fextl::string& CacheDBName, bool ReadOnly);
+    void PopulateIndex(Index& CacheIndex);
+    bool ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
+    bool StoreCacheBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, Index& CacheIndex);
+
+  private:
+    // give up after 2ms of trying to store - when we have an async thread we can increase this
+    static constexpr uint32_t STORE_LOCK_TIMEOUT_MS = 2;
+
+    FOZFile CacheFOZ;
+    FOZFile IndexFOZ;
+    bool ReadOnly = false;
+  };
+
+  class DiskCache {
+  public:
+    void Init(FEXCore::Context::ContextImpl* CTX);
+
+    std::optional<CodeHitData> Lookup(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP);
+    bool Store(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP,
+               std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
+               std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo);
+
+    bool IsWritingDiskCache() const {
+      return (bool)RWCacheDB;
+    }
+    bool IsReadingDiskCache() const {
+      return !ROCacheDBs.empty() || RWCacheDB != nullptr;
+    }
+
+  private:
+    bool OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly);
+
+    FEXCore::Context::ContextImpl* CTX;
+    fextl::vector<fextl::unique_ptr<IndexedDB>> ROCacheDBs;
+    fextl::unique_ptr<IndexedDB> RWCacheDB;
+    Index Index;
+    std::mutex Lock;
+
+    FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE);
+    FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER);
+    FEX_CONFIG_OPT(BasePathOverride, DISKCACHEPATH);
+    FEX_CONFIG_OPT(RODBNames, DISKCACHERODBNAMES);
+  };
+
+} // namespace DiskCache
+
+} // namespace FEXCore

FEXCore/include/FEXCore/Utils/File.h

@@ -4,9 +4,13 @@
 #include <FEXCore/fextl/string.h>
 #include <FEXCore/Utils/EnumOperators.h>
 
+#include <chrono>
+#include <thread>
+
 #ifndef _WIN32
 #include <fcntl.h>
 #include <unistd.h>
+#include <sys/file.h>
 #else
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
@@ -110,6 +114,38 @@ public:
 #endif
   }
 
+  bool Lock(uint32_t TimeoutMS) {
+    for (uint32_t i = 0;; ++i) {
+      if (TryLock()) {
+        return true;
+      }
+      if (i >= TimeoutMS) {
+        return false;
+      }
+      std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+  }
+
+  bool Unlock() {
+    if (!Locked) {
+      return false; // we could return true here :thonk:
+    }
+#ifndef _WIN32
+    if (flock(Handle, LOCK_UN) == -1) {
+      return false;
+    }
+#else
+    OVERLAPPED Overlapped {};
+    Overlapped.Offset = static_cast<DWORD>(LOCK_SENTINEL_OFFSET);
+    Overlapped.OffsetHigh = static_cast<DWORD>(LOCK_SENTINEL_OFFSET >> 32);
+    if (!UnlockFileEx(Handle, 0, 1, 0, &Overlapped)) {
+      return false;
+    }
+#endif
+    Locked = false;
+    return true;
+  }
+
   ~File() {
     if (!IsValidHandle) {
       return;
@@ -196,10 +232,32 @@ protected:
     , IsValidHandle {true}
     , Handle {Handle} {}
 private:
+  bool TryLock() {
+    if (Locked) {
+      return true;
+    }
+#ifndef _WIN32
+    if (flock(Handle, LOCK_EX | LOCK_NB) == -1) {
+      return false;
+    }
+#else
+    // mimic posix advisory-only lock by locking some unattainably-high bit
+    OVERLAPPED Overlapped {};
+    Overlapped.Offset = static_cast<DWORD>(LOCK_SENTINEL_OFFSET);
+    Overlapped.OffsetHigh = static_cast<DWORD>(LOCK_SENTINEL_OFFSET >> 32);
+    if (!LockFileEx(Handle, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &Overlapped)) {
+      return false;
+    }
+#endif
+    Locked = true;
+    return true;
+  }
+
   bool ShouldClose {};
   bool IsValidHandle {};
 
   FileHandleType Handle {};
+  bool Locked = false;
 #ifndef _WIN32
   static constexpr int DEFAULT_USER_PERMS = S_IRWXU | S_IRWXG | S_IRWXO;
 
@@ -240,6 +298,8 @@ private:
   }
 #else
   static constexpr int DEFAULT_SHARE_MODE = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
+  static constexpr uint64_t LOCK_SENTINEL_OFFSET = 1ULL << 62;
+
   struct Disposition {
     uint32_t CreationFlag;
     uint32_t Access;
@@ -254,7 +314,11 @@ private:
       Disp.Access |= GENERIC_WRITE;
     }
     if ((Modes & FileModes::CREATE) == FileModes::CREATE) {
-      Disp.CreationFlag = CREATE_ALWAYS;
+      if ((Modes & FileModes::TRUNCATE) == FileModes::TRUNCATE) {
+        Disp.CreationFlag = CREATE_ALWAYS;
+      } else {
+        Disp.CreationFlag = OPEN_ALWAYS;
+      }
     } else {
       Disp.CreationFlag = OPEN_ALWAYS;
     }

Scripts/reformat.sh

@@ -35,13 +35,13 @@ if [ "$CHANGED_ONLY" = true ]; then
 
     CHANGED_FILES=$(git ls-files -m '*.cpp' '*.h' '*.inl')
     if [ -n "$CHANGED_FILES" ]; then
-        echo "$CHANGED_FILES" | xargs -d '\n' -n 1 -P "$(nproc)" clang-format-19 -i
+        echo "$CHANGED_FILES" | xargs -d '\n' -n 1 -P "$(nproc)" $CLANG_FORMAT -i
     else
         echo "No changed files to format."
     fi
 else
     # Reformat whole tree (original behavior)
-    git ls-files -z '*.cpp' '*.h' '*.inl' | xargs -0 -n 1 -P "$(nproc)" clang-format-19 -i
+    git ls-files -z '*.cpp' '*.h' '*.inl' | xargs -0 -n 1 -P "$(nproc)" $CLANG_FORMAT -i
 fi
 
 cd "$DIR"

Source/Common/Config.cpp

@@ -741,5 +741,6 @@ void InitializeConfigs(const PortableInformation& PortableInfo) {
   FEXCore::Config::SetConfigDirectory(GetConfigDirectory(true, PortableInfo), true);
   FEXCore::Config::SetConfigFileLocation(GetConfigFileLocation(false, PortableInfo), false);
   FEXCore::Config::SetConfigFileLocation(GetConfigFileLocation(true, PortableInfo), true);
+  FEXCore::Config::SetCacheDirectory(GetCacheDirectory());
 }
 } // namespace FEX::Config

Source/Tools/FEXServer/SquashFS.cpp

@@ -40,7 +40,7 @@ static bool InitializeSquashFSPipe() {
     ServerRootFSLockFD = open(RootFSLockFile.c_str(), O_RDWR | O_CLOEXEC, USER_PERMS);
     if (ServerRootFSLockFD != -1) {
       // Now that we have opened the file, try to get a write lock.
-      flock lk {
+      struct flock lk {
         .l_type = F_WRLCK,
         .l_whence = SEEK_SET,
         .l_start = 0,
@@ -67,7 +67,7 @@ static bool InitializeSquashFSPipe() {
     return false;
   } else {
     // FIFO file was created. Try to get a write lock
-    flock lk {
+    struct flock lk {
       .l_type = F_WRLCK,
       .l_whence = SEEK_SET,
       .l_start = 0,
@@ -87,7 +87,7 @@ static bool InitializeSquashFSPipe() {
 }
 
 static bool DowngradeRootFSPipeToReadLock() {
-  flock lk {
+  struct flock lk {
     .l_type = F_RDLCK,
     .l_whence = SEEK_SET,
     .l_start = 0,

Source/Windows/Common/WinAPI/IO.cpp

@@ -187,6 +187,37 @@ DLLEXPORT_FUNC(WINBOOL, FlushFileBuffers, (HANDLE hFile)) {
   UNIMPLEMENTED();
 }
 
+DLLEXPORT_FUNC(
+  WINBOOL, LockFileEx,
+  (HANDLE hFile, DWORD dwFlags, DWORD dwReserved, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh, LPOVERLAPPED lpOverlapped)) {
+  if (!lpOverlapped || lpOverlapped->hEvent) {
+    UNIMPLEMENTED();
+  }
+  LARGE_INTEGER ByteOffset;
+  ByteOffset.QuadPart = (static_cast<LONGLONG>(lpOverlapped->OffsetHigh) << 32) | lpOverlapped->Offset;
+  LARGE_INTEGER Length;
+  Length.QuadPart = (static_cast<LONGLONG>(nNumberOfBytesToLockHigh) << 32) | nNumberOfBytesToLockLow;
+  // IoStatusBlock must be null on Wine :harold:
+  NTSTATUS Status = NtLockFile(hFile, nullptr, nullptr, nullptr, nullptr, &ByteOffset, &Length, 0, !!(dwFlags & LOCKFILE_FAIL_IMMEDIATELY),
+                               !!(dwFlags & LOCKFILE_EXCLUSIVE_LOCK));
+  return WinAPIReturn(Status);
+}
+
+DLLEXPORT_FUNC(WINBOOL, UnlockFileEx,
+               (HANDLE hFile, DWORD dwReserved, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh, LPOVERLAPPED lpOverlapped)) {
+  if (!lpOverlapped || lpOverlapped->hEvent) {
+    UNIMPLEMENTED();
+  }
+  LARGE_INTEGER ByteOffset;
+  ByteOffset.QuadPart = (static_cast<LONGLONG>(lpOverlapped->OffsetHigh) << 32) | lpOverlapped->Offset;
+  LARGE_INTEGER Length;
+  Length.QuadPart = (static_cast<LONGLONG>(nNumberOfBytesToUnlockHigh) << 32) | nNumberOfBytesToUnlockLow;
+  // IoStatusBlock must be non-null on Wine :harold:
+  IO_STATUS_BLOCK IoStatusBlock;
+  NTSTATUS Status = NtUnlockFile(hFile, &IoStatusBlock, &ByteOffset, &Length, 0);
+  return WinAPIReturn(Status);
+}
+
 DLLEXPORT_FUNC(DWORD, GetFinalPathNameByHandleA, (HANDLE hFile, LPSTR lpszFilePath, DWORD cchFilePath, DWORD dwFlags)) {
   UNIMPLEMENTED();
 }

Source/Windows/include/winternl.h

@@ -524,6 +524,7 @@ NTSTATUS WINAPI NtFlushInstructionCache(HANDLE, LPCVOID, SIZE_T);
 NTSTATUS WINAPI NtFreeVirtualMemory(HANDLE, PVOID*, SIZE_T*, ULONG);
 NTSTATUS WINAPI NtGetContextThread(HANDLE, CONTEXT*);
 ULONG WINAPI NtGetCurrentProcessorNumber(void);
+NTSTATUS WINAPI NtLockFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, PLARGE_INTEGER, PLARGE_INTEGER, ULONG, BOOLEAN, BOOLEAN);
 NTSYSAPI NTSTATUS WINAPI NtMapViewOfSection(HANDLE, HANDLE, PVOID*, ULONG_PTR, SIZE_T, const LARGE_INTEGER*, SIZE_T*, SECTION_INHERIT, ULONG, ULONG);
 NTSYSAPI NTSTATUS WINAPI NtUnmapViewOfSection(HANDLE, PVOID);
 NTSTATUS WINAPI NtOpenKeyEx(PHANDLE, ACCESS_MASK, const OBJECT_ATTRIBUTES*, ULONG);
@@ -537,6 +538,7 @@ NTSTATUS WINAPI NtReadFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BL
 NTSTATUS WINAPI NtSetContextThread(HANDLE, const CONTEXT*);
 NTSTATUS WINAPI NtSuspendThread(HANDLE, PULONG);
 NTSTATUS WINAPI NtTerminateProcess(HANDLE, LONG);
+NTSTATUS WINAPI NtUnlockFile(HANDLE, PIO_STATUS_BLOCK, PLARGE_INTEGER, PLARGE_INTEGER, ULONG);
 NTSTATUS WINAPI NtWriteFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, const void*, ULONG, PLARGE_INTEGER, PULONG);
 void WINAPI ProcessPendingCrossProcessEmulatorWork();
 void WINAPI RtlAcquirePebLock(void);