// SPDX-License-Identifier: MIT

#define XXH_STATIC_LINKING_ONLY

#include "Interface/Core/Frontend.h"
#include "Interface/Context/Context.h"

#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/DiskCache.h>
#include <FEXCore/Core/DiskCacheFileMapper.h>
#include <FEXCore/fextl/memory.h>
#include <FEXCore/fextl/string.h>
#include <FEXCore/HLE/SyscallHandler.h>
#include <FEXCore/Utils/File.h>
#include <FEXCore/Utils/FileUtils.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Profiler.h>
#include <FEXCore/Utils/TypeDefines.h>
#include <FEXHeaderUtils/Filesystem.h>

#include <cstdint>
#include <cstring>
#include <atomic>

namespace FEXCore {

namespace DiskCache {

  namespace MesaFOZ {

    enum { FOSSILIZE_COMPRESSION_NONE = 1, FOSSILIZE_COMPRESSION_DEFLATE = 2 };

    enum { FOSSILIZE_FORMAT_VERSION = 6, FOSSILIZE_FORMAT_MIN_COMPAT_VERSION = 5 };

#define FOZ_REF_MAGIC_SIZE 16

    static const uint8_t stream_reference_magic_and_version[FOZ_REF_MAGIC_SIZE] = {
      0x81, 'F', 'O', 'S', 'S', 'I', 'L', 'I', 'Z', 'E', 'D', 'B', 0, 0, 0, FOSSILIZE_FORMAT_VERSION, /* 4 bytes to use for versioning. */
    };

    struct __attribute__((packed)) mesa_index_db_file_entry {
      uint64_t hash;
      uint32_t size;
      uint64_t last_access_time;
      uint64_t cache_db_file_offset;
    };

  } // namespace MesaFOZ

  struct __attribute__((packed)) IndexExtraBlobHeader {
    XXH128_hash_t GuestHash; // StoreCacheBlob will assume it's here, care if moving
    uint64_t GuestFootprint;
    uint32_t GuestSize;
    uint32_t GuestExtentsCount;
  };

  static FileMapperFunc FileMapper = nullptr;

  bool FOZFile::Open(const fextl::string& FOZFileName, bool ReadOnly) {
    FileName = FOZFileName;
    this->ReadOnly = ReadOnly;

    File::FileModes Modes = File::FileModes::READ;
    if (!ReadOnly) {
      Modes = Modes | File::FileModes::WRITE | File::FileModes::CREATE;
    }
    FD = fextl::make_unique<File::File>(FileName.c_str(), Modes, false);
    if (!FD->IsValid()) {
      FD.reset();
      return false;
    }

    bool Valid = false;
    bool TookLock = false;
    ssize_t Size = FD->Size();

    if (Size < FOZ_REF_MAGIC_SIZE && !ReadOnly) {
      if (!FD->Lock(OPEN_LOCK_TIMEOUT_MS)) {
        FD.reset();
        return false;
      }
      TookLock = true;
      // check size again in case someone else made it while we waited
      Size = FD->Size();
    }

    if (Size == 0 && !ReadOnly) {
      Valid = FD->PWrite(MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE, 0) == FOZ_REF_MAGIC_SIZE;
    } else {
      uint8_t magic[FOZ_REF_MAGIC_SIZE];
      if (FD->PRead(magic, FOZ_REF_MAGIC_SIZE, 0) == FOZ_REF_MAGIC_SIZE &&
          memcmp(magic, MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE - 1) == 0) {
        int version = magic[FOZ_REF_MAGIC_SIZE - 1];
        Valid = version <= MesaFOZ::FOSSILIZE_FORMAT_VERSION && version >= MesaFOZ::FOSSILIZE_FORMAT_MIN_COMPAT_VERSION;
      }
    }

    if (TookLock) {
      FD->Unlock();
    }

    if (!Valid) {
      FD.reset();
    }
    return Valid;
  }

  ssize_t FOZFile::Size() {
    return FD ? FD->Size() : -1;
  }

  bool FOZFile::ReadAll(fextl::vector<uint8_t>& Out) {
    ssize_t FileSize = Size();
    if (FileSize < FOZ_REF_MAGIC_SIZE) {
      return false;
    }
    Out.resize((size_t)FileSize - FOZ_REF_MAGIC_SIZE);
    return FD->PRead(Out.data(), Out.size(), FOZ_REF_MAGIC_SIZE) == (ssize_t)Out.size();
  }

  bool FOZFile::ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
    if (FD->PRead(OutBlob.data(), OutBlob.size(), Offset) != (ssize_t)OutBlob.size()) {
      return false;
    }
    return true;
  }

  bool FOZFile::WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset) {
    ssize_t FileSize = FD->Size();
    if (FileSize < 0) {
      return false;
    }
    uint64_t WriteOffset = (uint64_t)FileSize;

    if (FD->PWrite(Key.bytes, sizeof(Key.bytes), WriteOffset) != sizeof(Key.bytes)) {
      return false;
    }
    WriteOffset += sizeof(Key.bytes);

    uint64_t TotalBlobSize = 0;
    for (const std::span<const uint8_t>& Chunk : BlobChunks) {
      TotalBlobSize += Chunk.size();
    }

    MesaFOZ::foz_payload_header ScratchHeader {.payload_size = (uint32_t)TotalBlobSize,
                                               .format = MesaFOZ::FOSSILIZE_COMPRESSION_NONE,
                                               .crc = 0, // todo? maybe
                                               .uncompressed_size = (uint32_t)TotalBlobSize};

    if (FD->PWrite(&ScratchHeader, sizeof(ScratchHeader), WriteOffset) != sizeof(ScratchHeader)) {
      return false;
    }
    WriteOffset += sizeof(ScratchHeader);

    OutBlobOffset = WriteOffset;

    for (const std::span<const uint8_t>& Chunk : BlobChunks) {
      if (Chunk.size() == 0) {
        continue;
      }
      if (FD->PWrite(Chunk.data(), Chunk.size(), WriteOffset) != (ssize_t)Chunk.size()) {
        return false;
      }
      WriteOffset += Chunk.size();
    }

    return true;
  }

  bool IndexedDB::Open(const fextl::string& CacheDBName, bool ReadOnly) {
    if (!CacheFOZ.Open(CacheDBName + ".foz", ReadOnly)) {
      return false;
    }
    if (!IndexFOZ.Open(CacheDBName + "_idx.foz", ReadOnly)) {
      return false;
    }

    File::File::FileHandleType CacheFileHandle = CacheFOZ.GetHandle();
    if (FileMapper && CacheFileHandle != (File::File::FileHandleType)-1) {
      CacheFileMapping = reinterpret_cast<uint8_t*>(FileMapper(CacheFileHandle, ReadOnly ? CacheFOZ.Size() : BIG_MAPPING_SIZE));
      CacheFileSize = CacheFOZ.Size();
    }

    this->ReadOnly = ReadOnly;
    return true;
  }

  void IndexedDB::PopulateIndex(Index& CacheIndex, bool& FoundMetadata) {
    fextl::vector<uint8_t> Data;
    if (!IndexFOZ.ReadAll(Data)) {
      return;
    }

    ssize_t CacheFOZSize = CacheFOZ.Size();
    if (CacheFOZSize < 0) {
      return;
    }

    const uint8_t* IndexDataStart = Data.data();
    const size_t IndexDataSize = Data.size();
    size_t ReadOffset = 0;
    while (ReadOffset + sizeof(MesaFOZ::foz_payload_key) + sizeof(MesaFOZ::foz_payload_header) <= IndexDataSize) {
      const auto* FOZKey = reinterpret_cast<const MesaFOZ::foz_payload_key*>(IndexDataStart + ReadOffset);
      ReadOffset += sizeof(MesaFOZ::foz_payload_key);
      const auto* FOZHeader = reinterpret_cast<const MesaFOZ::foz_payload_header*>(IndexDataStart + ReadOffset);
      ReadOffset += sizeof(MesaFOZ::foz_payload_header);

      uint64_t IndexBlobSize = sizeof(MesaFOZ::mesa_index_db_file_entry) + sizeof(IndexExtraBlobHeader);

      if (FOZHeader->payload_size < IndexBlobSize || ReadOffset + FOZHeader->payload_size > IndexDataSize) {
        break;
      }
      const auto* IndexBlobCommon = reinterpret_cast<const MesaFOZ::mesa_index_db_file_entry*>(IndexDataStart + ReadOffset);
      ReadOffset += sizeof(MesaFOZ::mesa_index_db_file_entry);
      const auto* IndexBlobExtra = reinterpret_cast<const IndexExtraBlobHeader*>(IndexDataStart + ReadOffset);
      ReadOffset += sizeof(IndexExtraBlobHeader);

      // skip corrupt (carefully) so we don't have to figure that out in the hot path later
      if (IndexBlobCommon->cache_db_file_offset > (uint64_t)CacheFOZSize ||
          IndexBlobCommon->size > (uint64_t)CacheFOZSize - IndexBlobCommon->cache_db_file_offset) {
        continue;
      }
      if (IndexBlobExtra->GuestSize + sizeof(BlobFixedHeader) > IndexBlobCommon->size) {
        continue;
      }
      IndexBlobSize += IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t);
      if (FOZHeader->payload_size != IndexBlobSize) {
        break;
      }
      const auto* GuestExtents = reinterpret_cast<const uint32_t*>(IndexDataStart + ReadOffset);
      ReadOffset += IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t);
      if (FOZKey->bytes[39] != 0xFF) {
        IndexEntry NewEntry {this, IndexBlobCommon->cache_db_file_offset, IndexBlobCommon->size, IndexBlobExtra->GuestSize,
                             IndexBlobExtra->GuestHash};
        NewEntry.GuestExtents.resize(IndexBlobExtra->GuestExtentsCount);
        memcpy(NewEntry.GuestExtents.data(), GuestExtents, IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t));
        bool ExtentsValid = true;
        for (uint32_t i = 0; i < NewEntry.GuestExtents.size(); i += 2) {
          if ((uint64_t)NewEntry.GuestExtents[i] + NewEntry.GuestExtents[i + 1] > IndexBlobExtra->GuestSize) {
            ExtentsValid = false;
            break;
          }
        }
        if (!ExtentsValid) {
          continue;
        }
        auto It = CacheIndex.find(IndexBlobCommon->hash);
        if (It == CacheIndex.end()) {
          CacheIndex.emplace(IndexBlobCommon->hash, IndexCacheHead {std::move(NewEntry), IndexBlobExtra->GuestFootprint, nullptr});
        } else {
          if (!It->second.MoreEntries.get()) {
            It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, IndexEntry>>();
          }
          It->second.MoreEntries->insert({IndexBlobExtra->GuestFootprint, std::move(NewEntry)});
        }
      } else {
        FoundMetadata = true;
      }
    }
    // could truncate/delete index if we don't end up perfectly at end here
  }

  bool IndexedDB::ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
    if (CacheFileMapping && (ReadOnly || Offset + OutBlob.size() <= BIG_MAPPING_SIZE)) {
      if (Offset + OutBlob.size() > CacheFileSize) {
        return false;
      }
      // todo could reduce copies by having a private mapping for relocs, etc
      memcpy(OutBlob.data(), CacheFileMapping + Offset, OutBlob.size());
      return true;
    } else {
      return CacheFOZ.ReadBlob(Offset, OutBlob);
    }
  }

  bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob,
                                 MesaFOZ::mesa_index_db_file_entry& IndexEntry, std::span<const uint8_t> IndexBlob) {
    if (ReadOnly) {
      // shouldn't happen
      return false;
    }

    if (MaxSizeReached || CacheFileSize + Blob.size() >= MaxFileSize) {
      MaxSizeReached = true;
      return false;
    }

    if (!CacheFOZ.Lock(STORE_LOCK_TIMEOUT_MS) || !IndexFOZ.Lock(STORE_LOCK_TIMEOUT_MS)) {
      CacheFOZ.Unlock();
      IndexFOZ.Unlock();
      return false;
    }

    // write cache side first so we get offset for index
    std::span<const uint8_t> BlobChunks[] = {Blob};
    uint64_t BlobOffset = 0;
    if (!CacheFOZ.WriteBlob(UniqueKey, BlobChunks, BlobOffset)) {
      CacheFOZ.Unlock();
      IndexFOZ.Unlock();
      return false;
    }

    IndexEntry = {.hash = LookupKey,
                  .size = (uint32_t)Blob.size(),
                  .last_access_time = 0, // todo..
                  .cache_db_file_offset = BlobOffset};

    std::span<const uint8_t> IndexBlobChunks[] = {
      {(const uint8_t*)&IndexEntry, sizeof(IndexEntry)},
      IndexBlob,
    };
    uint64_t UnusedIndexBlobOffset = 0;
    if (!IndexFOZ.WriteBlob(UniqueKey, IndexBlobChunks, UnusedIndexBlobOffset)) {
      CacheFOZ.Unlock();
      IndexFOZ.Unlock();
      return false;
    }

    CacheFOZ.Unlock();
    IndexFOZ.Unlock();

    // publish new file size for memory-mapped reads
    if (CacheFileMapping) {
      CacheFileSize = BlobOffset + Blob.size();
    }

    return true;
  }

  bool DiskCache::OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly) {
    fextl::unique_ptr<IndexedDB> CurDB;

    if (!ReadOnly && RWCacheDB) {
      // rw already opened, just support one
      return false;
    }

    CurDB = fextl::make_unique<IndexedDB>();
    if (!CurDB) {
      return false;
    }

    if (!CurDB->Open(CacheDBName, ReadOnly)) {
      CurDB.reset();
      return false;
    }

    CurDB->PopulateIndex(Index, FoundMetadata);

    if (ReadOnly) {
      ROCacheDBs.push_back(std::move(CurDB));
    } else {
      RWCacheDB = std::move(CurDB);
    }

    return true;
  }

  FEX_DEFAULT_VISIBILITY void SetFileMapper(FileMapperFunc Func) {
    FileMapper = Func;
  }

  static inline void PruneStaleEntries(std::string_view CacheBase, std::string_view MachineBucketHash) {
    FEXCORE_PROFILE_SCOPED("DiskCache::PruneStaleEntries");
    FEX_CONFIG_OPT(DiskCachePruneStaleEntries, DISKCACHEPRUNESTALEENTRIES);

    if (!DiskCachePruneStaleEntries) {
      return;
    }

    struct SimpleCapture {
      std::string_view CacheBase, MachineBucketHash;
    } const SimpleCapture {
      .CacheBase = CacheBase,
      .MachineBucketHash = MachineBucketHash,
    };

    FEXCore::FileUtils::WalkDirectory(
      CacheBase,
      [](std::string_view name, bool is_dir, const void* user_data) {
        if (!is_dir) {
          return;
        }

        auto capture = reinterpret_cast<const struct SimpleCapture*>(user_data);

        // Current behaviour is to remove entries that no longer match the MachineBucketHash.
        // This means that if the Disk Cache version no longer matches, or the FEXCore::HostFeatures differ, then they get removed.
        if (name != capture->MachineBucketHash) {
          FEXCore::FileUtils::RecursiveRemoveDirectory(fextl::fmt::format("{}/{}", capture->CacheBase, name));
        }
      },
      &SimpleCapture);
  }

  void DiskCache::Init(FEXCore::Context::ContextImpl* CTX) {
    this->CTX = CTX;

    if (!EnableDiskCache) {
      return;
    }

    fextl::string SerializedConfig = FEXCore::Config::SerializeForCache();

    const auto HostFeatureHash = CTX->HostFeatures.HashForCaching();

    struct __attribute__((packed)) {
      uint16_t FormatVersion;
      uint64_t HostFeaturesHash;
    } MachineBucketData = {FormatVersion, HostFeatureHash.HostFeaturesHash};

    fextl::vector<uint8_t> BucketBytes(sizeof(MachineBucketData) + (sizeof(uint8_t) * 2) + SerializedConfig.size());
    memcpy(BucketBytes.data(), &MachineBucketData, sizeof(MachineBucketData));

    // 64-bit mode and HostType is in the ProcessBucket hash only instead of the MachineBucketHash
    // These effect code-gen, but they aren't part of the MachineBucketHash, as it comes from Process state.
    BucketBytes[sizeof(MachineBucketData)] = CTX->Config.Is64BitMode;
    BucketBytes[sizeof(MachineBucketData) + 1] = FEXCore::ToUnderlying(HostFeatureHash.HostType);
    memcpy(BucketBytes.data() + sizeof(MachineBucketData) + 2, SerializedConfig.data(), SerializedConfig.size());

    uint64_t MachineBucketHash = XXH3_64bits(BucketBytes.data(), sizeof(MachineBucketData));
    uint64_t ProcessBucketHash = XXH3_64bits(BucketBytes.data() + sizeof(MachineBucketData), 2 + SerializedConfig.size());
    BucketHash.high64 = MachineBucketHash;
    BucketHash.low64 = ProcessBucketHash;

    fextl::string BasePath = BasePathOverride();
    if (BasePath.empty()) {
      BasePath = FEXCore::Config::GetCacheDirectory() + "DiskCache/";
    }

    const auto MachineBucketHashAsString = fextl::fmt::format("{:016x}", MachineBucketHash);
    PruneStaleEntries(BasePath, MachineBucketHashAsString);

    BasePath += MachineBucketHashAsString + "/";
    FHU::Filesystem::CreateDirectories(BasePath);

    if (!MapDiskCacheFiles) {
      FileMapper = nullptr;
    }

    const auto RWDBBasePath = fextl::fmt::format("{}RWCacheDB_{:016x}", BasePath, ProcessBucketHash);
    OpenCacheDB(RWDBBasePath, false);

    if (RWCacheDB && !FoundMetadata) {
      // we just opened a fresh cache, add a metadata blob
      MesaFOZ::foz_payload_key MetadataKey;
      memset(MetadataKey.bytes, 0xFF, sizeof(MetadataKey));
      IndexExtraBlobHeader MetaDataHeader = {};
      MesaFOZ::mesa_index_db_file_entry IndexEntry = {};
      RWCacheDB->StoreCacheBlob(MetadataKey, ~0, {BucketBytes.data(), BucketBytes.size()}, IndexEntry,
                                {reinterpret_cast<uint8_t*>(&MetaDataHeader), sizeof(MetaDataHeader)});
      Index.erase(~0);
    }

    std::string_view RONames = RODBNames();
    while (!RONames.empty()) {
      const auto Delim = RONames.find(',');
      const std::string_view ROName = RONames.substr(0, Delim);
      if (!ROName.empty()) {
        fextl::string RODBBasePath = BasePath;
        RODBBasePath += ROName;
        OpenCacheDB(RODBBasePath, true);
      }
      if (Delim == std::string_view::npos) {
        break;
      }
      // advance to next
      RONames.remove_prefix(Delim + 1);
    }

    WritingDiskCache = (bool)RWCacheDB;
    ReadingDiskCache = !ROCacheDBs.empty() || RWCacheDB != nullptr;

    if (IsWritingDiskCache()) {
      FEXCore::Threads::Flags WriterThreadFlags = {.LowPriority = true, .Internal = true};
      Writer = fextl::make_unique<WorkQueueThread>(WriterThreadFlags, "FEX:DiskCache");
    }
  }

  uint64_t DiskCache::MakeLookupKey(Core::InternalThreadState* Thread, const uint64_t CodeKey, bool Writable, bool MonoBackpatcher) {
    struct __attribute__((packed)) {
      uint64_t CodeKey;
      XXH128_hash_t BucketHash;
      uint8_t Flags;
    } BlobKeyBytes = {CodeKey, BucketHash, 0};

    if (Writable) {
      BlobKeyBytes.Flags |= 1 << 0;
    }
    if (CTX->AreMonoHacksActive()) {
      BlobKeyBytes.Flags |= 1 << 1;
    }
    if (Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC]) {
      BlobKeyBytes.Flags |= 1 << 2;
    }
    if (MonoBackpatcher) {
      BlobKeyBytes.Flags |= 1 << 3;
    }

    return XXH3_64bits(&BlobKeyBytes, sizeof(BlobKeyBytes));
  }

  struct DiskCache::PruneMemoryLRUWorkItem final : WorkQueueThread::WorkItem {
    DiskCache* Self;
    PruneMemoryLRUWorkItem(DiskCache* Self)
      : Self(Self) {}
    void Run() override {
      if (Self->MemoryLRUCurrentSize <= Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
        return;
      }
      std::lock_guard IndexGuard(Self->IndexLock);
      std::lock_guard LRUGuard(Self->MemoryLRULock);
      if (Self->MemoryLRU.empty()) {
        return;
      }
      auto Last = std::prev(Self->MemoryLRU.end());

      while (Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize && !Self->MemoryLRU.empty()) {
        auto LastKey = *Last;
        auto IndexEntry = Self->LookupLocked(LastKey.LookupKey, LastKey.GuestHash, LastKey.GuestFootprint);
        bool AtFront = (Last == Self->MemoryLRU.begin());
        if (IndexEntry && IndexEntry->MemoryBlob.use_count() > 1) {
          // being read rn, keep moving
          if (AtFront) {
            break;
          }
          Last--;
          continue;
        }
        if (IndexEntry) {
          IndexEntry->MemoryBlob.reset();
          IndexEntry->LRUEntry.reset();
        }
        Self->MemoryLRUCurrentSize -= LastKey.Size;
        auto Deleted = Last;
        if (AtFront) {
          Self->MemoryLRU.erase(Deleted);
          break;
        }
        Last--;
        Self->MemoryLRU.erase(Deleted);
      }
    }
  };

  std::optional<CodeHitData> DiskCache::Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region,
                                               uint64_t GuestRIP, std::optional<uint64_t>& GuestCodeKey) {
    if (!IsReadingDiskCache()) {
      return std::nullopt;
    }
    if (Region && Region->FileStartVA) {
      struct __attribute__((packed)) {
        uint64_t GuestOffset;
        uint64_t FileId;
      } FileBackedKey = {GuestRIP - Region->FileStartVA, Region->FileInfo.FileId};
      GuestCodeKey = XXH3_64bits(&FileBackedKey, sizeof(FileBackedKey));
    } else {
      if (!AnonCaching) {
        return std::nullopt;
      }
      Thread->FrontendDecoder->DecodeLoop(reinterpret_cast<const uint8_t*>(GuestRIP), AnonPrefixGuestBytes);
      const auto* BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();

      XXH3_state_t HashState;
      XXH3_64bits_reset(&HashState);
      for (auto& SubBlock : BlockInfo->Blocks) {
        if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
          return std::nullopt;
        }
        uint64_t HashStart = SubBlock.Entry;
        // skip over masked in-block data and data/etc gaps between blocks
        for (auto& DataMask : SubBlock.DataMasks) {
          XXH3_64bits_update(&HashState, reinterpret_cast<const uint8_t*>(HashStart), DataMask.FieldAddress - HashStart);
          HashStart = DataMask.FieldAddress + DataMask.ValueSize;
        }
        if (HashStart != SubBlock.Entry + SubBlock.Size) {
          XXH3_64bits_update(&HashState, reinterpret_cast<const uint8_t*>(HashStart), SubBlock.Size - (HashStart - SubBlock.Entry));
        }
      }
      GuestCodeKey = XXH3_64bits_digest(&HashState);
      // if (TotalSize < AnonPrefixGuestBytes) {
      //   GuestCodeKey = 0;
      //   return std::nullopt;
      // }
      // LogMan::Msg::IFmt("anon lookup! length {:d} {}", GuestCodeKey, TotalSize);
    }

    auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
    if (RangeInfo.Size == 0 || RangeInfo.Base > GuestRIP) {
      return std::nullopt;
    }
    uint64_t Available = RangeInfo.Base + RangeInfo.Size - GuestRIP;

    uint64_t LookupKey =
      MakeLookupKey(Thread, *GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));

    XXH128_hash_t LiveGuestHash;
    uint64_t LastFootprintHashed = 0;
    bool FirstHash = true; // 0 is probably a valid footprint hash so we need an extra bit there
    bool TriedMainEntry = false;
    IndexEntry MainEntry;
    uint64_t MainEntryFootprint;
    fextl::multimap<uint64_t, IndexEntry>::iterator MoreEntriesIt;
    fextl::multimap<uint64_t, IndexEntry>* MapPointer = nullptr;
    IndexEntry* EntryUnderReview;
    fextl::shared_ptr<fextl::vector<uint8_t>> BlobRef;
    std::optional<fextl::list<MemoryLRUKey>::iterator> LRUIter;
    uint64_t CurrentFootprint = 0;
    {
      std::lock_guard Guard(IndexLock);
      auto It = Index.find(LookupKey);
      if (It == Index.end()) {
        // definite miss
        return std::nullopt;
      }

      MainEntry = It->second.MainEntry;
      MainEntryFootprint = It->second.MainEntryFootprint;
      MapPointer = It->second.MoreEntries.get();
      if (MapPointer) {
        MoreEntriesIt = It->second.MoreEntries->begin();
      }
      if (MapPointer && MoreEntriesIt != It->second.MoreEntries->end()) {
        EntryUnderReview = &MoreEntriesIt->second;
        BlobRef = EntryUnderReview->MemoryBlob;
        LRUIter = EntryUnderReview->LRUEntry;
        CurrentFootprint = MoreEntriesIt->first;
      } else {
        EntryUnderReview = &MainEntry;
        BlobRef = EntryUnderReview->MemoryBlob;
        LRUIter = EntryUnderReview->LRUEntry;
        CurrentFootprint = MainEntryFootprint;
        TriedMainEntry = true;
      }
    }
    // found a lookup key match, check for guest hash match now
    bool FoundMatchingHash = false;
    bool Advance = false;
    while (!FoundMatchingHash) {
      if (Advance) {
        std::lock_guard Guard(IndexLock);

        EntryUnderReview = nullptr;
        if (MapPointer && MoreEntriesIt != MapPointer->end()) {
          // if the current footprint is also the main entry's footprint, give main entry a shot next
          if (!TriedMainEntry && MoreEntriesIt->first == MainEntryFootprint) {
            EntryUnderReview = &MainEntry;
            BlobRef = EntryUnderReview->MemoryBlob;
            LRUIter = EntryUnderReview->LRUEntry;
            CurrentFootprint = MainEntryFootprint;
            TriedMainEntry = true;
          } else {
            MoreEntriesIt++;
          }
        }
        if (!MapPointer || MoreEntriesIt == MapPointer->end()) {
          if (TriedMainEntry) {
            // ran out
            break;
          } else {
            EntryUnderReview = &MainEntry;
            BlobRef = EntryUnderReview->MemoryBlob;
            LRUIter = EntryUnderReview->LRUEntry;
            CurrentFootprint = MainEntryFootprint;
            TriedMainEntry = true;
          }
        }
        if (!EntryUnderReview && MapPointer && MoreEntriesIt != MapPointer->end()) {
          EntryUnderReview = &MoreEntriesIt->second;
          BlobRef = EntryUnderReview->MemoryBlob;
          LRUIter = EntryUnderReview->LRUEntry;
          CurrentFootprint = MoreEntriesIt->first;
        }
      }

      Advance = true;

      // entry not backed by anything right now - todo prune..
      if (!EntryUnderReview->DB && !BlobRef) {
        continue;
      }

      // do we have enough room in our live code to even hash GuestSize worth?
      if (Available < EntryUnderReview->GuestSize) {
        continue;
      }

      // if (Entry.GuestExtents.size()) {
      //   LogMan::Msg::IFmt("lookup! length {:d}", Entry.GuestSize);
      //   for(uint32_t i = 0; i < Entry.GuestExtents.size(); i+=2 ) {
      //     LogMan::Msg::IFmt("extent {} {}", Entry.GuestExtents[i], Entry.GuestExtents[i]+Entry.GuestExtents[i+1]);
      //   }
      // }
      {
        IndexEntry& Entry = *EntryUnderReview;
        // only renew the hash if the candidate footprint is different than previous
        if (FirstHash || LastFootprintHashed != CurrentFootprint) {
          XXH3_state_t HashState;
          XXH3_128bits_reset(&HashState);
          for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
            XXH3_128bits_update(&HashState, reinterpret_cast<uint8_t*>(GuestRIP) + Entry.GuestExtents[i], Entry.GuestExtents[i + 1]);
          }
          LiveGuestHash = XXH3_128bits_digest(&HashState);
          LastFootprintHashed = CurrentFootprint;
          FirstHash = false;
        }

        if (XXH128_isEqual(LiveGuestHash, Entry.GuestHash)) {
          FoundMatchingHash = true;
          break;
        } else if (Validation) {
          fextl::vector<uint8_t> GuestCode(Entry.GuestSize);
          if (Entry.Size >= Entry.GuestSize && Entry.DB && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
            const uint8_t* CachedGuest = GuestCode.data();
            const uint8_t* LiveGuest = reinterpret_cast<const uint8_t*>(GuestRIP);
            uint64_t DiffCount = 0;
            uint64_t DiffOffset = 0;
            for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
              uint32_t Begin = Entry.GuestExtents[i];
              uint32_t End = Begin + Entry.GuestExtents[i + 1];
              bool PreviousByteDiff = false;
              for (uint32_t Offset = Begin; Offset < End; Offset++) {
                if (LiveGuest[Offset] != CachedGuest[Offset]) {
                  if (DiffCount == 0) {
                    DiffOffset = Offset;
                  }
                  if (!PreviousByteDiff) {
                    DiffCount++;
                  }
                  PreviousByteDiff = true;
                } else {
                  PreviousByteDiff = false;
                }
              }
            }
            if (DiffCount) {
              uint64_t DiffStart = DiffOffset >= 8 ? DiffOffset - 8 : 0;
              uint64_t DiffContextBytes = std::min<uint64_t>(16, Entry.GuestSize - DiffStart);
              auto LiveDump = fmt::join(std::span<const uint8_t> {LiveGuest + DiffStart, DiffContextBytes}, " ");
              auto CacheDump = fmt::join(std::span<const uint8_t> {CachedGuest + DiffStart, DiffContextBytes}, " ");
              auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";
              LogMan::Msg::IFmt("DiskCache: lookup guest hash mismatch key={}-{:x} gsize={}, ndiff={}, first diff: offset={} "
                                "live=[{:02x}] "
                                "cached=[{:02x}]",
                                KeyPrefix, *GuestCodeKey, Entry.GuestSize, DiffCount, DiffOffset, LiveDump, CacheDump);
            } else {
              LogMan::Msg::IFmt("DiskCache: guest hash mismatch but no diff?");
            }
          }
        }
      }
    }

    if (!FoundMatchingHash || !EntryUnderReview) {
      return std::nullopt;
    }

    IndexEntry& Entry = *EntryUnderReview;

    fextl::vector<uint64_t> GuestPages;
    for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
      uint64_t FirstPage = (Entry.GuestExtents[i] + GuestRIP) & Utils::FEX_PAGE_MASK;
      uint64_t LastPage = (Entry.GuestExtents[i] + GuestRIP + Entry.GuestExtents[i + 1] - 1) & Utils::FEX_PAGE_MASK;
      for (uint64_t Page = FirstPage; Page <= LastPage; Page += Utils::FEX_PAGE_SIZE) {
        if (GuestPages.size() == 0 || Page != GuestPages.back()) {
          GuestPages.push_back(Page);
        }
      }
    }

    // this seems to be a full hit, pull from disk and check the entry is big enough to have everything (except GuestCode)
    CodeHitData HitData;
    uint32_t EntrySizeWithoutGuestCode = Entry.Size - Entry.GuestSize;
    HitData.Blob.resize(GuestPages.size() * sizeof(uint64_t) + EntrySizeWithoutGuestCode);
    memcpy(HitData.Blob.data(), GuestPages.data(), GuestPages.size() * sizeof(uint64_t));
    uint32_t BlobOffset = GuestPages.size() * sizeof(uint64_t);
    bool FoundInLRU = false;
    if (BlobRef && BlobRef->size() >= EntrySizeWithoutGuestCode) {
      FoundInLRU = true;
      memcpy(HitData.Blob.data() + BlobOffset, BlobRef->data(), EntrySizeWithoutGuestCode);
      if (LRUIter) {
        std::lock_guard Guard(MemoryLRULock);
        // avoided disk by nabbing from lru, bump to front
        // LogMan::Msg::IFmt("lru hit! {}", MemoryLRUCurrentSize);
        MemoryLRU.splice(MemoryLRU.begin(), MemoryLRU, *LRUIter);
      }
    } else if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) {
      return std::nullopt;
    }

    if (EntrySizeWithoutGuestCode < sizeof(BlobFixedHeader)) {
      return std::nullopt;
    }
    BlobFixedHeader Header;
    memcpy(&Header, HitData.Blob.data() + BlobOffset, sizeof(Header));
    BlobOffset += sizeof(Header);

    uint32_t SizeNeeded = sizeof(Header) + Header.HostSize + Header.EntryPointCount * (sizeof(uint64_t) + sizeof(uint32_t));
    SizeNeeded += Header.SmallRelocCount * sizeof(BlobSmallRelocation) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation);
    if (EntrySizeWithoutGuestCode != SizeNeeded) {
      return std::nullopt;
    }

    if (Entry.GuestSize != Header.GuestSize || !XXH128_isEqual(Header.GuestHash, Entry.GuestHash)) {
      return std::nullopt;
    }

    bool StoreInMemory = !FoundInLRU;
    if (EntrySizeWithoutGuestCode > MemoryLRUMaxSize) {
      StoreInMemory = false;
    }

    if (StoreInMemory) {
      auto NewBlobRef = fextl::make_shared<fextl::vector<uint8_t>>(EntrySizeWithoutGuestCode);
      memcpy(NewBlobRef->data(), HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode);
      std::lock_guard Guard(IndexLock);
      auto CurrentIndexEntry = LookupLocked(LookupKey, Header.GuestHash, LastFootprintHashed);
      if (CurrentIndexEntry && !CurrentIndexEntry->MemoryBlob) {
        std::lock_guard LRUGuard(MemoryLRULock);
        MemoryLRU.push_front({LookupKey, Header.GuestHash, LastFootprintHashed, EntrySizeWithoutGuestCode});
        CurrentIndexEntry->LRUEntry = MemoryLRU.begin();
        CurrentIndexEntry->MemoryBlob = NewBlobRef;
        MemoryLRUCurrentSize += EntrySizeWithoutGuestCode;
      }
    }

    if (StoreInMemory && MemoryLRUCurrentSize > MemoryLRUMaxSize + MemoryLRUEvictThreshold) {
      Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(this));
    }

    HitData.HostCode = {HitData.Blob.data() + BlobOffset, Header.HostSize};
    BlobOffset += Header.HostSize;
    HitData.EntryPointRIPs = {reinterpret_cast<uint64_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
    BlobOffset += Header.EntryPointCount * sizeof(uint64_t);
    HitData.EntryPointHostOffsets = {reinterpret_cast<const uint32_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
    BlobOffset += Header.EntryPointCount * sizeof(uint32_t);
    std::span<const BlobSmallRelocation> SmallRelocs = {reinterpret_cast<const BlobSmallRelocation*>(HitData.Blob.data() + BlobOffset),
                                                        Header.SmallRelocCount};
    BlobOffset += Header.SmallRelocCount * sizeof(BlobSmallRelocation);
    std::span<const BlobThunkRelocation> ThunkRelocs = {reinterpret_cast<const BlobThunkRelocation*>(HitData.Blob.data() + BlobOffset),
                                                        Header.ThunkRelocCount};
    BlobOffset += Header.ThunkRelocCount * sizeof(BlobThunkRelocation);

    HitData.GuestPages = {reinterpret_cast<uint64_t*>(HitData.Blob.data()), GuestPages.size()};

    for (auto& EntryPointRip : HitData.EntryPointRIPs) {
      EntryPointRip += GuestRIP;
    }

    if (!CTX->CodeCache.ApplyPackedCodeRelocations(GuestRIP, std::as_writable_bytes(HitData.HostCode), SmallRelocs, ThunkRelocs)) {
      return std::nullopt;
    }

    return HitData;
  }

  void DiskCache::Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
                           std::optional<ExecutableFileSectionInfo> Region) {
    if (!Validation) {
      return;
    }

    auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";

    if (Hit.HostCode.size() != CompiledCode.Size) {
      LogMan::Msg::EFmt("DiskCache: validate host size mismatch key={}-{:x} cached={} live={}", KeyPrefix, GuestCodeKey,
                        Hit.HostCode.size(), CompiledCode.Size);
    } else if (memcmp(Hit.HostCode.data(), CompiledCode.BlockBegin, CompiledCode.Size) != 0) {
      bool PreviousByteDiff = false;
      size_t FirstDiff = 0;
      size_t DiffCount = 0;
      for (size_t i = 0; i < CompiledCode.Size; i++) {
        if (Hit.HostCode[i] != CompiledCode.BlockBegin[i]) {
          if (DiffCount == 0) {
            FirstDiff = i;
          }
          if (!PreviousByteDiff) {
            DiffCount++;
          }
          PreviousByteDiff = true;
        } else {
          PreviousByteDiff = false;
        }
      }

      // align to 4 bytes to read host arm easier
      const size_t DiffStart = (FirstDiff & ~3ULL) >= 8 ? (FirstDiff & ~3ULL) - 8 : 0;
      const size_t ContextBytes = std::min<size_t>(16, CompiledCode.Size - DiffStart);
      LogMan::Msg::EFmt("DiskCache: validate host code mismatch key={}-{:x} size={} firstoffset={} ndiff={} cached=[{:02x}] live=[{:02x}]",
                        KeyPrefix, GuestCodeKey, CompiledCode.Size, FirstDiff, DiffCount,
                        fmt::join(std::span<const uint8_t> {Hit.HostCode.data() + DiffStart, ContextBytes}, " "),
                        fmt::join(std::span<const uint8_t> {CompiledCode.BlockBegin + DiffStart, ContextBytes}, " "));
    }
  }

  IndexEntry* DiskCache::LookupLocked(const uint64_t LookupKey, const XXH128_hash_t& GuestHash, const uint64_t GuestFootprint) {
    auto It = Index.find(LookupKey);
    if (It != Index.end()) {
      if (XXH128_isEqual(It->second.MainEntry.GuestHash, GuestHash)) {
        return &It->second.MainEntry;
      }
      if (It->second.MoreEntries.get()) {
        auto Range = It->second.MoreEntries->equal_range(GuestFootprint);
        for (auto MoreEntriesIt = Range.first; MoreEntriesIt != Range.second; MoreEntriesIt++) {
          if (XXH128_isEqual(MoreEntriesIt->second.GuestHash, GuestHash)) {
            return &MoreEntriesIt->second;
          }
        }
      }
    }
    return nullptr;
  }

  struct DiskCache::CacheStoreWorkItem final : WorkQueueThread::WorkItem {
    DiskCache* Self;
    IndexedDB* DB;
    MesaFOZ::foz_payload_key UniqueKey;
    uint64_t LookupKey;
    std::span<uint8_t> Blob;
    fextl::vector<uint8_t> IndexBlob;
    bool StoreDisk;
    CacheStoreWorkItem(DiskCache* Self, IndexedDB* DB, const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey,
                       std::span<uint8_t> Blob, fextl::vector<uint8_t>&& IndexBlob, bool StoreDisk)
      : Self(Self)
      , DB(DB)
      , UniqueKey(UniqueKey)
      , LookupKey(LookupKey)
      , Blob(Blob)
      , IndexBlob(std::move(IndexBlob))
      , StoreDisk(StoreDisk) {}
    void Run() override {
      struct MesaFOZ::mesa_index_db_file_entry IndexHeader;
      bool DiskSuccess = !StoreDisk || DB->StoreCacheBlob(UniqueKey, LookupKey, Blob, IndexHeader, IndexBlob);

      bool KeepEntryInMemory = true;
      // todo possible other lru condition here like entry size?
      if (Blob.size() > Self->MemoryLRUMaxSize) {
        KeepEntryInMemory = false;
      } else {
        Self->MemoryLRUCurrentSize += Blob.size();
      }

      const IndexExtraBlobHeader* IndexAfterHeader = reinterpret_cast<const IndexExtraBlobHeader*>(IndexBlob.data());

      fextl::list<MemoryLRUKey>::iterator NewLRUEntry;
      if (KeepEntryInMemory) {
        std::lock_guard Guard(Self->MemoryLRULock);
        Self->MemoryLRU.push_front({LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint, (uint32_t)Blob.size()});
        NewLRUEntry = Self->MemoryLRU.begin();
      }
      {
        std::lock_guard Guard(Self->IndexLock);
        auto IndexEntry = Self->LookupLocked(LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint);
        LOGMAN_THROW_A_FMT(IndexEntry != nullptr, "Stored Index entry not found?");
        if (IndexEntry) {
          if (StoreDisk && DiskSuccess) {
            IndexEntry->DB = DB;
            IndexEntry->Offset = IndexHeader.cache_db_file_offset;
          }
          if (!KeepEntryInMemory) {
            IndexEntry->MemoryBlob.reset();
          } else {
            IndexEntry->LRUEntry = NewLRUEntry;
          }
        }
      }

      if (KeepEntryInMemory && Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
        Self->Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(Self));
      }
    }
  };

  bool DiskCache::Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
                        uint64_t GuestCodeKey, std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
                        std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo) {
    if (!IsWritingDiskCache()) {
      return false;
    }
    if (!DecodedBlockInfo) {
      return false;
    }
    // check for any reloc targets outside of our jurisdiction
    // todo what are they exactly? caching those blocks is great when it works, so need to figure this out and make finer-grained if we can
    if (RelocationFilter && Region) {
      for (const auto& Reloc : Relocations) {
        if (Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE) {
          continue;
        }
        uint64_t Target = Reloc.GuestRIP.GuestRIP;
        if (Target >= Region->BeginVA && Target < Region->EndVA) {
          continue;
        }
        // let it through if it's inside the same ELF image? (like bss)
        if (Region->FileInfo.MappedSize && Target >= Region->FileStartVA && Target < Region->FileStartVA + Region->FileInfo.MappedSize) {
          continue;
        }

        auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target);
        if (!TargetSection || TargetSection->FileInfo.FileId != Region->FileInfo.FileId) {
          // we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm
          return false;
        }
      }
    }

    uint32_t SmallRelocCount = 0;
    uint32_t ThunkRelocCount = 0;
    for (const auto& Reloc : Relocations) {
      if (Reloc.Header.Type == CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE) {
        ThunkRelocCount++;
      } else {
        SmallRelocCount++;
      }
    }

    const uint32_t EntryPointCount = (uint32_t)CompiledCode.EntryPoints.size();

    const size_t HeaderOffset = 0;
    const size_t HostCodeOffset = HeaderOffset + sizeof(BlobFixedHeader);
    const size_t EntryPointRIPsOffset = HostCodeOffset + CompiledCode.Size;
    const size_t EntryPointHostOffsetsOffset = EntryPointRIPsOffset + EntryPointCount * sizeof(uint64_t);
    const size_t SmallRelocsOffset = EntryPointHostOffsetsOffset + EntryPointCount * sizeof(uint32_t);
    const size_t ThunkRelocsOffset = SmallRelocsOffset + SmallRelocCount * sizeof(BlobSmallRelocation);
    const size_t GuestCodeOffset = ThunkRelocsOffset + ThunkRelocCount * sizeof(BlobThunkRelocation);
    const size_t TotalSize = GuestCodeOffset + GuestCode.size();

    // we'll copy everything into here and pass it to the Writer, then return to caller quickly
    fextl::vector<uint8_t> Blob;
    Blob.resize(TotalSize);
    uint8_t* BlobData = Blob.data();

    BlobFixedHeader Header {
      .GuestSize = (uint32_t)GuestCode.size(),
      .HostSize = (uint32_t)CompiledCode.Size,
      .EntryPointCount = EntryPointCount,
      .SmallRelocCount = SmallRelocCount,
      .ThunkRelocCount = ThunkRelocCount,
    };

    memcpy(BlobData + HostCodeOffset, CompiledCode.BlockBegin, CompiledCode.Size);

    // pack and relocate entrypoints
    auto* EntryRIPs = reinterpret_cast<uint64_t*>(BlobData + EntryPointRIPsOffset);
    auto* EntryHostOffsets = reinterpret_cast<uint32_t*>(BlobData + EntryPointHostOffsetsOffset);
    uint32_t EntryIdx = 0;
    for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
      EntryRIPs[EntryIdx] = GuestAddr - GuestRIP;
      EntryHostOffsets[EntryIdx] = uint32_t(HostAddr - CompiledCode.BlockBegin);
      EntryIdx++;
    }

    fextl::set<uint64_t> DataMasksConsumed;

    // pack relocations
    auto* SmallRelocs = reinterpret_cast<BlobSmallRelocation*>(BlobData + SmallRelocsOffset);
    auto* ThunkRelocs = reinterpret_cast<BlobThunkRelocation*>(BlobData + ThunkRelocsOffset);
    uint32_t SmallIdx = 0;
    uint32_t ThunkIdx = 0;
    for (const auto& Reloc : Relocations) {
      switch (Reloc.Header.Type) {
      // it's important to zero-init the element completely so we don't have garbage in unused fields
      // this way, the caches stay deterministic across machines
      case CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
        BlobSmallRelocation SmallReloc = {};
        SmallReloc.Offset = Reloc.Header.Offset;
        SmallReloc.Type = uint8_t(Reloc.Header.Type);
        SmallReloc.Named.Symbol = uint32_t(Reloc.NamedSymbolLiteral.Symbol);
        SmallRelocs[SmallIdx++] = SmallReloc;
        break;
      }
      case CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {
        BlobSmallRelocation SmallReloc = {};
        SmallReloc.Offset = Reloc.Header.Offset;
        SmallReloc.Type = uint8_t(Reloc.Header.Type);
        SmallReloc.RIPLiteral.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
        SmallRelocs[SmallIdx++] = SmallReloc;
        break;
      }
      case CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {
        BlobSmallRelocation SmallReloc = {};
        SmallReloc.Offset = Reloc.Header.Offset;
        SmallReloc.Type = uint8_t(Reloc.Header.Type);
        SmallReloc.RIPMove.RegisterIndex = Reloc.GuestRIP.RegisterIndex;
        SmallReloc.RIPMove.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
        SmallRelocs[SmallIdx++] = SmallReloc;
        break;
      }
      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE:
      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE:
      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL:
      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
        // same data for all, relative vs. not and register vs. literal will depend on type on apply
        BlobSmallRelocation SmallReloc = {};
        SmallReloc.Offset = Reloc.Header.Offset;
        SmallReloc.Type = uint8_t(Reloc.Header.Type);
        SmallReloc.PatchableData.RegisterIndex = Reloc.GuestPatchableData.RegisterIndex;
        SmallReloc.PatchableData.ValueSize = Reloc.GuestPatchableData.ValueSize;
        SmallReloc.PatchableData.SiteOffset = uint32_t(Reloc.GuestPatchableData.SiteAddress - GuestRIP);
        SmallRelocs[SmallIdx++] = SmallReloc;

        // mark the corresponding data mask consumed
        DataMasksConsumed.insert(Reloc.GuestPatchableData.SiteAddress);

        break;
      }
      case CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
        BlobThunkRelocation BigReloc = {};
        BigReloc.Offset = Reloc.Header.Offset;
        BigReloc.RegisterIndex = Reloc.NamedThunkMove.RegisterIndex;
        memcpy(BigReloc.SymbolHash, &Reloc.NamedThunkMove.Symbol, sizeof(BigReloc.SymbolHash));
        ThunkRelocs[ThunkIdx++] = BigReloc;
        break;
      }
      }
    }

    fextl::vector<uint32_t> ExactGuestCodeExtents;
    uint64_t CurStartExtent = 0, CurEndExtent = 0;
    const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr;
    for (auto& SubBlock : DecodedBlockInfo->Blocks) {
      if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
        return false;
      }
      if (!CurStartExtent) {
        CurStartExtent = SubBlock.Entry;
        CurEndExtent = SubBlock.Entry + SubBlock.Size;
      } else {
        LOGMAN_THROW_A_FMT(SubBlock.Entry >= CurEndExtent, "DecodedBlocks not sorted or overlapping?");
        if (SubBlock.Entry == CurEndExtent) {
          CurEndExtent = SubBlock.Entry + SubBlock.Size;
        } else {
          ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
          ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
          CurStartExtent = SubBlock.Entry;
          CurEndExtent = SubBlock.Entry + SubBlock.Size;
        }
      }
      // split extents according to data masks as well
      for (auto& Mask : SubBlock.DataMasks) {
        if (Mask.Type == Frontend::Decoder::DataMaskType::NOP || DataMasksConsumed.find(Mask.FieldAddress) != DataMasksConsumed.end()) {
          if (Mask.FieldAddress > CurStartExtent) {
            ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
            ExactGuestCodeExtents.push_back(Mask.FieldAddress - CurStartExtent);
          }
          CurStartExtent = Mask.FieldAddress + Mask.ValueSize;
        }
      }
      LastBlock = &SubBlock;
    }
    if (LastBlock && (CurStartExtent != GuestRIP || CurEndExtent != GuestRIP + GuestCode.size())) {
      ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
      ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
    }

    if (ExactGuestCodeExtents.size() == 0) {
      ExactGuestCodeExtents.reserve(2);
      ExactGuestCodeExtents.push_back(0);
      ExactGuestCodeExtents.push_back(GuestCode.size());
    }

    uint64_t GuestFootprint = XXH3_64bits(ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));

    // if (ExactGuestCodeExtents.size()) {
    //   LogMan::Msg::IFmt("store! length {:d}", GuestCode.size());
    //   for(uint32_t i = 0; i < ExactGuestCodeExtents.size(); i+=2 ) {
    //     LogMan::Msg::IFmt("extent {} {}", ExactGuestCodeExtents[i], ExactGuestCodeExtents[i]+ExactGuestCodeExtents[i+1]);
    //   }
    // }
    {
      XXH3_state_t HashState;
      XXH3_128bits_reset(&HashState);
      for (uint32_t i = 0; i < ExactGuestCodeExtents.size(); i += 2) {
        XXH3_128bits_update(&HashState, GuestCode.data() + ExactGuestCodeExtents[i], ExactGuestCodeExtents[i + 1]);
      }
      Header.GuestHash = XXH3_128bits_digest(&HashState);
    }
    memcpy(BlobData + HeaderOffset, &Header, sizeof(Header));

    auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
    uint64_t LookupKey =
      MakeLookupKey(Thread, GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));

    // blob done, publish to index as in-memory for now, flush to disk below
    auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(std::move(Blob));
    struct IndexEntry NewEntry {nullptr, 0, (uint32_t)TotalSize, Header.GuestSize, Header.GuestHash, BlobRef};
    NewEntry.GuestExtents = ExactGuestCodeExtents;
    {
      std::lock_guard Guard(IndexLock);
      auto It = Index.find(LookupKey);
      if (It == Index.end()) {
        Index.emplace(LookupKey, IndexCacheHead {std::move(NewEntry), GuestFootprint, nullptr});
      } else {
        bool Dupe = false;
        if (It->second.MoreEntries.get()) {
          for (auto& [Key, Elem] : *It->second.MoreEntries) {
            if (XXH128_isEqual(Elem.GuestHash, Header.GuestHash)) {
              Dupe = true;
              break;
            }
          }
        }
        if (!Dupe && XXH128_isEqual(It->second.MainEntry.GuestHash, Header.GuestHash)) {
          Dupe = true;
        }
        // could happen if it's seen again while in flight in the store queue
        if (Dupe) {
          return true;
        }
        if (!It->second.MoreEntries.get()) {
          It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, struct IndexEntry>>();
        } else if (It->second.MoreEntries->size() >= LOOKUP_KEY_MAX_BUCKET_DEPTH) {
          return false;
        }
        It->second.MoreEntries->insert({GuestFootprint, std::move(NewEntry)});
      }
    }

    memcpy(BlobData + GuestCodeOffset, GuestCode.data(), GuestCode.size());

    MesaFOZ::foz_payload_key Key = {};
    {
      XXH3_state_t HashState;
      XXH3_128bits_reset(&HashState);
      XXH3_128bits_update(&HashState, &Header.GuestHash, sizeof(Header.GuestHash));
      XXH3_128bits_update(&HashState, &LookupKey, sizeof(LookupKey));
      XXH128_hash_t UniqueKey = XXH3_128bits_digest(&HashState);
      fextl::string BlobName = fextl::fmt::format("{:016x}{:016x}", UniqueKey.high64, UniqueKey.low64);
      memcpy(Key.bytes, BlobName.data(), BlobName.size());
    }

    fextl::vector<uint8_t> IndexBlob;
    IndexBlob.resize(sizeof(IndexExtraBlobHeader) + ExactGuestCodeExtents.size() * sizeof(uint32_t));

    IndexExtraBlobHeader IndexBlobHeader {Header.GuestHash, GuestFootprint, Header.GuestSize, (uint32_t)ExactGuestCodeExtents.size()};
    memcpy(IndexBlob.data(), &IndexBlobHeader, sizeof(IndexExtraBlobHeader));
    memcpy(IndexBlob.data() + sizeof(IndexExtraBlobHeader), ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));

    bool StoreDisk = true;

    if (RWCacheDB->Full()) {
      StoreDisk = false;
    }

    // hand the rest off to the writer thread
    Writer->QueueWork(fextl::make_unique<CacheStoreWorkItem>(this, RWCacheDB.get(), Key, LookupKey, std::span(BlobData, TotalSize),
                                                             std::move(IndexBlob), StoreDisk));
    return true;
  }

  uint16_t GetFormatVersion() {
    return FormatVersion;
  }
} // namespace DiskCache

} // namespace FEXCore