diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index ad2186989..26458cad9 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -18,6 +18,7 @@ set(SRCS Common/JitSymbols.cpp Interface/Context/Context.cpp Interface/Core/LookupCache.cpp + Interface/Core/DiskCache.cpp Interface/Core/CodeCache.cpp Interface/Core/Core.cpp Interface/Core/CPUBackend.cpp diff --git a/FEXCore/Source/Interface/Config/Config.cpp b/FEXCore/Source/Interface/Config/Config.cpp index 56ce0196b..dba0ba687 100644 --- a/FEXCore/Source/Interface/Config/Config.cpp +++ b/FEXCore/Source/Interface/Config/Config.cpp @@ -48,6 +48,7 @@ enum Paths { PATH_CONFIG_FILE_LOCAL, PATH_CONFIG_FILE_GLOBAL, PATH_CONFIG_TELEMETRY_FOLDER, + PATH_CACHE_DIR, PATH_LAST, }; static std::array Paths; @@ -64,6 +65,10 @@ void SetConfigFileLocation(const std::string_view Path, bool Global) { Paths[PATH_CONFIG_FILE_LOCAL + Global] = Path; } +void SetCacheDirectory(const std::string_view Path) { + Paths[PATH_CACHE_DIR] = Path; +} + const fextl::string& GetTelemetryDirectory() { auto& Path = Paths[PATH_CONFIG_TELEMETRY_FOLDER]; if (Path.empty()) { @@ -91,6 +96,10 @@ const fextl::string& GetConfigFileLocation(bool Global) { return Paths[PATH_CONFIG_FILE_LOCAL + Global]; } +const fextl::string& GetCacheDirectory() { + return Paths[PATH_CACHE_DIR]; +} + fextl::string GetApplicationConfig(const std::string_view Program, bool Global) { fextl::string ConfigFile = GetConfigDirectory(Global); diff --git a/FEXCore/Source/Interface/Config/Config.json.in b/FEXCore/Source/Interface/Config/Config.json.in index fa56db44c..b225b680a 100644 --- a/FEXCore/Source/Interface/Config/Config.json.in +++ b/FEXCore/Source/Interface/Config/Config.json.in @@ -132,6 +132,34 @@ "Desc": [ "Allows overriding cpu feature flags for manual testing" ] + }, + "DiskCache": { + "Type": "bool", + "Default": "false", + "Desc": [ + "Enables disk caching for code blocks" + ] + }, + "DiskCacheRelocationFilter": { + "Type": "bool", + "Default": "true", + "Desc": [ + "Don't cache blocks with relocations pointing outside of any known region" + ] + }, + "DiskCachePath": { + "Type": "str", + "Default": "", + "Desc": [ + "Optional base directory override for disk cache" + ] + }, + "DiskCacheRODBNames": { + "Type": "str", + "Default": "", + "Desc": [ + "Optional list of extra read-only disk cache DBs to consider" + ] } }, "Emulation": { diff --git a/FEXCore/Source/Interface/Context/Context.cpp b/FEXCore/Source/Interface/Context/Context.cpp index c31bbc067..7e0aafa87 100644 --- a/FEXCore/Source/Interface/Context/Context.cpp +++ b/FEXCore/Source/Interface/Context/Context.cpp @@ -58,7 +58,7 @@ bool FEXCore::Context::ContextImpl::IsAddressInCodeBuffer(FEXCore::Core::Interna bool FEXCore::Context::ContextImpl::RequiresRelocatableConstants() const { // Support relocation when generating a cache or when generating reference code for validation - return CodeCache.IsGeneratingCache || FEXCore::Config::Get_ENABLECODECACHEVALIDATION(); + return CodeCache.IsGeneratingCache || FEXCore::Config::Get_ENABLECODECACHEVALIDATION() || DiskCache.IsWritingDiskCache(); } } // namespace FEXCore::Context diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index b8b68b7c7..58807e898 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -202,6 +203,8 @@ public: FEXCore::CPUID::XCRResults RunXCRFunction(uint32_t Function) override; FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) override; + virtual void InitDiskCache() override {} + CodeCache& GetCodeCache() override { return CodeCache; } @@ -375,6 +378,7 @@ public: FEXCore::HLE::SourcecodeResolver* SourcecodeResolver {}; FEXCore::ThunkHandler* ThunkHandler {}; fextl::unique_ptr Dispatcher; + DiskCache::DiskCache DiskCache; CodeCache CodeCache; fextl::unique_ptr CodeMapWriter; diff --git a/FEXCore/Source/Interface/Core/CPUBackend.h b/FEXCore/Source/Interface/Core/CPUBackend.h index f80a1ecc2..c97f86917 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.h +++ b/FEXCore/Source/Interface/Core/CPUBackend.h @@ -18,6 +18,11 @@ $end_info$ #include #include +#include + +namespace FEXCore::DiskCache { +struct BlobEntryPoint; +} namespace FEXCore::CPU { union Relocation; @@ -56,6 +61,8 @@ namespace CPU { fextl::map EntryPoints; // The total size of the codeblock from [BlockBegin, BlockBegin+Size). size_t Size; + // Offset of BlockBegin from the start of the CodeBuffer it lives in + uint64_t HostCodeOffset; }; // Header that can live at the start of a JIT block. @@ -115,6 +122,10 @@ namespace CPU { virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR, FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0; + virtual CompiledCode LoadCachedCode(std::span HostBytes, std::span EntryPoints) { + return {}; + } + virtual fextl::vector TakeRelocations(uint64_t GuestBaseAddress) = 0; virtual void ClearCache() {} diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index a39c9a130..32ee7e9eb 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -103,6 +103,8 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features) // Track atomic TSO emulation configuration. UpdateAtomicTSOEmulationConfig(); + + DiskCache.Init(this); } struct GetFrameBlockInfoResult { @@ -858,6 +860,40 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ return HostCode; } + std::optional Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP); + std::optional Hit; + bool DiskCacheHitRelocationsApplied = false; + bool LoadDiskCacheCode = true; + if (Region && Region->FileStartVA != 0) { + Hit = DiskCache.Lookup(Thread, *Region, GuestRIP); + if (Hit) { + DiskCacheHitRelocationsApplied = CodeCache.ApplyCodeRelocations(GuestRIP, std::as_writable_bytes(Hit->HostCode), Hit->Relocations, 0, false); + + if (DiskCacheHitRelocationsApplied && LoadDiskCacheCode) { + auto LoadedCode = Thread->CPUBackend->LoadCachedCode(Hit->HostCode, Hit->EntryPoints); + if (LoadedCode.BlockBegin) { + + // annoying to unpack a different copy here, maybe better way to do this + fextl::set EntryPoints; + for (auto [GuestOffset, HostAddr] : LoadedCode.EntryPoints) { + EntryPoints.insert(GuestOffset + Region->FileStartVA); + } + for (auto CodePage : Hit->GuestPages) { + if (Thread->LookupCache->AddBlockExecutableRange(Thread, EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) { + SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE); + } + } + for (auto [GuestOffset, HostAddr] : LoadedCode.EntryPoints) { + Thread->LookupCache->AddBlockMapping(Thread, GuestOffset + Region->FileStartVA, Hit->GuestPages, HostAddr); + } + + uint64_t ModuleOffset = GuestRIP - Region->FileStartVA; + return reinterpret_cast(LoadedCode.EntryPoints[ModuleOffset]); + } + } + } + } + // Accumulate a JIT count now, as even if another thread raced us, it should count as a compile. FEXCORE_PROFILE_INSTANT_INCREMENT(Thread, AccumulatedJITCount, 1); @@ -870,6 +906,9 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ return reinterpret_cast(CodePtr); } + // if this ever fires, we need to serialize the offset into disk cache + LOGMAN_THROW_A_FMT(StartAddr == GuestRIP, "StartAddr offset from GuestRIP"); + // The core managed to compile the code. if (Config.BlockJITNaming()) { auto FragmentBasePtr = CompiledCode.BlockBegin; @@ -909,11 +948,6 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ } } - // Clear any relocations that might have been generated - if (!CodeCache.IsGeneratingCache) { - Thread->CPUBackend->ClearRelocations(); - } - fextl::vector CodePages; if (NeedsAddGuestCodeRanges) { @@ -929,17 +963,29 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ } } - // Insert to lookup cache + // Disk Cache + if (Region && Region->FileStartVA != 0) { + std::span Relocations; + if (DebugData && DebugData->Relocations) { + Relocations = *DebugData->Relocations; + } + std::span GuestCode = {reinterpret_cast(StartAddr), Length}; + const Frontend::Decoder::DecodedBlockInformation* BlockInfo = NeedsAddGuestCodeRanges ? Thread->FrontendDecoder->GetDecodedBlockInfo() : nullptr; + DiskCache.Store(Thread, *Region, GuestRIP, GuestCode, CompiledCode, Relocations, BlockInfo); + if (CodeMapWriter) { + CodeMapWriter->AppendBlock(*Region, GuestRIP); + } + } + + // Insert to lookup cache for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) { Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr); } - if (CodeMapWriter) { - auto Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP); - if (Region && Region->FileStartVA != 0) { - CodeMapWriter->AppendBlock(*Region, GuestRIP); - } + // Clear any relocations that might have been generated + if (!CodeCache.IsGeneratingCache) { + Thread->CPUBackend->ClearRelocations(); } return (uintptr_t)CodePtr; diff --git a/FEXCore/Source/Interface/Core/DiskCache.cpp b/FEXCore/Source/Interface/Core/DiskCache.cpp new file mode 100644 index 000000000..65c336c0e --- /dev/null +++ b/FEXCore/Source/Interface/Core/DiskCache.cpp @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: MIT + +#include "FEXHeaderUtils/Filesystem.h" +#include "FEXCore/Core/DiskCache.h" +#include "FEXCore/Utils/LogManager.h" +#include "Interface/Context/Context.h" +#include "FEXCore/HLE/SyscallHandler.h" +#include "FEXCore/Utils/File.h" +#include "FEXCore/fextl/memory.h" +#include +#include + +namespace FEXCore { + +namespace DiskCache { + + namespace MesaFOZ { + + enum { FOSSILIZE_COMPRESSION_NONE = 1, FOSSILIZE_COMPRESSION_DEFLATE = 2 }; + + enum { FOSSILIZE_FORMAT_VERSION = 6, FOSSILIZE_FORMAT_MIN_COMPAT_VERSION = 5 }; + +#define FOZ_REF_MAGIC_SIZE 16 + + static const uint8_t stream_reference_magic_and_version[FOZ_REF_MAGIC_SIZE] = { + 0x81, 'F', 'O', 'S', 'S', 'I', 'L', 'I', 'Z', 'E', 'D', 'B', 0, 0, 0, FOSSILIZE_FORMAT_VERSION, /* 4 bytes to use for versioning. */ + }; + + struct __attribute__((packed)) mesa_index_db_file_entry { + uint64_t hash; + uint32_t size; + uint64_t last_access_time; + uint64_t cache_db_file_offset; + }; + + } // namespace MesaFOZ + + bool FOZFile::Open(const fextl::string& FOZFileName, bool ReadOnly) { + FileName = FOZFileName; + this->ReadOnly = ReadOnly; + + File::FileModes Modes = File::FileModes::READ; + if (!ReadOnly) { + Modes = Modes | File::FileModes::WRITE | File::FileModes::CREATE; + } + FD = fextl::make_unique(FileName.c_str(), Modes); + if (!FD->IsValid()) { + FD.reset(); + return false; + } + + bool Valid = false; + bool TookLock = false; + ssize_t Size = FD->Seek(0, File::SeekOp::END); + + if (Size < FOZ_REF_MAGIC_SIZE && !ReadOnly) { + if (!FD->Lock(OPEN_LOCK_TIMEOUT_MS)) { + FD.reset(); + return false; + } + TookLock = true; + // seek in case someone else made it while we waited above + Size = FD->Seek(0, File::SeekOp::END); + } + + if (Size == 0 && !ReadOnly) { + Valid = FD->Write(MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE) == FOZ_REF_MAGIC_SIZE; + } else { + FD->Seek(0, File::SeekOp::BEGIN); + uint8_t magic[FOZ_REF_MAGIC_SIZE]; + if (FD->Read(magic, FOZ_REF_MAGIC_SIZE) == FOZ_REF_MAGIC_SIZE && + memcmp(magic, MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE - 1) == 0) { + int version = magic[FOZ_REF_MAGIC_SIZE - 1]; + Valid = version <= MesaFOZ::FOSSILIZE_FORMAT_VERSION && version >= MesaFOZ::FOSSILIZE_FORMAT_MIN_COMPAT_VERSION; + } + } + + if (TookLock) { + FD->Unlock(); + } + + if (!Valid) { + FD.reset(); + } + return Valid; + } + + bool FOZFile::ReadNextBlob(MesaFOZ::foz_payload_key& OutKey, MesaFOZ::foz_payload_header& OutHeader, fextl::vector& OutBlob) { + if (FD->Read(OutKey.bytes, sizeof(OutKey.bytes)) != sizeof(OutKey.bytes)) { + return false; + } + if (FD->Read(&OutHeader, sizeof(OutHeader)) != sizeof(OutHeader)) { + return false; + } + OutBlob.resize(OutHeader.payload_size); + if (FD->Read(OutBlob.data(), OutBlob.size()) != (ssize_t)OutBlob.size()) { + return false; + } + return true; + } + + bool FOZFile::ReadBlob(uint64_t Offset, std::span OutBlob) { + ssize_t SeekRet = FD->Seek(Offset, File::SeekOp::BEGIN); + if (SeekRet < 0) { + return false; + } + if (FD->Read(OutBlob.data(), OutBlob.size()) != (ssize_t)OutBlob.size()) { + return false; + } + + return true; + } + + bool FOZFile::WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span> BlobChunks, uint64_t& OutBlobOffset) { + uint64_t WriteOffset = 0; + ssize_t SeekRet = FD->Seek(0, File::SeekOp::END); + if (SeekRet < 0) { + return false; + } + WriteOffset = (uint64_t)SeekRet; + + if (FD->Write(Key.bytes, sizeof(Key.bytes)) != sizeof(Key.bytes)) { + return false; + } + WriteOffset += sizeof(Key.bytes); + + uint64_t TotalBlobSize = 0; + for (const std::span& Chunk : BlobChunks) { + TotalBlobSize += Chunk.size(); + } + + MesaFOZ::foz_payload_header ScratchHeader {.payload_size = (uint32_t)TotalBlobSize, + .format = MesaFOZ::FOSSILIZE_COMPRESSION_NONE, + .crc = 0, // todo? maybe + .uncompressed_size = (uint32_t)TotalBlobSize}; + + if (FD->Write(&ScratchHeader, sizeof(ScratchHeader)) != sizeof(ScratchHeader)) { + return false; + } + WriteOffset += sizeof(ScratchHeader); + + OutBlobOffset = WriteOffset; + + for (const std::span& Chunk : BlobChunks) { + if (Chunk.size() == 0) { + continue; + } + if (FD->Write(Chunk.data(), Chunk.size()) != (ssize_t)Chunk.size()) { + return false; + } + } + + return true; + } + + bool IndexedDB::Open(const fextl::string& CacheDBName, bool ReadOnly) { + if (!CacheFOZ.Open(CacheDBName + ".foz", ReadOnly)) { + return false; + } + if (!IndexFOZ.Open(CacheDBName + "_idx.foz", ReadOnly)) { + return false; + } + + this->ReadOnly = ReadOnly; + return true; + } + + void IndexedDB::PopulateIndex(Index& CacheIndex) { + MesaFOZ::foz_payload_key Key; + MesaFOZ::foz_payload_header Header; + fextl::vector Blob; + + while (IndexFOZ.ReadNextBlob(Key, Header, Blob)) { + if (Blob.size() != sizeof(MesaFOZ::mesa_index_db_file_entry)) { + break; + } + MesaFOZ::mesa_index_db_file_entry* IndexEntry = (MesaFOZ::mesa_index_db_file_entry*)Blob.data(); + if (IndexEntry->hash != XXH3_64bits(Key.bytes, FOSSILIZE_BLOB_HASH_LENGTH)) { + break; + } + CacheIndex.insert({IndexEntry->hash, {this, IndexEntry->cache_db_file_offset, IndexEntry->size}}); + } + // could truncate/delete index if we don't end up perfectly at end here + } + + bool IndexedDB::ReadCacheBlob(uint64_t Offset, std::span OutBlob) { + return CacheFOZ.ReadBlob(Offset, OutBlob); + } + + bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& Key, std::span> BlobChunks, Index& Index) { + if (ReadOnly) { + // shouldn't happen + return false; + } + uint64_t Hash = XXH3_64bits(Key.bytes, FOSSILIZE_BLOB_HASH_LENGTH); + if (Index.contains(Hash)) { + // shouldn't really happen.. assert or something? + return true; + } + + if (!CacheFOZ.Lock(STORE_LOCK_TIMEOUT_MS) || !IndexFOZ.Lock(STORE_LOCK_TIMEOUT_MS)) { + CacheFOZ.Unlock(); + IndexFOZ.Unlock(); + return false; + } + + // write cache side first so we get offset for index + uint64_t BlobOffset = 0; + if (!CacheFOZ.WriteBlob(Key, BlobChunks, BlobOffset)) { + CacheFOZ.Unlock(); + IndexFOZ.Unlock(); + return false; + } + + uint64_t TotalBlobSize = 0; + for (const std::span& Chunk : BlobChunks) { + TotalBlobSize += Chunk.size(); + } + + MesaFOZ::mesa_index_db_file_entry IndexEntry {.hash = Hash, + .size = (uint32_t)TotalBlobSize, + .last_access_time = 0, // todo.. + .cache_db_file_offset = BlobOffset}; + + std::span IndexBlobChunks[] = {{(const uint8_t*)&IndexEntry, sizeof(IndexEntry)}}; + uint64_t UnusedIndexBlobOffset = 0; + if (!IndexFOZ.WriteBlob(Key, IndexBlobChunks, UnusedIndexBlobOffset)) { + CacheFOZ.Unlock(); + IndexFOZ.Unlock(); + return false; + } + + CacheFOZ.Unlock(); + IndexFOZ.Unlock(); + + Index[Hash] = {this, BlobOffset, (uint32_t)TotalBlobSize}; + return true; + } + + bool DiskCache::OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly) { + fextl::unique_ptr CurDB; + + if (!ReadOnly && RWCacheDB) { + // rw already opened, just support one + return false; + } + + CurDB = fextl::make_unique(); + if (!CurDB) { + return false; + } + + if (!CurDB->Open(CacheDBName, ReadOnly)) { + CurDB.reset(); + return false; + } + + CurDB->PopulateIndex(Index); + + if (ReadOnly) { + ROCacheDBs.push_back(std::move(CurDB)); + } else { + RWCacheDB = std::move(CurDB); + } + + return true; + } + + void DiskCache::Init(FEXCore::Context::ContextImpl* CTX) { + this->CTX = CTX; + + if (!EnableDiskCache) { + return; + } + + // todo grab all CTX options that can change compilation here + any environmental/hw things and hash into a bucket key + + fextl::string BasePath = BasePathOverride(); + if (BasePath.empty()) { + // todo put bucket hash in that path + BasePath = FEXCore::Config::GetCacheDirectory() + "DiskCache/"; + } + FHU::Filesystem::CreateDirectories(BasePath); + + fextl::string RWDBBasePath = BasePath + "RWCacheDB"; + OpenCacheDB(RWDBBasePath, false); + + std::string_view RONames = RODBNames(); + while (!RONames.empty()) { + const auto Delim = RONames.find(','); + const std::string_view ROName = RONames.substr(0, Delim); + if (!ROName.empty()) { + fextl::string RODBBasePath = BasePath; + RODBBasePath += ROName; + OpenCacheDB(RODBBasePath, true); + } + if (Delim == std::string_view::npos) { + break; + } + // advance to next + RONames.remove_prefix(Delim + 1); + } + } + + std::optional DiskCache::Lookup(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP) { + if (!IsReadingDiskCache()) { + return std::nullopt; + } + std::lock_guard Guard(Lock); + uint64_t ModuleOffset = GuestRIP - Region.FileStartVA; + + // todo move key making to a helper once we have options and stuff (see Store) + MesaFOZ::foz_payload_key Key = {}; + memcpy(Key.bytes, &ModuleOffset, sizeof(ModuleOffset)); + + uint64_t Hash = XXH3_64bits(Key.bytes, FOSSILIZE_BLOB_HASH_LENGTH); + auto It = Index.find(Hash); + if (It == Index.end()) { + // definite miss + return std::nullopt; + } + const IndexEntry& Entry = It->second; + // found a key hash match, could still be a miss, read the blob and verify more + CodeHitData HitData; + HitData.Blob.resize(Entry.Size); + if (!Entry.DB->ReadCacheBlob(Entry.Offset, HitData.Blob)) { + return std::nullopt; + } + + if (Entry.Size < sizeof(BlobFixedHeader)) { + return std::nullopt; + } + BlobFixedHeader Header; + memcpy(&Header, HitData.Blob.data(), sizeof(Header)); + + // do we have enough room in our live code to even hash GuestSize worth? + auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP); + if (RangeInfo.Size == 0 || RangeInfo.Base > GuestRIP) { + return std::nullopt; + } + uint64_t Available = RangeInfo.Base + RangeInfo.Size - GuestRIP; + if (Available < Header.GuestSize) { + return std::nullopt; + } + + XXH128_hash_t LiveGuestHash = XXH3_128bits(reinterpret_cast(GuestRIP), Header.GuestSize); + if (std::memcmp(&LiveGuestHash, &Header.GuestHash, sizeof(Header.GuestHash)) != 0) { + // LogMan::Msg::IFmt("hash mismatch! length {:d}", Header.GuestSize); + return std::nullopt; + } + // LogMan::Msg::IFmt("hash ok! length {:d}", Header.GuestSize); + + // this seems to be a full hit, lastly, check the entry is big enough to have everything (except maybe GuestCode) + uint32_t SizeNeeded = sizeof(Header) + Header.HostSize + Header.EntryPointCount * sizeof(BlobEntryPoint); + SizeNeeded += Header.SmallRelocCount * sizeof(BlobSmallRelocation) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation) + + Header.TouchedGuestPagesCount * sizeof(int64_t); + if (Entry.Size < SizeNeeded) { + return std::nullopt; + } + + HitData.HostCode = {HitData.Blob.data() + sizeof(Header), Header.HostSize}; + HitData.EntryPoints = {reinterpret_cast(HitData.Blob.data() + sizeof(Header) + Header.HostSize), Header.EntryPointCount}; + + auto* SmallRelocs = reinterpret_cast( + HitData.Blob.data() + sizeof(Header) + Header.HostSize + Header.EntryPointCount * sizeof(BlobEntryPoint)); + auto* ThunkRelocs = reinterpret_cast( + reinterpret_cast(SmallRelocs) + Header.SmallRelocCount * sizeof(BlobSmallRelocation)); + + HitData.Relocations.reserve(Header.SmallRelocCount + Header.ThunkRelocCount); + for (uint32_t i = 0; i < Header.SmallRelocCount; ++i) { + const auto& SmallReloc = SmallRelocs[i]; + FEXCore::CPU::Relocation Reloc = FEXCore::CPU::Relocation::Default(); + Reloc.Header.Type = (CPU::RelocationTypes)SmallReloc.Type; + Reloc.Header.Offset = SmallReloc.Offset; + switch (SmallReloc.Type) { + case uint8_t(CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL): + Reloc.NamedSymbolLiteral.Symbol = CPU::RelocNamedSymbolLiteral::NamedSymbol(SmallReloc.Named.Symbol); + break; + case uint8_t(CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL): Reloc.GuestRIP.GuestRIP = SmallReloc.RIPLiteral.GuestRIP; break; + case uint8_t(CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE): + Reloc.GuestRIP.RegisterIndex = SmallReloc.RIPMove.RegisterIndex; + Reloc.GuestRIP.GuestRIP = SmallReloc.RIPMove.GuestRIP; + break; + default: return std::nullopt; + } + HitData.Relocations.push_back(Reloc); + } + for (uint32_t i = 0; i < Header.ThunkRelocCount; ++i) { + const auto& BigReloc = ThunkRelocs[i]; + FEXCore::CPU::Relocation Reloc = FEXCore::CPU::Relocation::Default(); + Reloc.NamedThunkMove.Header.Offset = BigReloc.Offset; + Reloc.NamedThunkMove.Header.Type = CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE; + Reloc.NamedThunkMove.RegisterIndex = BigReloc.RegisterIndex; + memcpy(&Reloc.NamedThunkMove.Symbol, BigReloc.SymbolHash, sizeof(BigReloc.SymbolHash)); + HitData.Relocations.push_back(Reloc); + } + + auto* PageOffsets = + reinterpret_cast(reinterpret_cast(ThunkRelocs) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation)); + HitData.GuestPages.reserve(Header.TouchedGuestPagesCount); + for (uint32_t i = 0; i < Header.TouchedGuestPagesCount; ++i) { + HitData.GuestPages.push_back(GuestRIP + PageOffsets[i]); + } + + return HitData; + } + + bool DiskCache::Store(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP, + std::span GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode, + std::span Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo) { + if (!IsWritingDiskCache()) { + return false; + } + if (!DecodedBlockInfo) { + return false; + } + std::lock_guard Guard(Lock); + + // check for any reloc targets outside of our jurisdiction + // todo what are they exactly? caching those blocks is great when it works, so need to figure this out and make finer-grained if we can + if (RelocationFilter) { + for (const auto& Reloc : Relocations) { + if (Reloc.Header.Offset < CompiledCode.HostCodeOffset || Reloc.Header.Offset >= CompiledCode.HostCodeOffset + CompiledCode.Size) { + continue; + } + if (Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE) { + continue; + } + uint64_t Target = Reloc.GuestRIP.GuestRIP; + if (Target >= Region.BeginVA && Target < Region.EndVA) { + continue; + } + auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target); + if (!TargetSection || TargetSection->FileInfo.FileId != Region.FileInfo.FileId) { + // we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm + return false; + } + } + } + + // pack entrypoints to disk format + fextl::vector CacheEntryPoints; + CacheEntryPoints.reserve(CompiledCode.EntryPoints.size()); + + for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) { + CacheEntryPoints.push_back({GuestAddr - Region.FileStartVA, uint32_t(HostAddr - CompiledCode.BlockBegin)}); + } + + // pack relocations to disk format + fextl::vector SmallRelocs; + fextl::vector ThunkRelocs; + + // todo discover sizes first and reserve vecs? + + for (const auto& Reloc : Relocations) { + // relocs aren't cleared every time if IsGeneratingCache, so filter just in case + if (Reloc.Header.Offset < CompiledCode.HostCodeOffset || Reloc.Header.Offset >= CompiledCode.HostCodeOffset + CompiledCode.Size) { + continue; + } + // re-relocate :harold: + uint32_t LocalOffset = uint32_t(Reloc.Header.Offset - CompiledCode.HostCodeOffset); + + switch (Reloc.Header.Type) { + // it's important to zero-init the element completely so we don't have garbage in unused fields + // this way, the caches stay deterministic across machines + case CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: { + BlobSmallRelocation SmallReloc = {}; + SmallReloc.Offset = LocalOffset; + SmallReloc.Type = uint8_t(Reloc.Header.Type); + SmallReloc.Named.Symbol = uint32_t(Reloc.NamedSymbolLiteral.Symbol); + SmallRelocs.push_back(SmallReloc); + break; + } + case CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: { + BlobSmallRelocation SmallReloc = {}; + SmallReloc.Offset = LocalOffset; + SmallReloc.Type = uint8_t(Reloc.Header.Type); + SmallReloc.RIPLiteral.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP; + SmallRelocs.push_back(SmallReloc); + break; + } + case CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: { + BlobSmallRelocation SmallReloc = {}; + SmallReloc.Offset = LocalOffset; + SmallReloc.Type = uint8_t(Reloc.Header.Type); + SmallReloc.RIPMove.RegisterIndex = Reloc.GuestRIP.RegisterIndex; + SmallReloc.RIPMove.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP; + SmallRelocs.push_back(SmallReloc); + break; + } + case CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: { + BlobThunkRelocation BigReloc = {}; + BigReloc.Offset = LocalOffset; + BigReloc.RegisterIndex = Reloc.NamedThunkMove.RegisterIndex; + memcpy(BigReloc.SymbolHash, &Reloc.NamedThunkMove.Symbol, sizeof(BigReloc.SymbolHash)); + ThunkRelocs.push_back(BigReloc); + break; + } + } + } + + // pack touched pages, relative to GuestRIP + // in theory we could save some size here, unlikely we need all 64bits + fextl::vector GuestPageOffsets; + if (DecodedBlockInfo) { + GuestPageOffsets.reserve(DecodedBlockInfo->CodePages.size()); + for (auto& GuestPage : DecodedBlockInfo->CodePages) { + GuestPageOffsets.push_back(GuestPage - GuestRIP); + } + } + + uint64_t ModuleOffset = GuestRIP - Region.FileStartVA; + + // todo also copy/hash options that affect codegen into the key + // todo should try to keep the key ascii i think? + MesaFOZ::foz_payload_key Key = {}; + memcpy(Key.bytes, &ModuleOffset, sizeof(ModuleOffset)); + + BlobFixedHeader Header { + .GuestSize = (uint32_t)GuestCode.size(), + .HostSize = (uint32_t)CompiledCode.Size, + .EntryPointCount = (uint32_t)CacheEntryPoints.size(), + .SmallRelocCount = (uint32_t)SmallRelocs.size(), + .ThunkRelocCount = (uint32_t)ThunkRelocs.size(), + .TouchedGuestPagesCount = (uint32_t)GuestPageOffsets.size(), + .GuestHash = XXH3_128bits(GuestCode.data(), GuestCode.size()), + }; + + std::span BlobChunks[] = { + {(const uint8_t*)&Header, sizeof(Header)}, + {(const uint8_t*)CompiledCode.BlockBegin, CompiledCode.Size}, + {(const uint8_t*)CacheEntryPoints.data(), CacheEntryPoints.size() * sizeof(BlobEntryPoint)}, + {(const uint8_t*)SmallRelocs.data(), SmallRelocs.size() * sizeof(BlobSmallRelocation)}, + {(const uint8_t*)ThunkRelocs.data(), ThunkRelocs.size() * sizeof(BlobThunkRelocation)}, + {(const uint8_t*)GuestPageOffsets.data(), GuestPageOffsets.size() * sizeof(int64_t)}, + GuestCode, + }; + + return RWCacheDB->StoreCacheBlob(Key, BlobChunks, Index); + } + +} // namespace DiskCache + +} // namespace FEXCore \ No newline at end of file diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index d486af24d..1228e8bd4 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -1117,6 +1117,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size EntryPoint.second += Delta; } CodeBegin += Delta; + CodeData.HostCodeOffset = CodeData.BlockBegin - CurrentCodeBuffer->Ptr; // Offset the relocations based on how far forward they moved from the temp buffer to the new buffer. // TODO: Relocations should instead be relocated based on the block entrypoint instead of the codebuffer base. @@ -1166,6 +1167,25 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size return std::move(CodeData); } +CPUBackend::CompiledCode Arm64JITCore::LoadCachedCode(std::span HostBytes, std::span EntryPoints) { + // we stored it aligned, better still be? + LOGMAN_THROW_A_FMT(HostBytes.size() % 16 == 0, "Needs to be 16B aligned!"); + auto AllocatedInfo = AllocateCodeBufferInSharedCache(HostBytes.size()); + + uint8_t* Dest = AllocatedInfo.BufferAllocationOffset; + memcpy(Dest, HostBytes.data(), HostBytes.size()); + ClearICache(Dest, HostBytes.size()); + + CPUBackend::CompiledCode Result; + Result.BlockBegin = Dest; + Result.Size = HostBytes.size(); + Result.HostCodeOffset = Dest - CurrentCodeBuffer->Ptr; + for (const auto& Ep : EntryPoints) { + Result.EntryPoints[Ep.GuestRIP] = Dest + Ep.HostOffset; + } + return Result; +} + void Arm64JITCore::ResetStack() { if (SpillSlots == 0) { return; diff --git a/FEXCore/Source/Interface/Core/JIT/JITClass.h b/FEXCore/Source/Interface/Core/JIT/JITClass.h index a06ee0664..aeccf73be 100644 --- a/FEXCore/Source/Interface/Core/JIT/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/JITClass.h @@ -54,6 +54,9 @@ public: CPUBackend::CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR, FEXCore::Core::DebugData* DebugData, bool CheckTF) override; + [[nodiscard]] + CPUBackend::CompiledCode LoadCachedCode(std::span HostBytes, std::span EntryPoints) override; + void ClearCache() override; void ClearRelocations() override { diff --git a/FEXCore/include/FEXCore/Config/Config.h b/FEXCore/include/FEXCore/Config/Config.h index f71419327..dae94634c 100644 --- a/FEXCore/include/FEXCore/Config/Config.h +++ b/FEXCore/include/FEXCore/Config/Config.h @@ -116,11 +116,13 @@ namespace detail { FEX_DEFAULT_VISIBILITY void SetDataDirectory(std::string_view Path, bool Global); FEX_DEFAULT_VISIBILITY void SetConfigDirectory(const std::string_view Path, bool Global); FEX_DEFAULT_VISIBILITY void SetConfigFileLocation(std::string_view Path, bool Global); +FEX_DEFAULT_VISIBILITY void SetCacheDirectory(const std::string_view Path); FEX_DEFAULT_VISIBILITY const fextl::string& GetDataDirectory(bool Global = false); FEX_DEFAULT_VISIBILITY const fextl::string& GetConfigDirectory(bool Global); FEX_DEFAULT_VISIBILITY const fextl::string& GetConfigFileLocation(bool Global = false); FEX_DEFAULT_VISIBILITY fextl::string GetApplicationConfig(const std::string_view Program, bool Global); +FEX_DEFAULT_VISIBILITY const fextl::string& GetCacheDirectory(); using LayerValue = std::variant< fextl::string, StringArrayType, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t, bool >; diff --git a/FEXCore/include/FEXCore/Core/Context.h b/FEXCore/include/FEXCore/Core/Context.h index ec61b90bf..66dd0377a 100644 --- a/FEXCore/include/FEXCore/Core/Context.h +++ b/FEXCore/include/FEXCore/Core/Context.h @@ -136,6 +136,8 @@ public: FEX_DEFAULT_VISIBILITY virtual FEXCore::CPUID::XCRResults RunXCRFunction(uint32_t Function) = 0; FEX_DEFAULT_VISIBILITY virtual FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) = 0; + virtual void InitDiskCache() = 0; + virtual AbstractCodeCache& GetCodeCache() = 0; virtual void SetCodeMapWriter(fextl::unique_ptr) = 0; virtual void FlushAndCloseCodeMap() = 0; diff --git a/FEXCore/include/FEXCore/Core/DiskCache.h b/FEXCore/include/FEXCore/Core/DiskCache.h new file mode 100644 index 000000000..c3abac6ee --- /dev/null +++ b/FEXCore/include/FEXCore/Core/DiskCache.h @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: MIT +#pragma once +#include "FEXCore/Core/CodeCache.h" +#include "FEXCore/Core/Context.h" +#include "Interface/Core/JIT/Relocations.h" +#include "Interface/Core/Frontend.h" +#include "Interface/Core/CPUBackend.h" +#include "FEXCore/Config/Config.h" +#include "FEXCore/Utils/File.h" +#include "FEXCore/fextl/memory.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FEXCore { + +namespace Context { + class ContextImpl; +} + +namespace DiskCache { + + namespace MesaFOZ { + +#define FOSSILIZE_BLOB_HASH_LENGTH 40 /* SHA1 hexadecimal string length */ + + struct __attribute__((packed)) foz_payload_key { + uint8_t bytes[FOSSILIZE_BLOB_HASH_LENGTH]; + }; + + struct __attribute__((packed)) foz_payload_header { + uint32_t payload_size; + uint32_t format; + uint32_t crc; + uint32_t uncompressed_size; + }; + + } // namespace MesaFOZ + + class IndexedDB; + + struct IndexEntry { + IndexedDB* DB; + uint64_t Offset; + uint32_t Size; + }; + + struct __attribute__((packed)) BlobFixedHeader { + uint32_t GuestSize; + uint32_t HostSize; + uint32_t EntryPointCount; + uint32_t SmallRelocCount; + uint32_t ThunkRelocCount; + uint32_t TouchedGuestPagesCount; + XXH128_hash_t GuestHash; + }; + + struct __attribute__((packed)) BlobEntryPoint { + uint64_t GuestRIP; // todo those have been made relative since i wrote this, can we get away with less size here? + uint32_t HostOffset; + }; + + // packed struct for types 0, 2 and 3. type 1 is bigger and separate below + struct __attribute__((packed)) BlobSmallRelocation { + uint32_t Offset; + uint8_t Type; + union { + struct __attribute__((packed)) { + uint32_t Symbol; + } Named; + struct __attribute__((packed)) { + uint64_t GuestRIP; + } RIPLiteral; + struct __attribute__((packed)) { + uint8_t RegisterIndex; + uint64_t GuestRIP; + } RIPMove; + }; + }; + + // type 1, implicit + struct __attribute__((packed)) BlobThunkRelocation { + uint32_t Offset; + uint8_t RegisterIndex; + uint8_t SymbolHash[32]; // sha256sum in the real RelocNamedThunkMove + }; + + struct CodeHitData { + fextl::vector Blob; + std::span HostCode; + std::span EntryPoints; + fextl::vector Relocations; + fextl::vector GuestPages; + + // the spans above point to memory owned by the Blob vec, so it's important this can't be copied + CodeHitData() = default; + CodeHitData(CodeHitData&&) = default; + CodeHitData& operator=(CodeHitData&&) = default; + CodeHitData(const CodeHitData&) = delete; + CodeHitData& operator=(const CodeHitData&) = delete; + }; + + using Index = fextl::robin_map; + + class FOZFile { + public: + bool Open(const fextl::string& CacheFileName, bool ReadOnly); + bool Lock(uint32_t TimeoutMS) { + if (!FD) { + return false; + } + return FD->Lock(TimeoutMS); + } + bool Unlock() { + if (!FD) { + return false; + } + return FD->Unlock(); + } + bool ReadNextBlob(MesaFOZ::foz_payload_key& OutKey, MesaFOZ::foz_payload_header& OutHeader, fextl::vector& OutBlob); + bool ReadBlob(uint64_t Offset, std::span OutBlob); + bool WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span> BlobChunks, uint64_t& OutBlobOffset); + + private: + static constexpr uint32_t OPEN_LOCK_TIMEOUT_MS = 100; + + fextl::string FileName; + fextl::unique_ptr FD; + bool ReadOnly = false; + }; + + class IndexedDB { + public: + bool Open(const fextl::string& CacheDBName, bool ReadOnly); + void PopulateIndex(Index& CacheIndex); + bool ReadCacheBlob(uint64_t Offset, std::span OutBlob); + bool StoreCacheBlob(const MesaFOZ::foz_payload_key& Key, std::span> BlobChunks, Index& CacheIndex); + + private: + // give up after 2ms of trying to store - when we have an async thread we can increase this + static constexpr uint32_t STORE_LOCK_TIMEOUT_MS = 2; + + FOZFile CacheFOZ; + FOZFile IndexFOZ; + bool ReadOnly = false; + }; + + class DiskCache { + public: + void Init(FEXCore::Context::ContextImpl* CTX); + + std::optional Lookup(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP); + bool Store(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP, + std::span GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode, + std::span Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo); + + bool IsWritingDiskCache() const { + return (bool)RWCacheDB; + } + bool IsReadingDiskCache() const { + return !ROCacheDBs.empty() || RWCacheDB != nullptr; + } + + private: + bool OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly); + + FEXCore::Context::ContextImpl* CTX; + fextl::vector> ROCacheDBs; + fextl::unique_ptr RWCacheDB; + Index Index; + std::mutex Lock; + + FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE); + FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER); + FEX_CONFIG_OPT(BasePathOverride, DISKCACHEPATH); + FEX_CONFIG_OPT(RODBNames, DISKCACHERODBNAMES); + }; + +} // namespace DiskCache + +} // namespace FEXCore \ No newline at end of file diff --git a/FEXCore/include/FEXCore/Utils/File.h b/FEXCore/include/FEXCore/Utils/File.h index f259c147f..a288c166e 100644 --- a/FEXCore/include/FEXCore/Utils/File.h +++ b/FEXCore/include/FEXCore/Utils/File.h @@ -4,9 +4,13 @@ #include #include +#include +#include + #ifndef _WIN32 #include #include +#include #else #define WIN32_LEAN_AND_MEAN #include @@ -110,6 +114,38 @@ public: #endif } + bool Lock(uint32_t TimeoutMS) { + for (uint32_t i = 0;; ++i) { + if (TryLock()) { + return true; + } + if (i >= TimeoutMS) { + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + + bool Unlock() { + if (!Locked) { + return false; // we could return true here :thonk: + } +#ifndef _WIN32 + if (flock(Handle, LOCK_UN) == -1) { + return false; + } +#else + OVERLAPPED Overlapped {}; + Overlapped.Offset = static_cast(LOCK_SENTINEL_OFFSET); + Overlapped.OffsetHigh = static_cast(LOCK_SENTINEL_OFFSET >> 32); + if (!UnlockFileEx(Handle, 0, 1, 0, &Overlapped)) { + return false; + } +#endif + Locked = false; + return true; + } + ~File() { if (!IsValidHandle) { return; @@ -196,10 +232,32 @@ protected: , IsValidHandle {true} , Handle {Handle} {} private: + bool TryLock() { + if (Locked) { + return true; + } +#ifndef _WIN32 + if (flock(Handle, LOCK_EX | LOCK_NB) == -1) { + return false; + } +#else + // mimic posix advisory-only lock by locking some unattainably-high bit + OVERLAPPED Overlapped {}; + Overlapped.Offset = static_cast(LOCK_SENTINEL_OFFSET); + Overlapped.OffsetHigh = static_cast(LOCK_SENTINEL_OFFSET >> 32); + if (!LockFileEx(Handle, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &Overlapped)) { + return false; + } +#endif + Locked = true; + return true; + } + bool ShouldClose {}; bool IsValidHandle {}; FileHandleType Handle {}; + bool Locked = false; #ifndef _WIN32 static constexpr int DEFAULT_USER_PERMS = S_IRWXU | S_IRWXG | S_IRWXO; @@ -240,6 +298,8 @@ private: } #else static constexpr int DEFAULT_SHARE_MODE = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + static constexpr uint64_t LOCK_SENTINEL_OFFSET = 1ULL << 62; + struct Disposition { uint32_t CreationFlag; uint32_t Access; @@ -254,7 +314,11 @@ private: Disp.Access |= GENERIC_WRITE; } if ((Modes & FileModes::CREATE) == FileModes::CREATE) { - Disp.CreationFlag = CREATE_ALWAYS; + if ((Modes & FileModes::TRUNCATE) == FileModes::TRUNCATE) { + Disp.CreationFlag = CREATE_ALWAYS; + } else { + Disp.CreationFlag = OPEN_ALWAYS; + } } else { Disp.CreationFlag = OPEN_ALWAYS; } diff --git a/Scripts/reformat.sh b/Scripts/reformat.sh index 41087a072..22ad3cf84 100755 --- a/Scripts/reformat.sh +++ b/Scripts/reformat.sh @@ -35,13 +35,13 @@ if [ "$CHANGED_ONLY" = true ]; then CHANGED_FILES=$(git ls-files -m '*.cpp' '*.h' '*.inl') if [ -n "$CHANGED_FILES" ]; then - echo "$CHANGED_FILES" | xargs -d '\n' -n 1 -P "$(nproc)" clang-format-19 -i + echo "$CHANGED_FILES" | xargs -d '\n' -n 1 -P "$(nproc)" $CLANG_FORMAT -i else echo "No changed files to format." fi else # Reformat whole tree (original behavior) - git ls-files -z '*.cpp' '*.h' '*.inl' | xargs -0 -n 1 -P "$(nproc)" clang-format-19 -i + git ls-files -z '*.cpp' '*.h' '*.inl' | xargs -0 -n 1 -P "$(nproc)" $CLANG_FORMAT -i fi cd "$DIR" diff --git a/Source/Common/Config.cpp b/Source/Common/Config.cpp index 13a922e5c..5d9d2ef98 100644 --- a/Source/Common/Config.cpp +++ b/Source/Common/Config.cpp @@ -741,5 +741,6 @@ void InitializeConfigs(const PortableInformation& PortableInfo) { FEXCore::Config::SetConfigDirectory(GetConfigDirectory(true, PortableInfo), true); FEXCore::Config::SetConfigFileLocation(GetConfigFileLocation(false, PortableInfo), false); FEXCore::Config::SetConfigFileLocation(GetConfigFileLocation(true, PortableInfo), true); + FEXCore::Config::SetCacheDirectory(GetCacheDirectory()); } } // namespace FEX::Config diff --git a/Source/Tools/FEXServer/SquashFS.cpp b/Source/Tools/FEXServer/SquashFS.cpp index 545a66eff..3a2675285 100644 --- a/Source/Tools/FEXServer/SquashFS.cpp +++ b/Source/Tools/FEXServer/SquashFS.cpp @@ -40,7 +40,7 @@ static bool InitializeSquashFSPipe() { ServerRootFSLockFD = open(RootFSLockFile.c_str(), O_RDWR | O_CLOEXEC, USER_PERMS); if (ServerRootFSLockFD != -1) { // Now that we have opened the file, try to get a write lock. - flock lk { + struct flock lk { .l_type = F_WRLCK, .l_whence = SEEK_SET, .l_start = 0, @@ -67,7 +67,7 @@ static bool InitializeSquashFSPipe() { return false; } else { // FIFO file was created. Try to get a write lock - flock lk { + struct flock lk { .l_type = F_WRLCK, .l_whence = SEEK_SET, .l_start = 0, @@ -87,7 +87,7 @@ static bool InitializeSquashFSPipe() { } static bool DowngradeRootFSPipeToReadLock() { - flock lk { + struct flock lk { .l_type = F_RDLCK, .l_whence = SEEK_SET, .l_start = 0, diff --git a/Source/Windows/Common/WinAPI/IO.cpp b/Source/Windows/Common/WinAPI/IO.cpp index db07df194..b9e3359bf 100644 --- a/Source/Windows/Common/WinAPI/IO.cpp +++ b/Source/Windows/Common/WinAPI/IO.cpp @@ -187,6 +187,37 @@ DLLEXPORT_FUNC(WINBOOL, FlushFileBuffers, (HANDLE hFile)) { UNIMPLEMENTED(); } +DLLEXPORT_FUNC( + WINBOOL, LockFileEx, + (HANDLE hFile, DWORD dwFlags, DWORD dwReserved, DWORD nNumberOfBytesToLockLow, DWORD nNumberOfBytesToLockHigh, LPOVERLAPPED lpOverlapped)) { + if (!lpOverlapped || lpOverlapped->hEvent) { + UNIMPLEMENTED(); + } + LARGE_INTEGER ByteOffset; + ByteOffset.QuadPart = (static_cast(lpOverlapped->OffsetHigh) << 32) | lpOverlapped->Offset; + LARGE_INTEGER Length; + Length.QuadPart = (static_cast(nNumberOfBytesToLockHigh) << 32) | nNumberOfBytesToLockLow; + // IoStatusBlock must be null on Wine :harold: + NTSTATUS Status = NtLockFile(hFile, nullptr, nullptr, nullptr, nullptr, &ByteOffset, &Length, 0, !!(dwFlags & LOCKFILE_FAIL_IMMEDIATELY), + !!(dwFlags & LOCKFILE_EXCLUSIVE_LOCK)); + return WinAPIReturn(Status); +} + +DLLEXPORT_FUNC(WINBOOL, UnlockFileEx, + (HANDLE hFile, DWORD dwReserved, DWORD nNumberOfBytesToUnlockLow, DWORD nNumberOfBytesToUnlockHigh, LPOVERLAPPED lpOverlapped)) { + if (!lpOverlapped || lpOverlapped->hEvent) { + UNIMPLEMENTED(); + } + LARGE_INTEGER ByteOffset; + ByteOffset.QuadPart = (static_cast(lpOverlapped->OffsetHigh) << 32) | lpOverlapped->Offset; + LARGE_INTEGER Length; + Length.QuadPart = (static_cast(nNumberOfBytesToUnlockHigh) << 32) | nNumberOfBytesToUnlockLow; + // IoStatusBlock must be non-null on Wine :harold: + IO_STATUS_BLOCK IoStatusBlock; + NTSTATUS Status = NtUnlockFile(hFile, &IoStatusBlock, &ByteOffset, &Length, 0); + return WinAPIReturn(Status); +} + DLLEXPORT_FUNC(DWORD, GetFinalPathNameByHandleA, (HANDLE hFile, LPSTR lpszFilePath, DWORD cchFilePath, DWORD dwFlags)) { UNIMPLEMENTED(); } diff --git a/Source/Windows/include/winternl.h b/Source/Windows/include/winternl.h index de9e28741..2ba99ba55 100644 --- a/Source/Windows/include/winternl.h +++ b/Source/Windows/include/winternl.h @@ -524,6 +524,7 @@ NTSTATUS WINAPI NtFlushInstructionCache(HANDLE, LPCVOID, SIZE_T); NTSTATUS WINAPI NtFreeVirtualMemory(HANDLE, PVOID*, SIZE_T*, ULONG); NTSTATUS WINAPI NtGetContextThread(HANDLE, CONTEXT*); ULONG WINAPI NtGetCurrentProcessorNumber(void); +NTSTATUS WINAPI NtLockFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, PLARGE_INTEGER, PLARGE_INTEGER, ULONG, BOOLEAN, BOOLEAN); NTSYSAPI NTSTATUS WINAPI NtMapViewOfSection(HANDLE, HANDLE, PVOID*, ULONG_PTR, SIZE_T, const LARGE_INTEGER*, SIZE_T*, SECTION_INHERIT, ULONG, ULONG); NTSYSAPI NTSTATUS WINAPI NtUnmapViewOfSection(HANDLE, PVOID); NTSTATUS WINAPI NtOpenKeyEx(PHANDLE, ACCESS_MASK, const OBJECT_ATTRIBUTES*, ULONG); @@ -537,6 +538,7 @@ NTSTATUS WINAPI NtReadFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BL NTSTATUS WINAPI NtSetContextThread(HANDLE, const CONTEXT*); NTSTATUS WINAPI NtSuspendThread(HANDLE, PULONG); NTSTATUS WINAPI NtTerminateProcess(HANDLE, LONG); +NTSTATUS WINAPI NtUnlockFile(HANDLE, PIO_STATUS_BLOCK, PLARGE_INTEGER, PLARGE_INTEGER, ULONG); NTSTATUS WINAPI NtWriteFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, const void*, ULONG, PLARGE_INTEGER, PULONG); void WINAPI ProcessPendingCrossProcessEmulatorWork(); void WINAPI RtlAcquirePebLock(void);