diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index 84088e21a..ae94d50de 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -23,9 +23,6 @@ set (SRCS Interface/Core/Addressing.cpp Interface/Core/CPUID.cpp Interface/Core/Frontend.cpp - Interface/Core/ObjectCache/JobHandling.cpp - Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp - Interface/Core/ObjectCache/ObjectCacheService.cpp Interface/Core/OpcodeDispatcher/AVX_128.cpp Interface/Core/OpcodeDispatcher/Crypto.cpp Interface/Core/OpcodeDispatcher/Flags.cpp diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index 4ba6536b1..aa2977a62 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -4,9 +4,9 @@ #include "Common/JitSymbols.h" #include "Interface/Core/CPUID.h" #include "Interface/Core/X86HelperGen.h" -#include "Interface/Core/ObjectCache/ObjectCacheService.h" #include "Interface/Core/Dispatcher/Dispatcher.h" #include "Interface/IR/AOTIR.h" +#include #include #include #include @@ -33,10 +33,6 @@ namespace FEXCore { class CodeLoader; class ThunkHandler; -namespace CodeSerialize { - class CodeObjectSerializeService; -} - namespace CPU { class Arm64JITCore; class Dispatcher; @@ -50,8 +46,6 @@ namespace HLE { } // namespace FEXCore namespace FEXCore::IR { -struct IRListCopy; -class IRListView; namespace Validation { class IRValidation; } @@ -149,22 +143,7 @@ public: FEXCore::IR::AOTIRCacheEntry* LoadAOTIRCacheEntry(const fextl::string& Name) override; void UnloadAOTIRCacheEntry(FEXCore::IR::AOTIRCacheEntry* Entry) override; - void SetAOTIRLoader(AOTIRLoaderCBFn CacheReader) override { - IRCaptureCache.SetAOTIRLoader(std::move(CacheReader)); - } - void SetAOTIRWriter(AOTIRWriterCBFn CacheWriter) override { - IRCaptureCache.SetAOTIRWriter(std::move(CacheWriter)); - } - void SetAOTIRRenamer(AOTIRRenamerCBFn CacheRenamer) override { - IRCaptureCache.SetAOTIRRenamer(std::move(CacheRenamer)); - } - - void FinalizeAOTIRCache() override { - IRCaptureCache.FinalizeAOTIRCache(); - } - void WriteFilesWithCode(AOTIRCodeFileWriterFn Writer) override { - IRCaptureCache.WriteFilesWithCode(Writer); - } + void FinalizeAOTIRCache() override {} void OnCodeBufferAllocated(CPU::CodeBuffer&) override; void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override; @@ -256,7 +235,6 @@ public: X86GeneratedCode X86CodeGen; ContextImpl(const FEXCore::HostFeatures& Features); - ~ContextImpl(); static bool ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP); @@ -362,7 +340,6 @@ private: void InitializeCompiler(FEXCore::Core::InternalThreadState* Thread); IR::AOTIRCaptureCache IRCaptureCache; - fextl::unique_ptr CodeObjectCacheService; bool IsMemoryShared = false; bool SupportsHardwareTSO = false; diff --git a/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h b/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h index 9fad59179..f87ff11a0 100644 --- a/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h +++ b/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h @@ -2,7 +2,7 @@ #pragma once #include "FEXCore/Utils/EnumUtils.h" -#include "Interface/Core/ObjectCache/Relocations.h" +#include "Interface/Core/JIT/Relocations.h" #ifdef VIXL_DISASSEMBLER #include diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 4c39250c3..5123baee2 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -14,7 +14,6 @@ $end_info$ #include "Interface/Core/CPUBackend.h" #include "Interface/Core/CPUID.h" #include "Interface/Core/Frontend.h" -#include "Interface/Core/ObjectCache/ObjectCacheService.h" #include "Interface/Core/OpcodeDispatcher.h" #include "Interface/Core/JIT/JITClass.h" #include "Interface/Core/Dispatcher/Dispatcher.h" @@ -79,9 +78,6 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features) : HostFeatures {Features} , CPUID {this} , IRCaptureCache {this} { - if (Config.CacheObjectCodeCompilation() != FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE) { - CodeObjectCacheService = fextl::make_unique(this); - } if (!Config.Is64BitMode()) { // When operating in 32-bit mode, the virtual memory we care about is only the lower 32-bits. Config.VirtualMemSize = 1ULL << 32; @@ -105,14 +101,6 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features) UpdateAtomicTSOEmulationConfig(); } -ContextImpl::~ContextImpl() { - { - if (CodeObjectCacheService) { - CodeObjectCacheService->Shutdown(); - } - } -} - struct GetFrameBlockInfoResult { const CPU::CPUBackend::JITCodeHeader* InlineHeader; const CPU::CPUBackend::JITCodeTail* InlineTail; @@ -376,12 +364,6 @@ void ContextImpl::HandleCallback(FEXCore::Core::InternalThreadState* Thread, uin void ContextImpl::ExecuteThread(FEXCore::Core::InternalThreadState* Thread) { Dispatcher->ExecuteDispatch(Thread->CurrentFrame); - if (CodeObjectCacheService) { - // Ensure the Code Object Serialization service has fully serialized this thread's data before clearing the cache - // Use the thread's object cache ref counter for this - CodeSerialize::CodeObjectSerializeService::WaitForEmptyJobQueue(&Thread->ObjectCacheRefCounter); - } - // If it is the parent thread that died then just leave // TODO: This doesn't make sense when the parent thread doesn't outlive its children } @@ -482,12 +464,6 @@ void ContextImpl::OnCodeBufferAllocated(CPU::CodeBuffer& Buffer) { void ContextImpl::ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer) { FEXCORE_PROFILE_INSTANT("ClearCodeCache"); - if (CodeObjectCacheService) { - // Ensure the Code Object Serialization service has fully serialized this thread's data before clearing the cache - // Use the thread's object cache ref counter for this - CodeSerialize::CodeObjectSerializeService::WaitForEmptyJobQueue(&Thread->ObjectCacheRefCounter); - } - if (NewCodeBuffer) { // Allocate new CodeBuffer + L3 LookupCache and clear L1+L2 caches Thread->CPUBackend->ClearCache(); @@ -724,26 +700,9 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue } ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst) { - // JIT Code object cache lookup - if (CodeObjectCacheService) { - auto CodeCacheEntry = CodeObjectCacheService->FetchCodeObjectFromCache(GuestRIP); - if (CodeCacheEntry) { - auto CompiledCode = Thread->CPUBackend->RelocateJITObjectCode(GuestRIP, CodeCacheEntry); - if (CompiledCode) { - return { - .CompiledCode = {}, - .DebugData = nullptr, // nullptr here ensures that code serialization doesn't occur on from cache read - .StartAddr = 0, // Unused - .Length = 0, // Unused - .NeedsAddGuestCodeRanges = false, - }; - } - } - } - if (SourcecodeResolver && Config.GDBSymbols()) { auto AOTIRCacheEntry = SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP); - if (AOTIRCacheEntry.Entry && !AOTIRCacheEntry.Entry->ContainsCode) { + if (AOTIRCacheEntry.Entry) { AOTIRCacheEntry.Entry->SourcecodeMap = SourcecodeResolver->GenerateMap(AOTIRCacheEntry.Entry->Filename, AOTIRCacheEntry.Entry->FileId); } } @@ -843,25 +802,10 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ } } - // Tell the object cache service to serialize the code if enabled - if (CodeObjectCacheService && Config.CacheObjectCodeCompilation == FEXCore::Config::ConfigObjectCodeHandler::CONFIG_READWRITE && DebugData) { - CodeObjectCacheService->AsyncAddSerializationJob( - fextl::make_unique(CodeSerialize::AsyncJobHandler::SerializationJobData { - .GuestRIP = GuestRIP, - .GuestCodeLength = Length, - .GuestCodeHash = 0, - .HostCodeBegin = CompiledCode.BlockBegin, - .HostCodeLength = CompiledCode.Size, - .HostCodeHash = 0, - .ThreadJobRefCount = &Thread->ObjectCacheRefCounter, - .Relocations = std::move(*DebugData->Relocations), - })); - } - // Clear any relocations that might have been generated Thread->CPUBackend->ClearRelocations(); - if (IRCaptureCache.PostCompileCode(Thread, CompiledCode.BlockBegin, GuestRIP, StartAddr, Length, {}, DebugData.get(), false)) { + if (IRCaptureCache.PostCompileCode(Thread, CompiledCode.BlockBegin, GuestRIP, StartAddr, Length, DebugData.get())) { // Early exit return (uintptr_t)CodePtr; } @@ -1080,9 +1024,7 @@ IR::AOTIRCacheEntry* ContextImpl::LoadAOTIRCacheEntry(const fextl::string& filen return rv; } -void ContextImpl::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry* Entry) { - IRCaptureCache.UnloadAOTIRCacheEntry(Entry); -} +void ContextImpl::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry* Entry) {} void ContextImpl::ConfigureAOTGen(FEXCore::Core::InternalThreadState* Thread, fextl::set* ExternalBranches, uint64_t SectionMaxAddress) { Thread->FrontendDecoder->SetExternalBranches(ExternalBranches); diff --git a/FEXCore/Source/Interface/Core/ObjectCache/Relocations.h b/FEXCore/Source/Interface/Core/JIT/Relocations.h similarity index 100% rename from FEXCore/Source/Interface/Core/ObjectCache/Relocations.h rename to FEXCore/Source/Interface/Core/JIT/Relocations.h diff --git a/FEXCore/Source/Interface/Core/ObjectCache/CodeObjectSerializationConfig.h b/FEXCore/Source/Interface/Core/ObjectCache/CodeObjectSerializationConfig.h deleted file mode 100644 index 499d2f060..000000000 --- a/FEXCore/Source/Interface/Core/ObjectCache/CodeObjectSerializationConfig.h +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: MIT -#pragma once - -#include - -#include - -namespace FEXCore::CodeSerialize { -// If any of the config options mismatch on load then the cache won't be used -// Any of these will result in codegen changes -struct FEX_PACKED CodeObjectSerializationConfig { - // Cookie in the header of the file, isn't part of the config hash - uint64_t Cookie {}; - - // Instructions per block configuration - int32_t MaxInstPerBlock {}; - - // Follows CPUID 4000_0001_EAX[3:0] - unsigned Arch : 4; - - // Multiblock enabled - unsigned MultiBlock : 1; - - // Hardware TSO enabled - unsigned HardwareTSOEnabled : 1; - - // TSO enabled - unsigned TSOEnabled : 1; - - // ABI local flag unsafe optimization - unsigned ABILocalFlags : 1; - - // Paranoid TSO mode enabled - unsigned ParanoidTSO : 1; - - // Guest code execution mode (We don't support live mode switch) - unsigned Is64BitMode : 1; - - // SMC checks style - unsigned SMCChecks : 2; - - // x87 reduced precision - unsigned x87ReducedPrecision : 1; - - // Padding to remove uninitialized data warning from asan - // Shows remaining amount of bits available for config - unsigned _Pad : 19; - - bool operator==(const CodeObjectSerializationConfig& other) const { - return Cookie == other.Cookie && MaxInstPerBlock == other.MaxInstPerBlock && Arch == other.Arch && MultiBlock == other.MultiBlock && - HardwareTSOEnabled == other.HardwareTSOEnabled && TSOEnabled == other.TSOEnabled && ABILocalFlags == other.ABILocalFlags && - ParanoidTSO == other.ParanoidTSO && Is64BitMode == other.Is64BitMode && SMCChecks == other.SMCChecks && - x87ReducedPrecision == other.x87ReducedPrecision; - } - static uint64_t GetHash(const CodeObjectSerializationConfig& other) { - // For < 64-bits of data just pack directly - // Skip the cookie - uint64_t Hash {}; - Hash <<= 32; - Hash |= other.MaxInstPerBlock; - Hash <<= 1; - Hash |= other.Arch; - Hash <<= 1; - Hash |= other.MultiBlock; - Hash <<= 1; - Hash |= other.HardwareTSOEnabled; - Hash <<= 1; - Hash |= other.TSOEnabled; - Hash <<= 1; - Hash |= other.ABILocalFlags; - Hash <<= 1; - Hash |= other.ParanoidTSO; - Hash <<= 1; - Hash |= other.Is64BitMode; - Hash <<= 2; - Hash |= other.SMCChecks; - Hash <<= 1; - Hash |= other.x87ReducedPrecision; - return Hash; - } -}; - -static_assert(sizeof(CodeObjectSerializationConfig) == 16, "Size changed"); -static_assert((sizeof(CodeObjectSerializationConfig) - sizeof(uint64_t)) == 8, "Config size exceeded 64its. Need to change how the hash is " - "generated!"); -} // namespace FEXCore::CodeSerialize diff --git a/FEXCore/Source/Interface/Core/ObjectCache/JobHandling.cpp b/FEXCore/Source/Interface/Core/ObjectCache/JobHandling.cpp deleted file mode 100644 index 5c707eb87..000000000 --- a/FEXCore/Source/Interface/Core/ObjectCache/JobHandling.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: MIT -#include "Interface/Core/ObjectCache/ObjectCacheService.h" - -#include -#include -#include -#include - -#include -#include - -namespace FEXCore::CodeSerialize { -void AsyncJobHandler::AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const fextl::string& filename) { -#ifndef _WIN32 - // This function adds a named region *JOB* to our named region handler - // This needs to be as fast as possible to keep out of the way of the JIT - - const fextl::string BaseFilename = FHU::Filesystem::GetFilename(filename); - - if (!BaseFilename.empty()) { - // Create a new entry that once set up will be put in to our section object map - auto Entry = fextl::make_unique(Base, Size, Offset, filename, NamedRegionHandler->DefaultCodeHeader(Base, Offset)); - - // Lock the job ref counter so we can block anything attempting to use the entry before it is loaded - Entry->NamedJobRefCountMutex.lock(); - - CodeRegionMapType::iterator EntryIterator; - { - std::unique_lock lk {CodeObjectCacheService->GetEntryMapMutex()}; - - auto& EntryMap = CodeObjectCacheService->GetEntryMap(); - - auto it = EntryMap.emplace(Base, std::move(Entry)); - if (!it.second) { - // This happens when an application overwrites a previous region without unmapping what was there - - // Lock this entry's Named job reference counter. - // Once this passes then we know that this section has been loaded. - it.first->second->NamedJobRefCountMutex.lock(); - - // Finalize anything the region needs to do first. - CodeObjectCacheService->DoCodeRegionClosure(it.first->second->Base, it.first->second.get()); - - // munmap the file that was mapped - FEXCore::Allocator::munmap(it.first->second->CodeData, it.first->second->FileSize); - - // Remove this entry from the unrelocated map as well - { - std::unique_lock lk2 {CodeObjectCacheService->GetUnrelocatedEntryMapMutex()}; - CodeObjectCacheService->GetUnrelocatedEntryMap().erase(it.first->second->EntryHeader.OriginalBase); - } - - // Now overwrite the entry in the map - it = EntryMap.insert_or_assign(Base, std::move(Entry)); - EntryIterator = it.first; - } else { - // No overwrite, just insert - EntryIterator = it.first; - } - } - - // Now that this entry has been added to the map, we can insert a load job using the entry iterator. - // This allows us to quickly unblock the JIT thread when it is loading multiple regions and have the async thread - // do the loading for us. - // - // Create the async work queue job now so it can load - NamedRegionHandler->AsyncAddNamedRegionWorkItem(BaseFilename, filename, true, EntryIterator); - - // Tell the async thread that it has work to do - CodeObjectCacheService->NotifyWork(); - } -#endif -} - -void AsyncJobHandler::AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size) { -#ifndef _WIN32 - // Removing a named region through the job system - // We need to find the entry that we are deleting first - fextl::unique_ptr EntryPointer; - { - std::unique_lock lk {CodeObjectCacheService->GetEntryMapMutex()}; - - auto& EntryMap = CodeObjectCacheService->GetEntryMap(); - auto it = EntryMap.find(Base); - if (it != EntryMap.end()) { - // Lock the job ref counter since we are erasing it - // Once this passes it will have been loaded - it->second->NamedJobRefCountMutex.lock(); - - // Take the pointer from the map - EntryPointer = std::move(it->second); - - // We can now unmap the file data - FEXCore::Allocator::munmap(EntryPointer->CodeData, EntryPointer->FileSize); - - // Remove this from the entry map - EntryMap.erase(it); - - // Remove this entry from the unrelocated map as well - { - std::unique_lock lk2 {CodeObjectCacheService->GetUnrelocatedEntryMapMutex()}; - CodeObjectCacheService->GetUnrelocatedEntryMap().erase(EntryPointer->EntryHeader.OriginalBase); - } - } else { - // Tried to remove something that wasn't in our code object tracking - return; - } - - // Create the async work queue job now so it can finalize what it needs to do - NamedRegionHandler->AsyncRemoveNamedRegionWorkItem(Base, Size, std::move(EntryPointer)); - - // Tell the async thread that it has work to do - CodeObjectCacheService->NotifyWork(); - } -#endif -} - -void AsyncJobHandler::AsyncAddSerializationJob(fextl::unique_ptr Data) { - // XXX: Actually add serialization job -} -} // namespace FEXCore::CodeSerialize diff --git a/FEXCore/Source/Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp b/FEXCore/Source/Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp deleted file mode 100644 index 2bfc378b4..000000000 --- a/FEXCore/Source/Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: MIT -#include "Interface/Core/ObjectCache/ObjectCacheService.h" - -#include -#include - -namespace FEXCore::CodeSerialize { -NamedRegionObjectHandler::NamedRegionObjectHandler(FEXCore::Context::ContextImpl* ctx) { - DefaultSerializationConfig.Cookie = CODE_COOKIE; - - // Initialize the Arch from CPUID - uint32_t Arch = ctx->CPUID.RunFunction(0x4000'0001, 0).eax & 0xF; - DefaultSerializationConfig.Arch = Arch; - - DefaultSerializationConfig.MaxInstPerBlock = ctx->Config.MaxInstPerBlock; - DefaultSerializationConfig.MultiBlock = ctx->Config.Multiblock; - DefaultSerializationConfig.TSOEnabled = ctx->Config.TSOEnabled; - DefaultSerializationConfig.ABILocalFlags = ctx->Config.ABILocalFlags; - DefaultSerializationConfig.ParanoidTSO = ctx->Config.ParanoidTSO; - DefaultSerializationConfig.Is64BitMode = ctx->Config.Is64BitMode; - DefaultSerializationConfig.SMCChecks = ctx->Config.SMCChecks; - DefaultSerializationConfig.x87ReducedPrecision = ctx->Config.x87ReducedPrecision; -} - -void NamedRegionObjectHandler::AddNamedRegionObject(CodeRegionMapType::iterator Entry, const fextl::string& base_filename, - const fextl::string& filename, bool Executable) { - // XXX: Add named region objects - - // XXX: Until entry loading is complete just claim it is loaded - Entry->second->NamedJobRefCountMutex.unlock(); -} - -void NamedRegionObjectHandler::RemoveNamedRegionObject(uintptr_t Base, uintptr_t Size, fextl::unique_ptr Entry) { - // XXX: Remove named region objects - - // XXX: Until entry loading is complete just claim it is loaded - Entry->NamedJobRefCountMutex.unlock(); -} - -void NamedRegionObjectHandler::HandleNamedRegionObjectJobs() { - // Walk through all of our jobs sequentially until the work queue is empty - while (NamedWorkQueueJobs.load()) { - fextl::unique_ptr WorkItem; - - { - // Lock the work queue mutex for a short moment and grab an item from the list - std::unique_lock lk {NamedWorkQueueMutex}; - size_t WorkItems = WorkQueue.size(); - if (WorkItems != 0) { - WorkItem = std::move(WorkQueue.front()); - WorkQueue.pop(); - } - - // Atomically update the number of jobs - --NamedWorkQueueJobs; - } - - if (WorkItem) { - if (WorkItem->GetType() == AsyncJobHandler::NamedRegionJobType::JOB_ADD_NAMED_REGION) { - auto WorkAdd = static_cast(WorkItem.get()); - AddNamedRegionObject(WorkAdd->Entry, WorkAdd->BaseFilename, WorkAdd->Filename, WorkAdd->Executable); - } - - if (WorkItem->GetType() == AsyncJobHandler::NamedRegionJobType::JOB_REMOVE_NAMED_REGION) { - auto WorkRemove = static_cast(WorkItem.get()); - RemoveNamedRegionObject(WorkRemove->Base, WorkRemove->Size, std::move(WorkRemove->Entry)); - } - } - } -} -} // namespace FEXCore::CodeSerialize diff --git a/FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.cpp b/FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.cpp deleted file mode 100644 index e4aaef127..000000000 --- a/FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: MIT -#include "Interface/Core/ObjectCache/ObjectCacheService.h" - -#include -#include -#include - -namespace { -static void* ThreadHandler(void* Arg) { - FEXCore::CodeSerialize::CodeObjectSerializeService* This = reinterpret_cast(Arg); - This->ExecutionThread(); - return nullptr; -} -} // namespace - -namespace FEXCore::CodeSerialize { -CodeObjectSerializeService::CodeObjectSerializeService(FEXCore::Context::ContextImpl* ctx) - : CTX {ctx} - , AsyncHandler {&NamedRegionHandler, this} - , NamedRegionHandler {ctx} { - Initialize(); -} - -void CodeObjectSerializeService::Shutdown() { - if (CTX->Config.CacheObjectCodeCompilation() == FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE) { - return; - } - - WorkerThreadShuttingDown = true; - - // Kick the working thread - WorkAvailable.NotifyAll(); - - if (WorkerThread->joinable()) { - // Wait for worker thread to close down - WorkerThread->join(nullptr); - } -} - -void CodeObjectSerializeService::Initialize() { - // Add a canary so we don't crash on empty map iterator handling - auto it = AddressToEntryMap.insert_or_assign(~0ULL, fextl::make_unique()); - UnrelocatedAddressToEntryMap.insert_or_assign(~0ULL, it.first->second.get()); - - uint64_t OldMask = FEXCore::Threads::SetSignalMask(~0ULL); - WorkerThread = FEXCore::Threads::Thread::Create(ThreadHandler, this); - FEXCore::Threads::SetSignalMask(OldMask); -} - -void CodeObjectSerializeService::DoCodeRegionClosure(uint64_t Base, CodeRegionEntry* it) { - if (Base == ~0ULL) { - // Don't do closure on canary - return; - } - // XXX: Do code region closure -} - -const CodeObjectFileSection* CodeObjectSerializeService::FetchCodeObjectFromCache(uint64_t GuestRIP) { - // XXX: Actually fetch code objects from cache - return nullptr; -} - -void CodeObjectSerializeService::ExecutionThread() { - // Set our thread name so we can see its relation - FEXCore::Threads::SetThreadName("ObjectCodeSeri\0"); - while (WorkerThreadShuttingDown.load() != true) { - // Wait for work - WorkAvailable.Wait(); - - // Handle named region async jobs first. Highest priority - NamedRegionHandler.HandleNamedRegionObjectJobs(); - - // XXX: Handle code serialization jobs second. - } - - // Do final code region closures on thread shutdown - for (auto& it : AddressToEntryMap) { - DoCodeRegionClosure(it.first, it.second.get()); - } - - // Safely clear our maps now - AddressToEntryMap.clear(); - UnrelocatedAddressToEntryMap.clear(); -} -} // namespace FEXCore::CodeSerialize diff --git a/FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.h b/FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.h deleted file mode 100644 index dc7caee24..000000000 --- a/FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.h +++ /dev/null @@ -1,457 +0,0 @@ -// SPDX-License-Identifier: MIT -#pragma once -#include "Interface/Context/Context.h" -#include "Interface/Core/ObjectCache/Relocations.h" -#include "Interface/Core/ObjectCache/CodeObjectSerializationConfig.h" -#include "Interface/IR/AOTIR.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace FEXCore::CodeSerialize { -// XXX: Does this need to be signal safe? -using CodeSerializationMutex = std::shared_mutex; -struct CodeSerializationData {}; - -struct CodeObjectFileSection { - bool Serialized; - bool Invalid; - const CodeSerializationData* Data; - const char* HostCode; - uint64_t NumRelocations; - const char* Relocations; -}; - -/** - * @brief This is the file header that lives at the start of an object cache file - * - * This header is updated from multiple processes! - * Care must be taken to use OS locks when updating the file backing including this header - */ -struct CodeObjectSerializationHeader { - // The configuration that this file has - CodeObjectSerializationConfig Config; - // The original RIP that this object section was mapped at - uint64_t OriginalBase {}; - // The original offset in to the file that this object section was loaded from - uint64_t OriginalOffset {}; - // Total amount of code that should be in this file - uint64_t TotalCodeSize {}; - // Used to reserve the TSL map - uint64_t NumCodeEntries {}; - // The number of relocations that point to this section - uint64_t NumRelocationsTo {}; - // Total relocations in this file - uint64_t TotalRelocationsCount {}; -}; - -struct CodeRegionEntry { - /** - * @name Threaded initialization objects for the initial object creation - * @{ */ - // Base address in memory where the code region is at - uint64_t Base {}; - - // Size of this code entry - uint64_t Size {}; - - // The offset inside the file that is mapped to Base - uint64_t Offset {}; - - // Filename of the object - fextl::string Filename {}; - - CodeObjectSerializationHeader EntryHeader {}; - /** @} */ - - // The filename of the object cache for this entry - fextl::string ObjectEntrySourceFilename {}; - - // In the case of file corruption that we can detect, we can disable serialization early for an entry - // We should be resiliant to corruption but things happen - bool StillSerializing {true}; - - // Long lived FD for serialization if we have multiple jobs to serialize - // Bursts of code entries are common and this reduces file lock overhead - // - // Especially useful over network mounts where file locks are very slow - int CurrentSerializedFD {-1}; - - /** - * @name Objects required to sync objects between threads - * @{ */ - // Refcount for the number of outstanding code entries waiting to be written for this object section - CodeSerializationMutex ObjectJobRefCountMutex; - - // Refcount for outstanding named object region entry loading itself - // Will block JIT code cache look up when this has a unique_lock held - CodeSerializationMutex NamedJobRefCountMutex; - /** @} */ - - /** - * @name Object Entry data management - * @{ */ - - /** - * @name This is the raw file data that we loaded from the code region entry file - * @{ */ - char* CodeData {}; - size_t FileSize {}; - - fextl::vector FileCodeSections; - /** @} */ - - // This per section map takes the most time to load and needs to be quick - // This is the map of all code segments for this entry - fextl::robin_map SectionLookupMap {}; - /** @} */ - - // Default initialization - CodeRegionEntry() = default; - - // Initializer specifically for threaded loading - CodeRegionEntry(uint64_t Base, uint64_t Size, uint64_t Offset, const fextl::string& Filename, const CodeObjectSerializationHeader& DefaultHeader) - : Base {Base} - , Size {Size} - , Offset {Offset} - , Filename {Filename} - , EntryHeader {DefaultHeader} {} -}; - -// Map type must use an interator that isn't invalidation on erase/insert -using CodeRegionMapType = fextl::map>; -using CodeRegionPtrMapType = fextl::map; - -class NamedRegionObjectHandler; -class CodeObjectSerializeService; - -class AsyncJobHandler final { -public: - /** - * @brief Structure containing all the data required to async serialize code objects - */ - struct SerializationJobData { - uint64_t GuestRIP; ///< The RIP for the guest - // XXX: Support multiblock - uint64_t GuestCodeLength; ///< The Guest's code length - uint64_t GuestCodeHash; ///< Hash of the guest code - - void* HostCodeBegin; ///< Host JIT code starting memory address - size_t HostCodeLength; ///< Host JIT code length - uint64_t HostCodeHash; ///< Host JIT code hash before any backpatching - - // This is the thread specific ref counter for outstanding jobs. - // This shared mutex is incremented when the job is added, then decremented when the job is complete. - // If a thread is shutting down or clearing code cache then the thread will pull a unique lock on this mutex. - // This way it will wait until the async job handler is complete with it. - CodeSerializationMutex* ThreadJobRefCount; - - // These are the reolocations for this serialization job - // Relatively small number of entries most of the time - fextl::vector Relocations; - - /** - * @name Objects filled in from the Code Object Serialization service when a job is added - * @{ */ - // This is the code region's ref counter for outstanding jobs. - // This shared mutex is incremented when the job is added, then decremented when the job is complete. - // If a named region is being removed then a unique lock will be pulled to wait for all jobs to complete and no new jobs to be added. - CodeSerializationMutex* ObjectJobRefCountMutexPtr; - - // This is the code region iterator to reduce the number of map lookups - // This will remain valid while jobs are outstanding for this region - CodeRegionMapType::iterator CodeRegionIterator; - /** @} */ - }; - - AsyncJobHandler(NamedRegionObjectHandler* NamedRegionHandler, CodeObjectSerializeService* CodeObjectCacheService) - : NamedRegionHandler {NamedRegionHandler} - , CodeObjectCacheService {CodeObjectCacheService} {} - -protected: - friend class CodeObjectSerializeService; - friend class NamedRegionObjectHandler; - /** - * @name Async job submission functions - * @{ */ - void AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const fextl::string& filename); - void AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size); - void AsyncAddSerializationJob(fextl::unique_ptr Data); - /** @} */ - - /** - * @name Async named region handling - * @{ */ - /** - * @brief The async named region jobs to handle. - * - * Only two, Code serialization goes in to a different queue. - */ - enum class NamedRegionJobType { - JOB_ADD_NAMED_REGION, - JOB_REMOVE_NAMED_REGION, - }; - - class NamedRegionWorkItem { - public: - NamedRegionJobType GetType() const { - return Type; - } - - protected: - friend class WorkItemAddNamedRegion; - NamedRegionWorkItem(NamedRegionJobType type) - : Type {type} {} - - private: - NamedRegionJobType Type; - }; - - class WorkItemAddNamedRegion : public NamedRegionWorkItem { - public: - WorkItemAddNamedRegion(const fextl::string& base, const fextl::string& filename, bool executable, CodeRegionMapType::iterator entry) - : NamedRegionWorkItem {NamedRegionJobType::JOB_ADD_NAMED_REGION} - , BaseFilename {base} - , Filename {filename} - , Executable {executable} - , Entry {entry} {} - const fextl::string BaseFilename; - const fextl::string Filename; - bool Executable; - CodeRegionMapType::iterator Entry; - }; - - class WorkItemRemoveNamedRegion : public NamedRegionWorkItem { - public: - WorkItemRemoveNamedRegion(uint64_t base, uint64_t size, fextl::unique_ptr entry) - : NamedRegionWorkItem {NamedRegionJobType::JOB_REMOVE_NAMED_REGION} - , Base {base} - , Size {size} - , Entry {std::move(entry)} {} - - uint64_t Base; - uint64_t Size; - fextl::unique_ptr Entry; - }; - /** @} */ - -private: - NamedRegionObjectHandler* NamedRegionHandler; - CodeObjectSerializeService* CodeObjectCacheService; -}; - -class NamedRegionObjectHandler final { -public: - NamedRegionObjectHandler(FEXCore::Context::ContextImpl* ctx); - - void HandleNamedRegionObjectJobs(); - - const CodeObjectSerializationConfig& GetDefaultSerializationConfig() const { - return DefaultSerializationConfig; - } - -protected: - friend class AsyncJobHandler; - - // Return a default code header based off the default serialization config - CodeObjectSerializationHeader DefaultCodeHeader(uint64_t Base, uint64_t Offset) const { - return CodeObjectSerializationHeader { - .Config = DefaultSerializationConfig, - .OriginalBase = Base, - .OriginalOffset = Offset, - .NumCodeEntries = 0, - .NumRelocationsTo = 0, - .TotalRelocationsCount = 0, - }; - } - - /** - * @brief Adds an asynchronous add named region work item to the object queue - * - * This adds the job that will do the loading of file resources and data tracking. - */ - void AsyncAddNamedRegionWorkItem(const fextl::string& base, const fextl::string& filename, bool executable, CodeRegionMapType::iterator entry) { - std::unique_lock lk {NamedWorkQueueMutex}; - WorkQueue.emplace(fextl::make_unique(base, filename, executable, entry)); - ++NamedWorkQueueJobs; - } - - void AsyncRemoveNamedRegionWorkItem(uint64_t Base, uint64_t Size, fextl::unique_ptr Entry) { - std::unique_lock lk {NamedWorkQueueMutex}; - WorkQueue.emplace(fextl::make_unique(Base, Size, std::move(Entry))); - ++NamedWorkQueueJobs; - } - -private: - // Code version. If the code emission changes then this needs to increment - constexpr static uint32_t CODE_VERSION = 0x0; - - // Default cookie header for the file header - constexpr static uint64_t CODE_COOKIE = FEXCore::IR::COOKIE_VERSION("FEXC", CODE_VERSION); - - // Code serialization config for our current process configuration - CodeObjectSerializationConfig DefaultSerializationConfig; - - // Atomic counter for number of jobs in the queue without needing to pull the mutex to check - std::atomic NamedWorkQueueJobs {}; - - // Mutex for ading new jobs to the work queue - std::mutex NamedWorkQueueMutex {}; - - // The job queue itself - // Jobs get consumed as a FIFO - // Jobs always get appended to the end - fextl::queue> WorkQueue {}; - - /** - * @name Named Region object handling - * @{ */ - void AddNamedRegionObject(CodeRegionMapType::iterator Entry, const fextl::string& base_filename, const fextl::string& filename, bool Executable); - void RemoveNamedRegionObject(uintptr_t Base, uintptr_t Size, fextl::unique_ptr Entry); - /** @} */ -}; - -/** - * @brief Context specific code object serialization class - * - * Contains everything required for FEXCore to serialize code objects - */ -class CodeObjectSerializeService final { -public: - CodeObjectSerializeService(FEXCore::Context::ContextImpl* ctx); - - /** - * @brief Initialize the internal interface - * - * Is a public interface to allow the service to reinitialize after forking - */ - void Initialize(); - - /** - * @brief Safely shut down the Code Object serialization service. - * - * This service needs to be resiliant to application crashes, but shutting down safely is still preferred. - */ - void Shutdown(); - - /** - * @name Async interface - * @{ */ - /** - * @brief Loads a named region in to the code serialization service. As async as possible. - * - * @param Base - Virtual address that this named region is loaded - * @param Size - The size of the region - * @param Offset - The offset from the file - * @param filename - The filename itself - */ - void AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const fextl::string& filename) { - AsyncHandler.AsyncAddNamedRegionJob(Base, Size, Offset, filename); - } - - /** - * @brief Unloads a named region from the code serialization service. As async as possible. - * - * @param Base - Virtual address of the named region - * @param Size - The size of the region - */ - void AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size) { - AsyncHandler.AsyncRemoveNamedRegionJob(Base, Size); - } - - /** - * @brief Adds a code object serialization job. As async as possible. - * Code hashing happens prior to async job serialization to catch invalidations due to backpatching. - * - * @param Data - A fully filled out struct containing all the code serialization - */ - void AsyncAddSerializationJob(fextl::unique_ptr Data) { - AsyncHandler.AsyncAddSerializationJob(std::move(Data)); - } - /** @} */ - - /** - * @name Synchronous interface - * @{ */ - /** - * @brief Synchronously waits for this thread's job queue to become empty. - * - * This is necessary for when a thread is shutting down - * - * @param ThreadJobRefCount - The shared mutex to wait on until to be empty - */ - static void WaitForEmptyJobQueue(CodeSerializationMutex* ThreadJobRefCount) { - // Once the shared mutex is empty this unique lock will be gained - std::unique_lock lk {*ThreadJobRefCount}; - } - - /** - * @brief Fetches object code from the Code Object Cache for JIT. - * - * @param GuestRIP - Which GuestRIP to search the cache for - * - * @return Data required for the JIT to relocate the Object code. - */ - const CodeObjectFileSection* FetchCodeObjectFromCache(uint64_t GuestRIP); - /** @} */ - - // Public for threading - void ExecutionThread(); - -protected: - friend class AsyncJobHandler; - - /** - * @brief Safely closes out code object regions from the map - * - * @param it - iterator to do a closure on - */ - void DoCodeRegionClosure(uint64_t Base, CodeRegionEntry* it); - - CodeSerializationMutex& GetEntryMapMutex() { - return EntryMapMutex; - } - CodeSerializationMutex& GetUnrelocatedEntryMapMutex() { - return EntryMapMutex; - } - - CodeRegionMapType& GetEntryMap() { - return AddressToEntryMap; - } - CodeRegionPtrMapType& GetUnrelocatedEntryMap() { - return UnrelocatedAddressToEntryMap; - } - - /** - * @brief Notify the async thread that it has work to do - */ - void NotifyWork() { - WorkAvailable.NotifyOne(); - } - -private: - FEXCore::Context::ContextImpl* CTX; - - Event WorkAvailable {}; - fextl::unique_ptr WorkerThread; - std::atomic_bool WorkerThreadShuttingDown {false}; - AsyncJobHandler AsyncHandler; - NamedRegionObjectHandler NamedRegionHandler; - - // Mutex to hold when modifying the entry maps - CodeSerializationMutex EntryMapMutex; - CodeSerializationMutex UnrelocatedEntryMapMutex; - - // Entry maps - CodeRegionMapType AddressToEntryMap; - CodeRegionPtrMapType UnrelocatedAddressToEntryMap; -}; -} // namespace FEXCore::CodeSerialize diff --git a/FEXCore/Source/Interface/IR/AOTIR.cpp b/FEXCore/Source/Interface/IR/AOTIR.cpp index ce6c1e90f..1426d186c 100644 --- a/FEXCore/Source/Interface/IR/AOTIR.cpp +++ b/FEXCore/Source/Interface/IR/AOTIR.cpp @@ -2,311 +2,23 @@ #include "FEXHeaderUtils/Filesystem.h" #include "Interface/Context/Context.h" #include "Interface/IR/AOTIR.h" -#include "Interface/IR/IntrusiveIRList.h" -#include "Interface/IR/RegisterAllocationData.h" #include #include #include -#include #include #include -#include -#include -#include -#include -#include #include - namespace FEXCore::IR { -AOTIRInlineEntry* AOTIRInlineIndex::GetInlineEntry(uint64_t DataOffset) { - uintptr_t This = (uintptr_t)this; - - return (AOTIRInlineEntry*)(This + DataBase + DataOffset); -} - -AOTIRInlineEntry* AOTIRInlineIndex::Find(uint64_t GuestStart) { - ssize_t l = 0; - ssize_t r = Count - 1; - - while (l <= r) { - size_t m = l + (r - l) / 2; - - if (Entries[m].GuestStart == GuestStart) { - return GetInlineEntry(Entries[m].DataOffset); - } else if (Entries[m].GuestStart < GuestStart) { - l = m + 1; - } else { - r = m - 1; - } - } - - return nullptr; -} - -IR::IRListView* AOTIRInlineEntry::GetIRData() { - return (IR::IRListView*)InlineData; -} - -void AOTIRCaptureCacheEntry::AppendAOTIRCaptureCache(uint64_t GuestRIP, uint64_t Start, uint64_t Length, uint64_t Hash, - const FEXCore::IR::IRListView& IRList) { - auto Inserted = Index.emplace(GuestRIP, Stream->Offset()); - - if (Inserted.second) { - AOTIRInlineEntry entry { - .GuestHash = Hash, - .GuestLength = Length, - }; - Stream->Write((const char*)&entry, sizeof(entry)); - - // IRData (inline) - IRList.Serialize(*Stream); - } -} - -static bool readAll(int fd, void* data, size_t size) { - int rv = read(fd, data, size); - - if (rv != size) { - return false; - } else { - return true; - } -} - -static bool LoadAOTIRCache(AOTIRCacheEntry* Entry, int streamfd) { -#ifndef _WIN32 - uint64_t tag; - - if (!readAll(streamfd, (char*)&tag, sizeof(tag)) || tag != FEXCore::IR::AOTIR_COOKIE) { - return false; - } - - fextl::string Module; - uint64_t ModSize; - uint64_t IndexSize; - - lseek(streamfd, -sizeof(ModSize), SEEK_END); - - if (!readAll(streamfd, (char*)&ModSize, sizeof(ModSize))) { - return false; - } - - Module.resize(ModSize); - - lseek(streamfd, -sizeof(ModSize) - ModSize, SEEK_END); - - if (!readAll(streamfd, (char*)&Module[0], Module.size())) { - return false; - } - - if (Entry->FileId != Module) { - return false; - } - - lseek(streamfd, -sizeof(ModSize) - ModSize - sizeof(IndexSize), SEEK_END); - - if (!readAll(streamfd, (char*)&IndexSize, sizeof(IndexSize))) { - return false; - } - - struct stat fileinfo; - if (fstat(streamfd, &fileinfo) < 0) { - return false; - } - size_t Size = (fileinfo.st_size + 4095) & ~4095; - - size_t IndexOffset = fileinfo.st_size - IndexSize - sizeof(ModSize) - ModSize - sizeof(IndexSize); - - void* FilePtr = FEXCore::Allocator::mmap(nullptr, Size, PROT_READ, MAP_SHARED, streamfd, 0); - - if (FilePtr == MAP_FAILED) { - return false; - } - - auto Array = (AOTIRInlineIndex*)((char*)FilePtr + IndexOffset); - - LOGMAN_THROW_A_FMT(Entry->Array == nullptr && Entry->FilePtr == nullptr, "Entry must not be initialized here"); - Entry->Array = Array; - Entry->FilePtr = FilePtr; - Entry->Size = Size; - - LogMan::Msg::DFmt("AOTIR: Module {} has {} functions", Module, Array->Count); - - return true; -#else - return false; -#endif -} - -void AOTIRCaptureCache::FinalizeAOTIRCache() { - AOTIRCaptureCacheWriteoutQueue_Flush(); - - std::unique_lock lk(AOTIRCacheLock); - - for (auto& [String, Entry] : AOTIRCaptureCacheMap) { - if (!Entry.Stream) { - continue; - } - - const auto ModSize = String.size(); - auto& stream = Entry.Stream; - - // pad to 32 bytes - constexpr char Zero = 0; - while (stream->Offset() & 31) { - stream->Write(&Zero, 1); - } - - AOTIRInlineIndex index { - .Count = Entry.Index.size(), - .DataBase = -stream->Offset(), - }; - stream->Write((const char*)&index, sizeof(index)); - - for (const auto& [GuestStart, DataOffset] : Entry.Index) { - AOTIRInlineIndexEntry entry { - .GuestStart = GuestStart, - .DataOffset = DataOffset, - }; - - stream->Write((const char*)&entry, sizeof(entry)); - } - - // End of file header - const auto IndexSize = sizeof(AOTIRInlineIndex) + index.Count * sizeof(FEXCore::IR::AOTIRInlineIndexEntry); - stream->Write((const char*)&IndexSize, sizeof(IndexSize)); - stream->Write(String.c_str(), ModSize); - stream->Write((const char*)&ModSize, sizeof(ModSize)); - - // Close the stream - stream->Close(); - - // Rename the file to atomically update the cache with the temporary file - AOTIRRenamer(String); - } -} - -void AOTIRCaptureCache::AOTIRCaptureCacheWriteoutQueue_Flush() { - { - std::shared_lock lk {AOTIRCaptureCacheWriteoutLock}; - if (AOTIRCaptureCacheWriteoutQueue.size() == 0) { - AOTIRCaptureCacheWriteoutFlusing.store(false); - return; - } - } - - for (;;) { - // This code is tricky to refactor so it doesn't allocate memory through glibc. - // The moved std::function object deallocates memory at the end of scope. - FEXCore::Allocator::YesIKnowImNotSupposedToUseTheGlibcAllocator glibc; - - AOTIRCaptureCacheWriteoutLock.lock(); - WriteOutFn fn = std::move(AOTIRCaptureCacheWriteoutQueue.front()); - bool MaybeEmpty = false; - AOTIRCaptureCacheWriteoutQueue.pop(); - MaybeEmpty = AOTIRCaptureCacheWriteoutQueue.size() == 0; - AOTIRCaptureCacheWriteoutLock.unlock(); - - fn(); - if (MaybeEmpty) { - std::shared_lock lk {AOTIRCaptureCacheWriteoutLock}; - if (AOTIRCaptureCacheWriteoutQueue.size() == 0) { - AOTIRCaptureCacheWriteoutFlusing.store(false); - return; - } - } - } - - LOGMAN_MSG_A_FMT("Must never get here"); -} - -void AOTIRCaptureCache::AOTIRCaptureCacheWriteoutQueue_Append(const WriteOutFn& fn) { - bool Flush = false; - - { - std::unique_lock lk {AOTIRCaptureCacheWriteoutLock}; - AOTIRCaptureCacheWriteoutQueue.push(fn); - if (AOTIRCaptureCacheWriteoutQueue.size() > 10000) { - Flush = true; - } - } - - bool test_val = false; - if (Flush && AOTIRCaptureCacheWriteoutFlusing.compare_exchange_strong(test_val, true)) { - AOTIRCaptureCacheWriteoutQueue_Flush(); - } -} - -void AOTIRCaptureCache::WriteFilesWithCode(const Context::AOTIRCodeFileWriterFn& Writer) { - std::shared_lock lk(AOTIRCacheLock); - for (const auto& Entry : AOTIRCache) { - if (Entry.second.ContainsCode) { - Writer(Entry.second.FileId, Entry.second.Filename); - } - } -} - -// IRStorageBase with memory owned by IR cache -class IRInlineStorage : public IRStorageBase { - AOTIRInlineEntry& entry; -public: - IRInlineStorage(AOTIRInlineEntry& entry) - : entry(entry) {} - IRListView GetIRView() override { - return entry.GetIRData(); - } -}; - - -std::optional -AOTIRCaptureCache::PreGenerateIRFetch(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) { - auto AOTIRCacheEntry = CTX->SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP); - - PreGenerateIRFetchResult Result {}; - - if (AOTIRCacheEntry.Entry) { - AOTIRCacheEntry.Entry->ContainsCode = true; - - if (CTX->Config.AOTIRLoad()) { - auto Mod = AOTIRCacheEntry.Entry->Array; - - if (Mod != nullptr) { - auto AOTEntry = Mod->Find(GuestRIP - AOTIRCacheEntry.VAFileStart); - - if (AOTEntry) { - // verify hash - auto MappedStart = GuestRIP; - auto hash = XXH3_64bits((void*)MappedStart, AOTEntry->GuestLength); - if (hash == AOTEntry->GuestHash) { - Result.IR = fextl::make_unique(*AOTEntry); - // LogMan::Msg::DFmt("using {} + {:x} -> {:x}\n", file->second.fileid, AOTEntry->first, GuestRIP); - Result.DebugData = new FEXCore::Core::DebugData(); - Result.StartAddr = MappedStart; - Result.Length = AOTEntry->GuestLength; - return Result; - } else { - LogMan::Msg::IFmt("AOTIR: hash check failed {:x}\n", MappedStart); - } - } else { - // LogMan::Msg::IFmt("AOTIR: Failed to find {:x}, {:x}, {}\n", GuestRIP, GuestRIP - file->second.Start + file->second.Offset, file->second.fileid); - } - } - } - } - - return std::nullopt; -} bool AOTIRCaptureCache::PostCompileCode(FEXCore::Core::InternalThreadState* Thread, void* CodePtr, uint64_t GuestRIP, uint64_t StartAddr, - uint64_t Length, fextl::unique_ptr IR, - FEXCore::Core::DebugData* DebugData, bool GeneratedIR) { + uint64_t Length, FEXCore::Core::DebugData* DebugData) { // Both generated ir and LibraryJITName need a named region lookup - if (GeneratedIR || CTX->Config.LibraryJITNaming() || CTX->Config.GDBSymbols()) { + if (CTX->Config.LibraryJITNaming() || CTX->Config.GDBSymbols()) { auto AOTIRCacheEntry = CTX->SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP); @@ -318,44 +30,7 @@ bool AOTIRCaptureCache::PostCompileCode(FEXCore::Core::InternalThreadState* Thre if (CTX->Config.GDBSymbols()) { GDBJITRegister(AOTIRCacheEntry.Entry, AOTIRCacheEntry.VAFileStart, GuestRIP, (uintptr_t)CodePtr, DebugData); } - - // Add to AOT cache if aot generation is enabled - if (GeneratedIR && (CTX->Config.AOTIRCapture() || CTX->Config.AOTIRGenerate())) { - - auto hash = XXH3_64bits((void*)StartAddr, Length); - - auto LocalRIP = GuestRIP - AOTIRCacheEntry.VAFileStart; - auto LocalStartAddr = StartAddr - AOTIRCacheEntry.VAFileStart; - const auto& FileId = AOTIRCacheEntry.Entry->FileId; - - // The lambda is converted to std::function. This is tricky to refactor so it doesn't allocate memory through glibc. - // NOTE: unique_ptr must be passed as a raw pointer since std::function requires lambda captures to be copyable - FEXCore::Allocator::YesIKnowImNotSupposedToUseTheGlibcAllocator glibc; - AOTIRCaptureCacheWriteoutQueue_Append([this, LocalRIP, LocalStartAddr, Length, hash, IRRaw = IR.release(), FileId]() { - fextl::unique_ptr IR(IRRaw); - - // It is guaranteed via AOTIRCaptureCacheWriteoutLock and AOTIRCaptureCacheWriteoutFlusing that this will not run concurrently - // Memory coherency is guaranteed via AOTIRCaptureCacheWriteoutLock - - auto* AotFile = &AOTIRCaptureCacheMap[FileId]; - - if (!AotFile->Stream) { - AotFile->Stream = AOTIRWriter(FileId); - uint64_t tag = FEXCore::IR::AOTIR_COOKIE; - AotFile->Stream->Write(&tag, sizeof(tag)); - } - AotFile->AppendAOTIRCaptureCache(LocalRIP, LocalStartAddr, Length, hash, IR->GetIRView()); - }); - - if (CTX->Config.AOTIRGenerate()) { - // cleanup memory and early exit here -- we're not running the application - Thread->CPUBackend->ClearCache(); - return true; - } - } } - - // Insert to caches if we generated IR } return false; @@ -376,31 +51,10 @@ AOTIRCacheEntry* AOTIRCaptureCache::LoadAOTIRCacheEntry(const fextl::string& fil auto Inserted = AOTIRCache.insert({fileid, AOTIRCacheEntry {.FileId = fileid, .Filename = filename}}); auto Entry = &(Inserted.first->second); - LOGMAN_THROW_A_FMT(Entry->Array == nullptr, "Duplicate LoadAOTIRCacheEntry"); - - if (CTX->Config.AOTIRLoad && AOTIRLoader) { - auto streamfd = AOTIRLoader(fileid); - if (streamfd != -1) { - FEXCore::IR::LoadAOTIRCache(Entry, streamfd); - close(streamfd); - } - } return Entry; } return nullptr; } -void AOTIRCaptureCache::UnloadAOTIRCacheEntry(AOTIRCacheEntry* Entry) { -#ifndef _WIN32 - LOGMAN_THROW_A_FMT(Entry != nullptr, "Removing not existing entry"); - - if (Entry->Array) { - FEXCore::Allocator::munmap(Entry->FilePtr, Entry->Size); - Entry->Array = nullptr; - Entry->FilePtr = nullptr; - Entry->Size = 0; - } -#endif -} } // namespace FEXCore::IR diff --git a/FEXCore/Source/Interface/IR/AOTIR.h b/FEXCore/Source/Interface/IR/AOTIR.h index 1fc7fd0ac..86f2aa51b 100644 --- a/FEXCore/Source/Interface/IR/AOTIR.h +++ b/FEXCore/Source/Interface/IR/AOTIR.h @@ -1,27 +1,21 @@ // SPDX-License-Identifier: MIT #pragma once -#include "Interface/IR/RegisterAllocationData.h" -#include "Interface/IR/IntrusiveIRList.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include +#include #include +#include +#include +#include + +#include +#include namespace FEXCore::CPU { union Relocation; } // namespace FEXCore::CPU namespace FEXCore::Core { +struct InternalThreadState; struct DebugDataSubblock { uint32_t HostCodeOffset; uint32_t HostCodeSize; @@ -50,123 +44,29 @@ class ContextImpl; } namespace FEXCore::IR { -class IRListView; - -constexpr auto COOKIE_VERSION = [](const char CookieText[4], uint32_t Version) { - uint64_t Cookie = Version; - Cookie <<= 32; - - // Make the cookie text be the lower bits - Cookie |= CookieText[3]; - Cookie <<= 8; - Cookie |= CookieText[2]; - Cookie <<= 8; - Cookie |= CookieText[1]; - Cookie <<= 8; - Cookie |= CookieText[0]; - - return Cookie; -}; -constexpr static uint32_t AOTIR_VERSION = 0x0000'00004; -constexpr static uint64_t AOTIR_COOKIE = COOKIE_VERSION("FEXI", AOTIR_VERSION); - -struct AOTIRInlineEntry { - uint64_t GuestHash; - uint64_t GuestLength; - - /* IRData */ - uint8_t InlineData[0]; - - IR::IRListView* GetIRData(); -}; - -struct AOTIRInlineIndexEntry { - uint64_t GuestStart; - uint64_t DataOffset; -}; - -struct AOTIRInlineIndex { - uint64_t Count; - uint64_t DataBase; - AOTIRInlineIndexEntry Entries[0]; - - AOTIRInlineEntry* Find(uint64_t GuestStart); - AOTIRInlineEntry* GetInlineEntry(uint64_t DataOffset); -}; - -struct AOTIRCaptureCacheEntry { - fextl::unique_ptr Stream; - fextl::map Index; - - void AppendAOTIRCaptureCache(uint64_t GuestRIP, uint64_t Start, uint64_t Length, uint64_t Hash, const FEXCore::IR::IRListView& IRList); -}; struct AOTIRCacheEntry { - AOTIRInlineIndex* Array; - void* FilePtr; - size_t Size; fextl::unique_ptr SourcecodeMap; fextl::string FileId; fextl::string Filename; - bool ContainsCode; }; -using AOTCacheType = fextl::unordered_map; - class AOTIRCaptureCache final { public: - using WriteOutFn = std::function; - AOTIRCaptureCache(FEXCore::Context::ContextImpl* ctx) : CTX {ctx} {} - void FinalizeAOTIRCache(); - void AOTIRCaptureCacheWriteoutQueue_Flush(); - void AOTIRCaptureCacheWriteoutQueue_Append(const WriteOutFn& fn); - void WriteFilesWithCode(const Context::AOTIRCodeFileWriterFn& Writer); - - struct PreGenerateIRFetchResult { - fextl::unique_ptr IR; - FEXCore::Core::DebugData* DebugData {}; - uint64_t StartAddr {}; - uint64_t Length {}; - }; - [[nodiscard]] - std::optional PreGenerateIRFetch(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP); - bool PostCompileCode(FEXCore::Core::InternalThreadState* Thread, void* CodePtr, uint64_t GuestRIP, uint64_t StartAddr, uint64_t Length, - fextl::unique_ptr IR, FEXCore::Core::DebugData* DebugData, bool GeneratedIR); + FEXCore::Core::DebugData* DebugData); AOTIRCacheEntry* LoadAOTIRCacheEntry(const fextl::string& filename); - void UnloadAOTIRCacheEntry(AOTIRCacheEntry* Entry); - - // Callbacks - void SetAOTIRLoader(Context::AOTIRLoaderCBFn CacheReader) { - AOTIRLoader = std::move(CacheReader); - } - - void SetAOTIRWriter(Context::AOTIRWriterCBFn CacheWriter) { - AOTIRWriter = std::move(CacheWriter); - } - - void SetAOTIRRenamer(Context::AOTIRRenamerCBFn CacheRenamer) { - AOTIRRenamer = std::move(CacheRenamer); - } private: FEXCore::Context::ContextImpl* CTX; std::shared_mutex AOTIRCacheLock; - std::shared_mutex AOTIRCaptureCacheWriteoutLock; - std::atomic AOTIRCaptureCacheWriteoutFlusing; - fextl::queue AOTIRCaptureCacheWriteoutQueue; - - FEXCore::IR::AOTCacheType AOTIRCache; - - Context::AOTIRLoaderCBFn AOTIRLoader; - Context::AOTIRWriterCBFn AOTIRWriter; - Context::AOTIRRenamerCBFn AOTIRRenamer; - fextl::unordered_map AOTIRCaptureCacheMap; + fextl::unordered_map AOTIRCache; }; + } // namespace FEXCore::IR diff --git a/FEXCore/Source/Interface/IR/IntrusiveIRList.h b/FEXCore/Source/Interface/IR/IntrusiveIRList.h index 64f3b55e8..ac1bf5d5f 100644 --- a/FEXCore/Source/Interface/IR/IntrusiveIRList.h +++ b/FEXCore/Source/Interface/IR/IntrusiveIRList.h @@ -156,44 +156,6 @@ public: , DataSize(DataSize_) , ListSize(ListSize_) {} - void Serialize(FEXCore::Context::AOTIRWriter& stream) const { - void* nul = nullptr; - // void *IRDataInternal; - stream.Write((const char*)&nul, sizeof(nul)); - // void *ListDataInternal; - stream.Write((const char*)&nul, sizeof(nul)); - // size_t DataSize; - stream.Write((const char*)&DataSize, sizeof(DataSize)); - // size_t ListSize; - stream.Write((const char*)&ListSize, sizeof(ListSize)); - - // inline data - stream.Write((const char*)GetData(), DataSize); - stream.Write((const char*)GetListData(), ListSize); - } - - void Serialize(uint8_t* ptr) const { - void* nul = nullptr; - // void *IRDataInternal; - memcpy(ptr, &nul, sizeof(nul)); - ptr += sizeof(nul); - // void *ListDataInternal; - memcpy(ptr, &nul, sizeof(nul)); - ptr += sizeof(nul); - // size_t DataSize; - memcpy(ptr, &DataSize, sizeof(DataSize)); - ptr += sizeof(DataSize); - // size_t ListSize; - memcpy(ptr, &ListSize, sizeof(ListSize)); - ptr += sizeof(ListSize); - - // inline data - memcpy(ptr, (const void*)GetData(), DataSize); - ptr += DataSize; - memcpy(ptr, (const void*)GetListData(), ListSize); - ptr += ListSize; - } - [[nodiscard]] size_t GetInlineSize() const { static_assert(sizeof(*this) == 32); @@ -411,11 +373,4 @@ private: uint8_t InlineData[0]; }; -class IRStorageBase { -public: - virtual ~IRStorageBase() = default; - - virtual IRListView GetIRView() = 0; -}; - } // namespace FEXCore::IR diff --git a/FEXCore/include/FEXCore/Core/Context.h b/FEXCore/include/FEXCore/Core/Context.h index 25e8606a4..201987886 100644 --- a/FEXCore/include/FEXCore/Core/Context.h +++ b/FEXCore/include/FEXCore/Core/Context.h @@ -44,24 +44,12 @@ class IREmitter; } // namespace FEXCore::IR namespace FEXCore::Context { -class Context; enum OperatingMode { MODE_32BIT, MODE_64BIT, }; -/** - * @brief IR Serialization handler class. - */ -class AOTIRWriter { -public: - virtual ~AOTIRWriter() = default; - virtual void Write(const void* Data, size_t Size) = 0; - virtual size_t Offset() = 0; - virtual void Close() = 0; -}; - using CodeRangeInvalidationFn = std::function; // Nested vector of guest block entrypoints @@ -71,11 +59,6 @@ using CustomIREntrypointHandler = std::function; -using AOTIRCodeFileWriterFn = std::function; -using AOTIRLoaderCBFn = std::function; -using AOTIRRenamerCBFn = std::function; -using AOTIRWriterCBFn = std::function(const fextl::string&)>; - class Context { public: virtual ~Context() = default; @@ -168,12 +151,7 @@ public: FEX_DEFAULT_VISIBILITY virtual FEXCore::IR::AOTIRCacheEntry* LoadAOTIRCacheEntry(const fextl::string& Name) = 0; FEX_DEFAULT_VISIBILITY virtual void UnloadAOTIRCacheEntry(FEXCore::IR::AOTIRCacheEntry* Entry) = 0; - FEX_DEFAULT_VISIBILITY virtual void SetAOTIRLoader(AOTIRLoaderCBFn CacheReader) = 0; - FEX_DEFAULT_VISIBILITY virtual void SetAOTIRWriter(AOTIRWriterCBFn CacheWriter) = 0; - FEX_DEFAULT_VISIBILITY virtual void SetAOTIRRenamer(AOTIRRenamerCBFn CacheRenamer) = 0; - FEX_DEFAULT_VISIBILITY virtual void FinalizeAOTIRCache() = 0; - FEX_DEFAULT_VISIBILITY virtual void WriteFilesWithCode(AOTIRCodeFileWriterFn Writer) = 0; FEX_DEFAULT_VISIBILITY virtual void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) = 0; FEX_DEFAULT_VISIBILITY virtual void InvalidateGuestCodeRange( diff --git a/Source/Tools/FEXLoader/FEXLoader.cpp b/Source/Tools/FEXLoader/FEXLoader.cpp index b2dea73e7..ba32dc8c6 100644 --- a/Source/Tools/FEXLoader/FEXLoader.cpp +++ b/Source/Tools/FEXLoader/FEXLoader.cpp @@ -99,42 +99,6 @@ void AssertHandler(const char* Message) { } } // namespace FEXServerLogging -namespace AOTIR { -class AOTIRWriterFD final : public FEXCore::Context::AOTIRWriter { -public: - AOTIRWriterFD(const fextl::string& Path) { - // Create and truncate if exists. - constexpr int USER_PERMS = S_IRWXU | S_IRWXG | S_IRWXO; - FD = open(Path.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, USER_PERMS); - } - - operator bool() const { - return FD != -1; - } - - void Write(const void* Data, size_t Size) override { - write(FD, Data, Size); - } - - size_t Offset() override { - return lseek(FD, 0, SEEK_CUR); - } - - void Close() override { - if (FD != -1) { - close(FD); - FD = -1; - } - } - - virtual ~AOTIRWriterFD() { - Close(); - } -private: - int FD {-1}; -}; -} // namespace AOTIR - bool InterpreterHandler(fextl::string* Filename, const fextl::string& RootFS, fextl::vector* args) { int FD {-1}; @@ -612,67 +576,11 @@ int main(int argc, char** argv, char** const envp) { SyscallHandler->DeserializeSeccompFD(ParentThread, FEXSeccompFD); - const bool AOTEnabled = AOTIRLoad() || AOTIRCapture() || AOTIRGenerate(); - if (AOTEnabled) { - LogMan::Msg::IFmt("Warning: AOTIR is experimental, and might lead to crashes. " - "Capture doesn't work with programs that fork."); - - CTX->SetAOTIRLoader([](const fextl::string& fileid) -> int { - const auto filepath = fextl::fmt::format("{}/aotir/{}.aotir", FEXCore::Config::GetDataDirectory(), fileid); - return open(filepath.c_str(), O_RDONLY); - }); - - CTX->SetAOTIRWriter([](const fextl::string& fileid) -> fextl::unique_ptr { - const auto filepath = fextl::fmt::format("{}/aotir/{}.aotir.tmp", FEXCore::Config::GetDataDirectory(), fileid); - auto AOTWrite = fextl::make_unique(filepath); - if (*AOTWrite) { - LogMan::Msg::IFmt("AOTIR: Storing {}", fileid); - } else { - LogMan::Msg::IFmt("AOTIR: Failed to store {}", fileid); - } - return AOTWrite; - }); - - CTX->SetAOTIRRenamer([](const fextl::string& fileid) -> void { - const auto TmpFilepath = fextl::fmt::format("{}/aotir/{}.aotir.tmp", FEXCore::Config::GetDataDirectory(), fileid); - const auto NewFilepath = fextl::fmt::format("{}/aotir/{}.aotir", FEXCore::Config::GetDataDirectory(), fileid); - - // Rename the temporary file to atomically update the file - if (!FHU::Filesystem::RenameFile(TmpFilepath, NewFilepath)) { - LogMan::Msg::IFmt("Couldn't rename aotir"); - } - }); - } - - if (AOTIRGenerate()) { - for (auto& Section : Loader.Sections) { - FEX::AOT::AOTGenSection(CTX.get(), Section); - } - } else { - CTX->ExecuteThread(ParentThread->Thread); - } + CTX->ExecuteThread(ParentThread->Thread); DebugServer.reset(); SyscallHandler->TM.Stop(); - if (AOTEnabled) { - if (FHU::Filesystem::CreateDirectories(fextl::fmt::format("{}/aotir", FEXCore::Config::GetDataDirectory()))) { - CTX->WriteFilesWithCode([](const fextl::string& fileid, const fextl::string& filename) { - const auto filepath = fextl::fmt::format("{}/aotir/{}.path", FEXCore::Config::GetDataDirectory(), fileid); - int fd = open(filepath.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0644); - if (fd != -1) { - write(fd, filename.c_str(), filename.size()); - close(fd); - } - }); - } - - if (AOTIRCapture() || AOTIRGenerate()) { - CTX->FinalizeAOTIRCache(); - LogMan::Msg::IFmt("AOTIR Cache Stored"); - } - } - auto ProgramStatus = ParentThread->StatusCode; SignalDelegation->UninstallTLSState(ParentThread);