diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index 5948fccb2..70d67ea4c 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -314,7 +314,7 @@ bool CodeCache::SaveData(Core::InternalThreadState& Thread, int fd, const Execut std::ranges::copy(GIT_HASH, header.FEXVersion); header.NumBlocks = LookupCache.BlockList.size(); header.NumCodePages = LookupCache.CodePages.size(); - header.CodeBufferSize = FEXCore::AlignUp(CTX.GetAllocatedSize(), Utils::FEX_PAGE_SIZE); + header.CodeBufferSize = FEXCore::AlignUp(CodeBuffer->GetAllocatedSize(), Utils::FEX_PAGE_SIZE); header.NumRelocations = Relocations.size(); header.SerializedBaseAddress = SerializedBaseAddress; ::write(fd, &header, sizeof(header)); @@ -361,7 +361,8 @@ bool CodeCache::SaveData(Core::InternalThreadState& Thread, int fd, const Execut } // Dump the host code (relocated for position-independent serialization) - std::span CodeBufferData(reinterpret_cast(CodeBuffer->Ptr), reinterpret_cast(CodeBuffer->Ptr) + CTX.GetAllocatedSize()); + std::span CodeBufferData(reinterpret_cast(CodeBuffer->Ptr), + reinterpret_cast(CodeBuffer->Ptr) + CodeBuffer->GetAllocatedSize()); if (!ApplyCodeRelocations(SerializedBaseAddress, CodeBufferData, Relocations, 0, true)) { LOGMAN_THROW_A_FMT(false, "Failed to apply code relocations"); return false; @@ -446,10 +447,10 @@ void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::setGetAllocatedSize() <= CodeBufferRangeRef.size()) { + if (NewCodeBuffer->GetAllocatedSize() <= CodeBufferRangeRef.size()) { // Reference compilation produced fewer bytes than our cache, so validation is going to fail. // Make sure we don't output any garbage bytes though. - CodeBufferRangeRef = CodeBufferRangeRef.subspan(0, ValidationCTX->GetAllocatedSize()); + CodeBufferRangeRef = CodeBufferRangeRef.subspan(0, NewCodeBuffer->GetAllocatedSize()); } auto [Mismatch, _] = std::mismatch(CodeBufferRangeRef.begin(), CodeBufferRangeRef.end(), CachedCode.begin()); @@ -500,7 +501,7 @@ void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::setLookupCache->ClearCache(ValidationThread->LookupCache->AcquireWriteLock()); - ValidationCTX->CodeBufferOffset = ValidationCTX->CodeBufferBase; + NewCodeBuffer->CodeBufferOffset = NewCodeBuffer->Ptr; LogMan::Msg::IFmt(" successfully validated cache"); } diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 1228e8bd4..8f509ec9f 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -816,8 +816,8 @@ void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool C } -SharedCodeBufferManager::CodeBufferAllocation Arm64JITCore::AllocateCodeBufferInSharedCache(size_t Size) { - SharedCodeBufferManager::CodeBufferAllocation AllocatedInfo {}; +CodeBuffer::CodeBufferAllocation Arm64JITCore::AllocateCodeBufferInSharedCache(size_t Size) { + CodeBuffer::CodeBufferAllocation AllocatedInfo {}; LOGMAN_THROW_A_FMT(CurrentCodeBuffer->LookupCache.get() == ThreadState->LookupCache->Shared, "INVARIANT VIOLATED: SharedLookupCache " "doesn't match up!\n"); // Bring CodeBuffer up to date @@ -829,7 +829,7 @@ SharedCodeBufferManager::CodeBufferAllocation Arm64JITCore::AllocateCodeBufferIn // Attempt to allocate a buffer from the SharedCodeBuffers. while (AllocatedInfo.BufferAllocationOffset == nullptr) { - AllocatedInfo = SharedCodeBuffers.AtomicAllocateBuffer(Size); + AllocatedInfo = CurrentCodeBuffer->AtomicAllocateBuffer(Size); if (AllocatedInfo.BufferAllocationOffset == nullptr) { // If it didn't fit then clear the buffer and try again. diff --git a/FEXCore/Source/Interface/Core/JIT/JITClass.h b/FEXCore/Source/Interface/Core/JIT/JITClass.h index aeccf73be..e42e6cb3b 100644 --- a/FEXCore/Source/Interface/Core/JIT/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/JITClass.h @@ -627,7 +627,7 @@ private: void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF); - [[nodiscard]] SharedCodeBufferManager::CodeBufferAllocation AllocateCodeBufferInSharedCache(size_t Size); + [[nodiscard]] CodeBuffer::CodeBufferAllocation AllocateCodeBufferInSharedCache(size_t Size); #define DEF_OP(x) void Op_##x(IR::IROp_Header const* IROp, IR::Ref Node) diff --git a/FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp index 781618139..78d6108d6 100644 --- a/FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp +++ b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp @@ -34,6 +34,9 @@ CodeBuffer::CodeBuffer(size_t Size) FEXCore::Allocator::VirtualTHPControl(Ptr, Size, FEXCore::Allocator::THPControl::Enable); LookupCache = fextl::make_unique(); + + CodeBufferEnd = Ptr + UsableSize(); + CodeBufferOffset = Ptr; } CodeBuffer::~CodeBuffer() { @@ -66,9 +69,6 @@ fextl::shared_ptr SharedCodeBufferManager::AllocateNew(size_t Size) auto Buffer = fextl::make_shared(Size); Latest = Buffer; - CodeBufferBase = Buffer->Ptr; - CodeBufferEnd = Buffer->Ptr + Buffer->UsableSize(); - CodeBufferOffset = CodeBufferBase; OnCodeBufferAllocated(Buffer); diff --git a/FEXCore/Source/Interface/Core/SharedCodeBufferManager.h b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.h index a3ffa5765..f81196cfb 100644 --- a/FEXCore/Source/Interface/Core/SharedCodeBufferManager.h +++ b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.h @@ -22,8 +22,12 @@ struct GuestToHostMap; namespace FEXCore::CPU { struct CodeBuffer { uint8_t* Ptr; + uint8_t* CodeBufferEnd; size_t AllocatedSize; // including guard page; see UsableSize() + // Code buffer allocation information. + std::atomic CodeBufferOffset {}; + fextl::unique_ptr LookupCache; CodeBuffer(size_t Size); @@ -38,35 +42,6 @@ struct CodeBuffer { size_t UsableSize() const { return AllocatedSize - FEXCore::Utils::FEX_PAGE_SIZE; } -}; - -/** - * A manager that coordinates access to the CodeBuffer used for compiling new code across threads. - * - * The CodeBuffer is managed as a partially persistent data structure: - * - Exactly one CodeBuffer is now designated as "active", which means data can be appended to it - * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads) - * - Instead, such lossy modifications trigger a new "version" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads. - * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version - */ -class SharedCodeBufferManager { -public: - // Get the CodeBuffer that was most recently allocated. - // This is the only CodeBuffer that data may be written to. - fextl::shared_ptr GetLatest(); - - // Allocate a new CodeBuffer with geometric growth up to an internal maximum. - // Subsequent calls to GetLatest will point to the returned buffer. - fextl::shared_ptr StartLargerCodeBuffer(); - - // Allocate a new CodeBuffer with maximum internal size. - // Subsequent calls to GetLatest will point to the returned buffer. - fextl::shared_ptr StartMaximalCodeBuffer(); - - // Write offset into the latest CodeBuffer - uint8_t* CodeBufferBase {}; - uint8_t* CodeBufferEnd {}; - std::atomic CodeBufferOffset {}; // Atomically allocate a fixed size buffer out of the current allocated codebuffer. // Lockless because it's just a linear allocator. @@ -98,14 +73,40 @@ public: // Managed to fit. return { - .BufferBase = CodeBufferBase, + .BufferBase = Ptr, .BufferAllocationOffset = ExpectedOffset, }; } size_t GetAllocatedSize() const { - return CodeBufferOffset - CodeBufferBase; + return CodeBufferOffset - Ptr; } +}; + +/** + * A manager that coordinates access to the CodeBuffer used for compiling new code across threads. + * + * The CodeBuffer is managed as a partially persistent data structure: + * - Exactly one CodeBuffer is now designated as "active", which means data can be appended to it + * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads) + * - Instead, such lossy modifications trigger a new "version" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads. + * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version + */ +class SharedCodeBufferManager { +public: + virtual ~SharedCodeBufferManager() = default; + + // Get the CodeBuffer that was most recently allocated. + // This is the only CodeBuffer that data may be written to. + fextl::shared_ptr GetLatest(); + + // Allocate a new CodeBuffer with geometric growth up to an internal maximum. + // Subsequent calls to GetLatest will point to the returned buffer. + fextl::shared_ptr StartLargerCodeBuffer(); + + // Allocate a new CodeBuffer with maximum internal size. + // Subsequent calls to GetLatest will point to the returned buffer. + fextl::shared_ptr StartMaximalCodeBuffer(); virtual void OnCodeBufferAllocated(const std::shared_ptr&) {};