diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index 850a96e30..ad2186989 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -24,6 +24,7 @@ set(SRCS Interface/Core/Addressing.cpp Interface/Core/CPUID.cpp Interface/Core/Frontend.cpp + Interface/Core/SharedCodeBufferManager.cpp Interface/Core/OpcodeDispatcher/AVX_128.cpp Interface/Core/OpcodeDispatcher/Crypto.cpp Interface/Core/OpcodeDispatcher/Flags.cpp diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index eb04799ff..b8b68b7c7 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -4,6 +4,7 @@ #include "Common/JitSymbols.h" #include "Interface/Core/CPUBackend.h" #include "Interface/Core/CPUID.h" +#include "Interface/Core/SharedCodeBufferManager.h" #include #include #include @@ -130,7 +131,7 @@ public: uint32_t RelocationOffset, bool ForStorage); }; -class ContextImpl final : public FEXCore::Context::Context, public CPU::CodeBufferManager { +class ContextImpl final : public FEXCore::Context::Context, public CPU::SharedCodeBufferManager { public: // Context base class implementation. bool InitCore() override; diff --git a/FEXCore/Source/Interface/Core/CPUBackend.cpp b/FEXCore/Source/Interface/Core/CPUBackend.cpp index 1ca72539b..e7eac0b03 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.cpp +++ b/FEXCore/Source/Interface/Core/CPUBackend.cpp @@ -11,17 +11,8 @@ #include -#ifndef _WIN32 -#include -#endif - namespace FEXCore { namespace CPU { - - static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16; - // We don't want to move above 128MB atm because that means we will have to encode longer jumps - static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128; - constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = { {0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX {0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER @@ -275,9 +266,9 @@ namespace CPU { return TotalLUT; }()}; - CPUBackend::CPUBackend(CodeBufferManager& CodeBuffers, FEXCore::Core::InternalThreadState* ThreadState) + CPUBackend::CPUBackend(SharedCodeBufferManager& SharedCodeBuffers, FEXCore::Core::InternalThreadState* ThreadState) : ThreadState(ThreadState) - , CodeBuffers(CodeBuffers) { + , SharedCodeBuffers(SharedCodeBuffers) { auto& Ptrs = ThreadState->CurrentFrame->Pointers; @@ -316,11 +307,11 @@ namespace CPU { CPUBackend::~CPUBackend() = default; - auto CPUBackend::GetEmptyCodeBuffer() -> CodeBuffer* { + auto CPUBackend::GetEmptySharedCodeBuffer() -> CodeBuffer* { auto PrevCodeBuffer = CurrentCodeBuffer; // Resize the code buffer and reallocate our code size - CurrentCodeBuffer = CodeBuffers.StartLargerCodeBuffer(); + CurrentCodeBuffer = SharedCodeBuffers.StartLargerCodeBuffer(); RegisterForSignalHandler(std::move(PrevCodeBuffer)); return CurrentCodeBuffer.get(); @@ -338,7 +329,7 @@ namespace CPU { } fextl::shared_ptr CPUBackend::CheckCodeBufferUpdate() { - auto NewCodeBuffer = CodeBuffers.GetLatest(); + auto NewCodeBuffer = SharedCodeBuffers.GetLatest(); if (CurrentCodeBuffer != NewCodeBuffer) { RegisterForSignalHandler(CurrentCodeBuffer); return std::exchange(CurrentCodeBuffer, NewCodeBuffer); @@ -346,89 +337,6 @@ namespace CPU { return nullptr; } - CodeBuffer::CodeBuffer(size_t Size) - : AllocatedSize(Size) { - Ptr = static_cast(FEXCore::Allocator::VirtualAlloc(Size, true)); - LOGMAN_THROW_A_FMT(!!Ptr, "Couldn't allocate code buffer"); - - // Protect the last page of the allocated buffer to trigger SIGSEGV on write access - uintptr_t LastPageAddr = AlignDown(reinterpret_cast(Ptr) + Size - 1, FEXCore::Utils::FEX_PAGE_SIZE); - if (!FEXCore::Allocator::VirtualProtect(reinterpret_cast(LastPageAddr), FEXCore::Utils::FEX_PAGE_SIZE, - FEXCore::Allocator::ProtectOptions::None)) { - LogMan::Msg::EFmt("Failed to mprotect last page of code buffer."); - } - - FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size); - - // Huge-pages reduce the amount of iTLB misses dramatically when it works. - FEXCore::Allocator::VirtualTHPControl(Ptr, Size, FEXCore::Allocator::THPControl::Enable); - - LookupCache = fextl::make_unique(); - } - - CodeBuffer::~CodeBuffer() { - FEXCore::Allocator::VirtualFree(Ptr, AllocatedSize); - } - - auto CodeBufferManager::AllocateNew(size_t Size) -> fextl::shared_ptr { -#ifndef _WIN32 -// MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature. -// It's equivalent to systemd's `MemoryDenyWriteExecute` but implemented entirely in the kernel. -// -// MDWE prevents applications from creating RWX memory mappings. -// This prevents FEX from doing anything JIT related, as FEX uses RWX for JIT memory mappings. -// -// A potential workaround to make FEX work with MDWE is to call mprotect every time we need to write or modify code. -// Alternatively, FEX could use a memory mirror where one half is mapped as RW and the other is RX. -// -// Once MDWE is enabled with the prctl, the feature is sealed and it can /NOT/ be turned off. -// -// Status of MDWE is queried through prctl using `PR_GET_MDWE`: -// -1: The kernel doesn't support MDWE -// 0: MDWE is supported but disabled -// >0: MDWE is enabled, hence prohibiting RWX mappings -#ifndef PR_GET_MDWE -#define PR_GET_MDWE 66 -#endif - int MDWE = ::prctl(PR_GET_MDWE, 0, 0, 0, 0); - if (MDWE != -1 && MDWE != 0) { - LogMan::Msg::EFmt("MDWE was set to 0x{:x} which means FEX can't allocate executable memory", MDWE); - } -#endif - - auto Buffer = fextl::make_shared(Size); - - Latest = Buffer; - LatestOffset = 0; - - OnCodeBufferAllocated(Buffer); - - return Buffer; - } - - fextl::shared_ptr CodeBufferManager::GetLatest() { - if (!Latest) { - AllocateNew(INITIAL_CODE_SIZE); - } - return Latest; - } - - fextl::shared_ptr CodeBufferManager::StartLargerCodeBuffer() { - if (!Latest) { - // Allocate initial CodeBuffer and return it - return GetLatest(); - } - - auto NewCodeBufferSize = GetLatest()->AllocatedSize; - NewCodeBufferSize = std::min(NewCodeBufferSize * 2, MAX_CODE_SIZE); - return AllocateNew(NewCodeBufferSize); - } - - fextl::shared_ptr CodeBufferManager::StartMaximalCodeBuffer() { - return AllocateNew(MAX_CODE_SIZE); - } - - bool CPUBackend::IsAddressInCodeBuffer(uintptr_t Address) const { const auto CheckCodeBuffer = [](const CodeBuffer& Buffer, uintptr_t Address) { const auto BufferPtr = reinterpret_cast(Buffer.Ptr); diff --git a/FEXCore/Source/Interface/Core/CPUBackend.h b/FEXCore/Source/Interface/Core/CPUBackend.h index cc8e7db21..f80a1ecc2 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.h +++ b/FEXCore/Source/Interface/Core/CPUBackend.h @@ -8,6 +8,8 @@ $end_info$ #pragma once +#include "Interface/Core/SharedCodeBufferManager.h" + #include #include #include @@ -41,67 +43,10 @@ namespace CodeSerialize { struct GuestToHostMap; namespace CPU { - struct CodeBuffer { - uint8_t* Ptr; - size_t AllocatedSize; // including guard page; see UsableSize() - - fextl::unique_ptr LookupCache; - - CodeBuffer(size_t Size); - CodeBuffer(const CodeBuffer&) = delete; - CodeBuffer& operator=(const CodeBuffer&) = delete; - CodeBuffer(CodeBuffer&& oth) = delete; - CodeBuffer& operator=(CodeBuffer&&) = delete; - - ~CodeBuffer(); - - /// Returns the number of bytes available for storing code - size_t UsableSize() const { - return AllocatedSize - FEXCore::Utils::FEX_PAGE_SIZE; - } - }; - - /** - * A manager that coordinates access to the CodeBuffer used for compiling new code across threads. - * - * The CodeBuffer is managed as a partially persistent data structure: - * - Exactly one CodeBuffer is now designated as "active", which means data can be appended to it - * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads) - * - Instead, such lossy modifications trigger a new "version" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads. - * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version - */ - class CodeBufferManager { - public: - // Get the CodeBuffer that was most recently allocated. - // This is the only CodeBuffer that data may be written to. - fextl::shared_ptr GetLatest(); - - // Allocate a new CodeBuffer with geometric growth up to an internal maximum. - // Subsequent calls to GetLatest will point to the returned buffer. - fextl::shared_ptr StartLargerCodeBuffer(); - - // Allocate a new CodeBuffer with maximum internal size. - // Subsequent calls to GetLatest will point to the returned buffer. - fextl::shared_ptr StartMaximalCodeBuffer(); - - // Write offset into the latest CodeBuffer - std::size_t LatestOffset {}; - - // Protects writes to the latest CodeBuffer and changes to LatestOffset - FEXCore::ForkableUniqueMutex CodeBufferWriteMutex; - - virtual void OnCodeBufferAllocated(const std::shared_ptr&) {}; - - private: - fextl::shared_ptr Latest; - - fextl::shared_ptr AllocateNew(size_t Size); - }; - class CPUBackend { public: - CPUBackend(CodeBufferManager&, FEXCore::Core::InternalThreadState*); + CPUBackend(SharedCodeBufferManager&, FEXCore::Core::InternalThreadState*); virtual ~CPUBackend(); @@ -194,7 +139,7 @@ namespace CPU { FEXCore::Core::InternalThreadState* ThreadState; [[nodiscard]] - CodeBuffer* GetEmptyCodeBuffer(); + CodeBuffer* GetEmptySharedCodeBuffer(); // This is the code buffer containing the main code under execution by this thread. // CheckCodeBufferUpdate must be used before compiling new code. @@ -203,7 +148,7 @@ namespace CPU { // Old CodeBuffer generations required to be valid until returning from signal handlers fextl::vector> SignalHandlerCodeBuffers; - CodeBufferManager& CodeBuffers; + SharedCodeBufferManager& SharedCodeBuffers; private: void RegisterForSignalHandler(fextl::shared_ptr); diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 743b4ea3d..1e21d5b4a 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -666,7 +666,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In Ptrs.LDIV = reinterpret_cast(LDIV); } - CurrentCodeBuffer = CodeBuffers.GetLatest(); + CurrentCodeBuffer = SharedCodeBuffers.GetLatest(); ThreadState->LookupCache->Shared = CurrentCodeBuffer->LookupCache.get(); } @@ -675,7 +675,7 @@ void Arm64JITCore::ClearCache() { auto PrevCodeBuffer = CurrentCodeBuffer; auto lk = PrevCodeBuffer->LookupCache->AcquireWriteLock(); - auto CodeBuffer = GetEmptyCodeBuffer(); + auto CodeBuffer = GetEmptySharedCodeBuffer(); SetBuffer(CodeBuffer->Ptr, CodeBuffer->AllocatedSize); ThreadState->LookupCache->ChangeGuestToHostMapping(*PrevCodeBuffer, *CurrentCodeBuffer->LookupCache, lk); @@ -1072,7 +1072,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size // Migrate the compile output from temporary storage to the actual CodeBuffer. // This can block progress in other compiling threads, so the duration of the lock should be as small as possible. { - auto CodeBufferLock = std::unique_lock {CodeBuffers.CodeBufferWriteMutex}; + auto CodeBufferLock = std::unique_lock {SharedCodeBuffers.CodeBufferWriteMutex}; // Query size of generated code const auto TempSize = GetCursorOffset(); @@ -1089,13 +1089,13 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size // NOTE: 16-byte alignment of the new cursor offset must be preserved for block linking records SetBuffer(CurrentCodeBuffer->Ptr, CurrentCodeBuffer->AllocatedSize); - SetCursorOffset(CodeBuffers.LatestOffset); + SetCursorOffset(SharedCodeBuffers.LatestOffset); Align16B(); if ((GetCursorOffset() + TempSize) > CurrentCodeBuffer->UsableSize()) { CTX->ClearCodeCache(ThreadState); } - CodeBuffers.LatestOffset = GetCursorOffset(); + SharedCodeBuffers.LatestOffset = GetCursorOffset(); } // Adjust host addresses @@ -1107,14 +1107,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size CodeBegin += Delta; for (std::size_t Idx = PrevNumAllocations; Idx != Relocations.size(); ++Idx) { - Relocations[Idx].Header.Offset += CodeBuffers.LatestOffset; + Relocations[Idx].Header.Offset += SharedCodeBuffers.LatestOffset; } // Copy over CodeBuffer contents memcpy(GetCursorAddress(), TempCodeBuffer, TempSize); - SetCursorOffset(CodeBuffers.LatestOffset + TempSize); + SetCursorOffset(SharedCodeBuffers.LatestOffset + TempSize); - CodeBuffers.LatestOffset = GetCursorOffset(); + SharedCodeBuffers.LatestOffset = GetCursorOffset(); } TempCodeBufferAllocator.DelayedDisownBuffer(); diff --git a/FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp new file mode 100644 index 000000000..db2ccdc58 --- /dev/null +++ b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: MIT +#include "Interface/Core/LookupCache.h" +#include "Interface/Core/SharedCodeBufferManager.h" + +#include +#include +#include + +#ifndef _WIN32 +#include +#endif + +namespace FEXCore::CPU { +static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16; +// We don't want to move above 128MB atm because that means we will have to encode longer jumps +static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128; + +CodeBuffer::CodeBuffer(size_t Size) + : AllocatedSize(Size) { + Ptr = static_cast(FEXCore::Allocator::VirtualAlloc(Size, true)); + LOGMAN_THROW_A_FMT(!!Ptr, "Couldn't allocate code buffer"); + + // Protect the last page of the allocated buffer to trigger SIGSEGV on write access + uintptr_t LastPageAddr = AlignDown(reinterpret_cast(Ptr) + Size - 1, FEXCore::Utils::FEX_PAGE_SIZE); + if (!FEXCore::Allocator::VirtualProtect(reinterpret_cast(LastPageAddr), FEXCore::Utils::FEX_PAGE_SIZE, + FEXCore::Allocator::ProtectOptions::None)) { + LogMan::Msg::EFmt("Failed to mprotect last page of code buffer."); + } + + FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size); + + // Huge-pages reduce the amount of iTLB misses dramatically when it works. + FEXCore::Allocator::VirtualTHPControl(Ptr, Size, FEXCore::Allocator::THPControl::Enable); + + LookupCache = fextl::make_unique(); +} + +CodeBuffer::~CodeBuffer() { + FEXCore::Allocator::VirtualFree(Ptr, AllocatedSize); +} + +auto SharedCodeBufferManager::AllocateNew(size_t Size) -> fextl::shared_ptr { +#ifndef _WIN32 +// MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature. +// It's equivalent to systemd's `MemoryDenyWriteExecute` but implemented entirely in the kernel. +// +// MDWE prevents applications from creating RWX memory mappings. +// This prevents FEX from doing anything JIT related, as FEX uses RWX for JIT memory mappings. +// +// A potential workaround to make FEX work with MDWE is to call mprotect every time we need to write or modify code. +// Alternatively, FEX could use a memory mirror where one half is mapped as RW and the other is RX. +// +// Once MDWE is enabled with the prctl, the feature is sealed and it can /NOT/ be turned off. +// +// Status of MDWE is queried through prctl using `PR_GET_MDWE`: +// -1: The kernel doesn't support MDWE +// 0: MDWE is supported but disabled +// >0: MDWE is enabled, hence prohibiting RWX mappings +#ifndef PR_GET_MDWE +#define PR_GET_MDWE 66 +#endif + int MDWE = ::prctl(PR_GET_MDWE, 0, 0, 0, 0); + if (MDWE != -1 && MDWE != 0) { + LogMan::Msg::EFmt("MDWE was set to 0x{:x} which means FEX can't allocate executable memory", MDWE); + } +#endif + + auto Buffer = fextl::make_shared(Size); + + Latest = Buffer; + LatestOffset = 0; + + OnCodeBufferAllocated(Buffer); + + return Buffer; +} + +fextl::shared_ptr SharedCodeBufferManager::GetLatest() { + if (!Latest) { + AllocateNew(INITIAL_CODE_SIZE); + } + return Latest; +} + +fextl::shared_ptr SharedCodeBufferManager::StartLargerCodeBuffer() { + if (!Latest) { + // Allocate initial CodeBuffer and return it + return GetLatest(); + } + + auto NewCodeBufferSize = GetLatest()->AllocatedSize; + NewCodeBufferSize = std::min(NewCodeBufferSize * 2, MAX_CODE_SIZE); + return AllocateNew(NewCodeBufferSize); +} + +fextl::shared_ptr SharedCodeBufferManager::StartMaximalCodeBuffer() { + return AllocateNew(MAX_CODE_SIZE); +} +} // namespace FEXCore::CPU diff --git a/FEXCore/Source/Interface/Core/SharedCodeBufferManager.h b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.h new file mode 100644 index 000000000..79ec8b82e --- /dev/null +++ b/FEXCore/Source/Interface/Core/SharedCodeBufferManager.h @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT +/* +$info$ +category: Thread shared code buffer management +tags: backend|shared +$end_info$ +*/ +#pragma once +#include +#include + +namespace FEXCore { +struct GuestToHostMap; +} + +namespace FEXCore::CPU { +struct CodeBuffer { + uint8_t* Ptr; + size_t AllocatedSize; // including guard page; see UsableSize() + + fextl::unique_ptr LookupCache; + + CodeBuffer(size_t Size); + CodeBuffer(const CodeBuffer&) = delete; + CodeBuffer& operator=(const CodeBuffer&) = delete; + CodeBuffer(CodeBuffer&& oth) = delete; + CodeBuffer& operator=(CodeBuffer&&) = delete; + + ~CodeBuffer(); + + /// Returns the number of bytes available for storing code + size_t UsableSize() const { + return AllocatedSize - FEXCore::Utils::FEX_PAGE_SIZE; + } +}; + +/** + * A manager that coordinates access to the CodeBuffer used for compiling new code across threads. + * + * The CodeBuffer is managed as a partially persistent data structure: + * - Exactly one CodeBuffer is now designated as "active", which means data can be appended to it + * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads) + * - Instead, such lossy modifications trigger a new "version" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads. + * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version + */ +class SharedCodeBufferManager { +public: + // Get the CodeBuffer that was most recently allocated. + // This is the only CodeBuffer that data may be written to. + fextl::shared_ptr GetLatest(); + + // Allocate a new CodeBuffer with geometric growth up to an internal maximum. + // Subsequent calls to GetLatest will point to the returned buffer. + fextl::shared_ptr StartLargerCodeBuffer(); + + // Allocate a new CodeBuffer with maximum internal size. + // Subsequent calls to GetLatest will point to the returned buffer. + fextl::shared_ptr StartMaximalCodeBuffer(); + + // Write offset into the latest CodeBuffer + std::size_t LatestOffset {}; + + // Protects writes to the latest CodeBuffer and changes to LatestOffset + FEXCore::ForkableUniqueMutex CodeBufferWriteMutex; + + virtual void OnCodeBufferAllocated(const std::shared_ptr&) {}; + +private: + fextl::shared_ptr Latest; + + fextl::shared_ptr AllocateNew(size_t Size); +}; +} // namespace FEXCore::CPU