Merge pull request #5777 from Sonicadvance1/191

FEXCore: Split out CodeBuffer management to its own file

FEXCore/Source/CMakeLists.txt

@@ -24,6 +24,7 @@ set(SRCS
   Interface/Core/Addressing.cpp
   Interface/Core/CPUID.cpp
   Interface/Core/Frontend.cpp
+  Interface/Core/SharedCodeBufferManager.cpp
   Interface/Core/OpcodeDispatcher/AVX_128.cpp
   Interface/Core/OpcodeDispatcher/Crypto.cpp
   Interface/Core/OpcodeDispatcher/Flags.cpp

FEXCore/Source/Interface/Context/Context.h

@@ -4,6 +4,7 @@
 #include "Common/JitSymbols.h"
 #include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/CPUID.h"
+#include "Interface/Core/SharedCodeBufferManager.h"
 #include <Interface/IR/IntrusiveIRList.h>
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/Context.h>
@@ -130,7 +131,7 @@ public:
                             uint32_t RelocationOffset, bool ForStorage);
 };
 
-class ContextImpl final : public FEXCore::Context::Context, public CPU::CodeBufferManager {
+class ContextImpl final : public FEXCore::Context::Context, public CPU::SharedCodeBufferManager {
 public:
   // Context base class implementation.
   bool InitCore() override;

FEXCore/Source/Interface/Core/CPUBackend.cpp

@@ -11,17 +11,8 @@
 
 #include <cstdint>
 
-#ifndef _WIN32
-#include <sys/prctl.h>
-#endif
-
 namespace FEXCore {
 namespace CPU {
-
-  static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
-  // We don't want to move above 128MB atm because that means we will have to encode longer jumps
-  static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
-
   constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = {
     {0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX
     {0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER
@@ -275,9 +266,9 @@ namespace CPU {
     return TotalLUT;
   }()};
 
-  CPUBackend::CPUBackend(CodeBufferManager& CodeBuffers, FEXCore::Core::InternalThreadState* ThreadState)
+  CPUBackend::CPUBackend(SharedCodeBufferManager& SharedCodeBuffers, FEXCore::Core::InternalThreadState* ThreadState)
     : ThreadState(ThreadState)
-    , CodeBuffers(CodeBuffers) {
+    , SharedCodeBuffers(SharedCodeBuffers) {
 
     auto& Ptrs = ThreadState->CurrentFrame->Pointers;
 
@@ -316,11 +307,11 @@ namespace CPU {
 
   CPUBackend::~CPUBackend() = default;
 
-  auto CPUBackend::GetEmptyCodeBuffer() -> CodeBuffer* {
+  auto CPUBackend::GetEmptySharedCodeBuffer() -> CodeBuffer* {
     auto PrevCodeBuffer = CurrentCodeBuffer;
 
     // Resize the code buffer and reallocate our code size
-    CurrentCodeBuffer = CodeBuffers.StartLargerCodeBuffer();
+    CurrentCodeBuffer = SharedCodeBuffers.StartLargerCodeBuffer();
 
     RegisterForSignalHandler(std::move(PrevCodeBuffer));
     return CurrentCodeBuffer.get();
@@ -338,7 +329,7 @@ namespace CPU {
   }
 
   fextl::shared_ptr<CodeBuffer> CPUBackend::CheckCodeBufferUpdate() {
-    auto NewCodeBuffer = CodeBuffers.GetLatest();
+    auto NewCodeBuffer = SharedCodeBuffers.GetLatest();
     if (CurrentCodeBuffer != NewCodeBuffer) {
       RegisterForSignalHandler(CurrentCodeBuffer);
       return std::exchange(CurrentCodeBuffer, NewCodeBuffer);
@@ -346,89 +337,6 @@ namespace CPU {
     return nullptr;
   }
 
-  CodeBuffer::CodeBuffer(size_t Size)
-    : AllocatedSize(Size) {
-    Ptr = static_cast<uint8_t*>(FEXCore::Allocator::VirtualAlloc(Size, true));
-    LOGMAN_THROW_A_FMT(!!Ptr, "Couldn't allocate code buffer");
-
-    // Protect the last page of the allocated buffer to trigger SIGSEGV on write access
-    uintptr_t LastPageAddr = AlignDown(reinterpret_cast<uintptr_t>(Ptr) + Size - 1, FEXCore::Utils::FEX_PAGE_SIZE);
-    if (!FEXCore::Allocator::VirtualProtect(reinterpret_cast<void*>(LastPageAddr), FEXCore::Utils::FEX_PAGE_SIZE,
-                                            FEXCore::Allocator::ProtectOptions::None)) {
-      LogMan::Msg::EFmt("Failed to mprotect last page of code buffer.");
-    }
-
-    FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size);
-
-    // Huge-pages reduce the amount of iTLB misses dramatically when it works.
-    FEXCore::Allocator::VirtualTHPControl(Ptr, Size, FEXCore::Allocator::THPControl::Enable);
-
-    LookupCache = fextl::make_unique<GuestToHostMap>();
-  }
-
-  CodeBuffer::~CodeBuffer() {
-    FEXCore::Allocator::VirtualFree(Ptr, AllocatedSize);
-  }
-
-  auto CodeBufferManager::AllocateNew(size_t Size) -> fextl::shared_ptr<CodeBuffer> {
-#ifndef _WIN32
-// MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature.
-// It's equivalent to systemd's `MemoryDenyWriteExecute` but implemented entirely in the kernel.
-//
-// MDWE prevents applications from creating RWX memory mappings.
-// This prevents FEX from doing anything JIT related, as FEX uses RWX for JIT memory mappings.
-//
-// A potential workaround to make FEX work with MDWE is to call mprotect every time we need to write or modify code.
-// Alternatively, FEX could use a memory mirror where one half is mapped as RW and the other is RX.
-//
-// Once MDWE is enabled with the prctl, the feature is sealed and it can /NOT/ be turned off.
-//
-// Status of MDWE is queried through prctl using `PR_GET_MDWE`:
-// -1: The kernel doesn't support MDWE
-// 0: MDWE is supported but disabled
-// >0: MDWE is enabled, hence prohibiting RWX mappings
-#ifndef PR_GET_MDWE
-#define PR_GET_MDWE 66
-#endif
-    int MDWE = ::prctl(PR_GET_MDWE, 0, 0, 0, 0);
-    if (MDWE != -1 && MDWE != 0) {
-      LogMan::Msg::EFmt("MDWE was set to 0x{:x} which means FEX can't allocate executable memory", MDWE);
-    }
-#endif
-
-    auto Buffer = fextl::make_shared<CodeBuffer>(Size);
-
-    Latest = Buffer;
-    LatestOffset = 0;
-
-    OnCodeBufferAllocated(Buffer);
-
-    return Buffer;
-  }
-
-  fextl::shared_ptr<CodeBuffer> CodeBufferManager::GetLatest() {
-    if (!Latest) {
-      AllocateNew(INITIAL_CODE_SIZE);
-    }
-    return Latest;
-  }
-
-  fextl::shared_ptr<CodeBuffer> CodeBufferManager::StartLargerCodeBuffer() {
-    if (!Latest) {
-      // Allocate initial CodeBuffer and return it
-      return GetLatest();
-    }
-
-    auto NewCodeBufferSize = GetLatest()->AllocatedSize;
-    NewCodeBufferSize = std::min<size_t>(NewCodeBufferSize * 2, MAX_CODE_SIZE);
-    return AllocateNew(NewCodeBufferSize);
-  }
-
-  fextl::shared_ptr<CodeBuffer> CodeBufferManager::StartMaximalCodeBuffer() {
-    return AllocateNew(MAX_CODE_SIZE);
-  }
-
-
   bool CPUBackend::IsAddressInCodeBuffer(uintptr_t Address) const {
     const auto CheckCodeBuffer = [](const CodeBuffer& Buffer, uintptr_t Address) {
       const auto BufferPtr = reinterpret_cast<uintptr_t>(Buffer.Ptr);

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -8,6 +8,8 @@ $end_info$
 
 #pragma once
 
+#include "Interface/Core/SharedCodeBufferManager.h"
+
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/SignalScopeGuards.h>
 #include <FEXCore/fextl/memory.h>
@@ -41,67 +43,10 @@ namespace CodeSerialize {
 struct GuestToHostMap;
 
 namespace CPU {
-  struct CodeBuffer {
-    uint8_t* Ptr;
-    size_t AllocatedSize; // including guard page; see UsableSize()
-
-    fextl::unique_ptr<GuestToHostMap> LookupCache;
-
-    CodeBuffer(size_t Size);
-    CodeBuffer(const CodeBuffer&) = delete;
-    CodeBuffer& operator=(const CodeBuffer&) = delete;
-    CodeBuffer(CodeBuffer&& oth) = delete;
-    CodeBuffer& operator=(CodeBuffer&&) = delete;
-
-    ~CodeBuffer();
-
-    /// Returns the number of bytes available for storing code
-    size_t UsableSize() const {
-      return AllocatedSize - FEXCore::Utils::FEX_PAGE_SIZE;
-    }
-  };
-
-  /**
-   * A manager that coordinates access to the CodeBuffer used for compiling new code across threads.
-   *
-   * The CodeBuffer is managed as a partially persistent data structure:
-   * - Exactly one CodeBuffer is now designated as "active", which means data can be appended to it
-   * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads)
-   * - Instead, such lossy modifications trigger a new "version" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads.
-   * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version
-   */
-  class CodeBufferManager {
-  public:
-    // Get the CodeBuffer that was most recently allocated.
-    // This is the only CodeBuffer that data may be written to.
-    fextl::shared_ptr<CodeBuffer> GetLatest();
-
-    // Allocate a new CodeBuffer with geometric growth up to an internal maximum.
-    // Subsequent calls to GetLatest will point to the returned buffer.
-    fextl::shared_ptr<CodeBuffer> StartLargerCodeBuffer();
-
-    // Allocate a new CodeBuffer with maximum internal size.
-    // Subsequent calls to GetLatest will point to the returned buffer.
-    fextl::shared_ptr<CodeBuffer> StartMaximalCodeBuffer();
-
-    // Write offset into the latest CodeBuffer
-    std::size_t LatestOffset {};
-
-    // Protects writes to the latest CodeBuffer and changes to LatestOffset
-    FEXCore::ForkableUniqueMutex CodeBufferWriteMutex;
-
-    virtual void OnCodeBufferAllocated(const std::shared_ptr<CodeBuffer>&) {};
-
-  private:
-    fextl::shared_ptr<CodeBuffer> Latest;
-
-    fextl::shared_ptr<CodeBuffer> AllocateNew(size_t Size);
-  };
-
   class CPUBackend {
   public:
 
-    CPUBackend(CodeBufferManager&, FEXCore::Core::InternalThreadState*);
+    CPUBackend(SharedCodeBufferManager&, FEXCore::Core::InternalThreadState*);
 
     virtual ~CPUBackend();
 
@@ -194,7 +139,7 @@ namespace CPU {
     FEXCore::Core::InternalThreadState* ThreadState;
 
     [[nodiscard]]
-    CodeBuffer* GetEmptyCodeBuffer();
+    CodeBuffer* GetEmptySharedCodeBuffer();
 
     // This is the code buffer containing the main code under execution by this thread.
     // CheckCodeBufferUpdate must be used before compiling new code.
@@ -203,7 +148,7 @@ namespace CPU {
     // Old CodeBuffer generations required to be valid until returning from signal handlers
     fextl::vector<fextl::shared_ptr<CodeBuffer>> SignalHandlerCodeBuffers;
 
-    CodeBufferManager& CodeBuffers;
+    SharedCodeBufferManager& SharedCodeBuffers;
 
   private:
     void RegisterForSignalHandler(fextl::shared_ptr<CodeBuffer>);

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -666,7 +666,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
     Ptrs.LDIV = reinterpret_cast<uint64_t>(LDIV);
   }
 
-  CurrentCodeBuffer = CodeBuffers.GetLatest();
+  CurrentCodeBuffer = SharedCodeBuffers.GetLatest();
   ThreadState->LookupCache->Shared = CurrentCodeBuffer->LookupCache.get();
 }
 
@@ -675,7 +675,7 @@ void Arm64JITCore::ClearCache() {
   auto PrevCodeBuffer = CurrentCodeBuffer;
   auto lk = PrevCodeBuffer->LookupCache->AcquireWriteLock();
 
-  auto CodeBuffer = GetEmptyCodeBuffer();
+  auto CodeBuffer = GetEmptySharedCodeBuffer();
   SetBuffer(CodeBuffer->Ptr, CodeBuffer->AllocatedSize);
 
   ThreadState->LookupCache->ChangeGuestToHostMapping(*PrevCodeBuffer, *CurrentCodeBuffer->LookupCache, lk);
@@ -1072,7 +1072,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   // Migrate the compile output from temporary storage to the actual CodeBuffer.
   // This can block progress in other compiling threads, so the duration of the lock should be as small as possible.
   {
-    auto CodeBufferLock = std::unique_lock {CodeBuffers.CodeBufferWriteMutex};
+    auto CodeBufferLock = std::unique_lock {SharedCodeBuffers.CodeBufferWriteMutex};
 
     // Query size of generated code
     const auto TempSize = GetCursorOffset();
@@ -1089,13 +1089,13 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
       // NOTE: 16-byte alignment of the new cursor offset must be preserved for block linking records
       SetBuffer(CurrentCodeBuffer->Ptr, CurrentCodeBuffer->AllocatedSize);
-      SetCursorOffset(CodeBuffers.LatestOffset);
+      SetCursorOffset(SharedCodeBuffers.LatestOffset);
       Align16B();
       if ((GetCursorOffset() + TempSize) > CurrentCodeBuffer->UsableSize()) {
         CTX->ClearCodeCache(ThreadState);
       }
 
-      CodeBuffers.LatestOffset = GetCursorOffset();
+      SharedCodeBuffers.LatestOffset = GetCursorOffset();
     }
 
     // Adjust host addresses
@@ -1107,14 +1107,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
     CodeBegin += Delta;
 
     for (std::size_t Idx = PrevNumAllocations; Idx != Relocations.size(); ++Idx) {
-      Relocations[Idx].Header.Offset += CodeBuffers.LatestOffset;
+      Relocations[Idx].Header.Offset += SharedCodeBuffers.LatestOffset;
     }
 
     // Copy over CodeBuffer contents
     memcpy(GetCursorAddress<uint8_t*>(), TempCodeBuffer, TempSize);
-    SetCursorOffset(CodeBuffers.LatestOffset + TempSize);
+    SetCursorOffset(SharedCodeBuffers.LatestOffset + TempSize);
 
-    CodeBuffers.LatestOffset = GetCursorOffset();
+    SharedCodeBuffers.LatestOffset = GetCursorOffset();
   }
 
   TempCodeBufferAllocator.DelayedDisownBuffer();

FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp

@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: MIT
+#include "Interface/Core/LookupCache.h"
+#include "Interface/Core/SharedCodeBufferManager.h"
+
+#include <FEXCore/fextl/memory.h>
+#include <FEXCore/Utils/AllocatorHooks.h>
+#include <FEXCore/Utils/MathUtils.h>
+
+#ifndef _WIN32
+#include <sys/prctl.h>
+#endif
+
+namespace FEXCore::CPU {
+static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
+// We don't want to move above 128MB atm because that means we will have to encode longer jumps
+static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
+
+CodeBuffer::CodeBuffer(size_t Size)
+  : AllocatedSize(Size) {
+  Ptr = static_cast<uint8_t*>(FEXCore::Allocator::VirtualAlloc(Size, true));
+  LOGMAN_THROW_A_FMT(!!Ptr, "Couldn't allocate code buffer");
+
+  // Protect the last page of the allocated buffer to trigger SIGSEGV on write access
+  uintptr_t LastPageAddr = AlignDown(reinterpret_cast<uintptr_t>(Ptr) + Size - 1, FEXCore::Utils::FEX_PAGE_SIZE);
+  if (!FEXCore::Allocator::VirtualProtect(reinterpret_cast<void*>(LastPageAddr), FEXCore::Utils::FEX_PAGE_SIZE,
+                                          FEXCore::Allocator::ProtectOptions::None)) {
+    LogMan::Msg::EFmt("Failed to mprotect last page of code buffer.");
+  }
+
+  FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size);
+
+  // Huge-pages reduce the amount of iTLB misses dramatically when it works.
+  FEXCore::Allocator::VirtualTHPControl(Ptr, Size, FEXCore::Allocator::THPControl::Enable);
+
+  LookupCache = fextl::make_unique<GuestToHostMap>();
+}
+
+CodeBuffer::~CodeBuffer() {
+  FEXCore::Allocator::VirtualFree(Ptr, AllocatedSize);
+}
+
+auto SharedCodeBufferManager::AllocateNew(size_t Size) -> fextl::shared_ptr<CodeBuffer> {
+#ifndef _WIN32
+// MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature.
+// It's equivalent to systemd's `MemoryDenyWriteExecute` but implemented entirely in the kernel.
+//
+// MDWE prevents applications from creating RWX memory mappings.
+// This prevents FEX from doing anything JIT related, as FEX uses RWX for JIT memory mappings.
+//
+// A potential workaround to make FEX work with MDWE is to call mprotect every time we need to write or modify code.
+// Alternatively, FEX could use a memory mirror where one half is mapped as RW and the other is RX.
+//
+// Once MDWE is enabled with the prctl, the feature is sealed and it can /NOT/ be turned off.
+//
+// Status of MDWE is queried through prctl using `PR_GET_MDWE`:
+// -1: The kernel doesn't support MDWE
+// 0: MDWE is supported but disabled
+// >0: MDWE is enabled, hence prohibiting RWX mappings
+#ifndef PR_GET_MDWE
+#define PR_GET_MDWE 66
+#endif
+  int MDWE = ::prctl(PR_GET_MDWE, 0, 0, 0, 0);
+  if (MDWE != -1 && MDWE != 0) {
+    LogMan::Msg::EFmt("MDWE was set to 0x{:x} which means FEX can't allocate executable memory", MDWE);
+  }
+#endif
+
+  auto Buffer = fextl::make_shared<CodeBuffer>(Size);
+
+  Latest = Buffer;
+  LatestOffset = 0;
+
+  OnCodeBufferAllocated(Buffer);
+
+  return Buffer;
+}
+
+fextl::shared_ptr<CodeBuffer> SharedCodeBufferManager::GetLatest() {
+  if (!Latest) {
+    AllocateNew(INITIAL_CODE_SIZE);
+  }
+  return Latest;
+}
+
+fextl::shared_ptr<CodeBuffer> SharedCodeBufferManager::StartLargerCodeBuffer() {
+  if (!Latest) {
+    // Allocate initial CodeBuffer and return it
+    return GetLatest();
+  }
+
+  auto NewCodeBufferSize = GetLatest()->AllocatedSize;
+  NewCodeBufferSize = std::min<size_t>(NewCodeBufferSize * 2, MAX_CODE_SIZE);
+  return AllocateNew(NewCodeBufferSize);
+}
+
+fextl::shared_ptr<CodeBuffer> SharedCodeBufferManager::StartMaximalCodeBuffer() {
+  return AllocateNew(MAX_CODE_SIZE);
+}
+} // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/SharedCodeBufferManager.h

@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: MIT
+/*
+$info$
+category: Thread shared code buffer management
+tags: backend|shared
+$end_info$
+*/
+#pragma once
+#include <FEXCore/fextl/memory.h>
+#include <FEXCore/Utils/SignalScopeGuards.h>
+
+namespace FEXCore {
+struct GuestToHostMap;
+}
+
+namespace FEXCore::CPU {
+struct CodeBuffer {
+  uint8_t* Ptr;
+  size_t AllocatedSize; // including guard page; see UsableSize()
+
+  fextl::unique_ptr<GuestToHostMap> LookupCache;
+
+  CodeBuffer(size_t Size);
+  CodeBuffer(const CodeBuffer&) = delete;
+  CodeBuffer& operator=(const CodeBuffer&) = delete;
+  CodeBuffer(CodeBuffer&& oth) = delete;
+  CodeBuffer& operator=(CodeBuffer&&) = delete;
+
+  ~CodeBuffer();
+
+  /// Returns the number of bytes available for storing code
+  size_t UsableSize() const {
+    return AllocatedSize - FEXCore::Utils::FEX_PAGE_SIZE;
+  }
+};
+
+/**
+ * A manager that coordinates access to the CodeBuffer used for compiling new code across threads.
+ *
+ * The CodeBuffer is managed as a partially persistent data structure:
+ * - Exactly one CodeBuffer is now designated as "active", which means data can be appended to it
+ * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads)
+ * - Instead, such lossy modifications trigger a new "version" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads.
+ * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version
+ */
+class SharedCodeBufferManager {
+public:
+  // Get the CodeBuffer that was most recently allocated.
+  // This is the only CodeBuffer that data may be written to.
+  fextl::shared_ptr<CodeBuffer> GetLatest();
+
+  // Allocate a new CodeBuffer with geometric growth up to an internal maximum.
+  // Subsequent calls to GetLatest will point to the returned buffer.
+  fextl::shared_ptr<CodeBuffer> StartLargerCodeBuffer();
+
+  // Allocate a new CodeBuffer with maximum internal size.
+  // Subsequent calls to GetLatest will point to the returned buffer.
+  fextl::shared_ptr<CodeBuffer> StartMaximalCodeBuffer();
+
+  // Write offset into the latest CodeBuffer
+  std::size_t LatestOffset {};
+
+  // Protects writes to the latest CodeBuffer and changes to LatestOffset
+  FEXCore::ForkableUniqueMutex CodeBufferWriteMutex;
+
+  virtual void OnCodeBufferAllocated(const std::shared_ptr<CodeBuffer>&) {};
+
+private:
+  fextl::shared_ptr<CodeBuffer> Latest;
+
+  fextl::shared_ptr<CodeBuffer> AllocateNew(size_t Size);
+};
+} // namespace FEXCore::CPU