Merge pull request #5217 from Sonicadvance1/52

FEX: Move SBRK handling to the frontend

FEXCore/Source/Utils/Allocator.cpp

@@ -72,54 +72,6 @@ int FEX_munmap(void* addr, size_t length) {
   return Result;
 }
 
-// This function disables glibc's ability to allocate memory through the `sbrk` interface.
-// This is run early in the lifecycle of FEX in order to make sure no 64-bit pointers can make it to the guest 32-bit application.
-//
-// How this works is that this allocates a single page at the current sbrk pointer (aligned upward to page size). This makes it
-// so that when the sbrk syscall is used to allocate more memory, it fails with an ENOMEM since it runs in to the allocated guard page.
-//
-// glibc notices the sbrk failure and falls back to regular mmap based allocations when this occurs. Ensuring that memory can still be allocated.
-void* DisableSBRKAllocations() {
-  void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
-  // Get the starting sbrk pointer.
-  void* StartingSBRK = sbrk(0);
-  if (StartingSBRK == INVALID_PTR) {
-    // If sbrk is already returning invalid pointers then nothing to do here.
-    return INVALID_PTR;
-  }
-
-  // Now allocate the next page after the sbrk address to ensure it can't grow.
-  // In most cases at the start of `main` this will already be page aligned, which means subsequent `sbrk`
-  // calls won't allocate any memory through that.
-  void* AlignedBRK = reinterpret_cast<void*>(FEXCore::AlignUp(reinterpret_cast<uintptr_t>(StartingSBRK), FEXCore::Utils::FEX_PAGE_SIZE));
-  void* AfterBRK =
-    ::mmap(AlignedBRK, FEXCore::Utils::FEX_PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, -1, 0);
-  if (AfterBRK == INVALID_PTR) {
-    // Couldn't allocate the page after the aligned brk? This should never happen.
-    // FEXCore::LogMan isn't configured yet so we just need to print the message.
-    fextl::fmt::print("Couldn't allocate page after SBRK.\n");
-    FEX_TRAP_EXECUTION;
-    return INVALID_PTR;
-  }
-
-  // Now that the page after sbrk is allocated, FEX needs to consume the remaining sbrk space.
-  // This will be anywhere from [0, 4096) bytes.
-  // Start allocating from 1024 byte increments just to make any steps a bit faster.
-  intptr_t IncrementAmount = 1024;
-  for (; IncrementAmount != 0; IncrementAmount >>= 1) {
-    while (sbrk(IncrementAmount) != INVALID_PTR)
-      ;
-  }
-  return AlignedBRK;
-}
-
-void ReenableSBRKAllocations(void* Ptr) {
-  const void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
-  if (Ptr != INVALID_PTR) {
-    munmap(Ptr, FEXCore::Utils::FEX_PAGE_SIZE);
-  }
-}
-
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 static void AssignHookOverrides() {

FEXCore/include/FEXCore/Utils/Allocator.h

@@ -63,13 +63,6 @@ public:
 };
 #endif
 
-// Disable allocations through glibc's sbrk allocation method.
-// Returns a pointer at the end of the sbrk region.
-FEX_DEFAULT_VISIBILITY void* DisableSBRKAllocations();
-
-// Allow sbrk again. Pass in the pointer returned by `DisableSBRKAllocations`
-FEX_DEFAULT_VISIBILITY void ReenableSBRKAllocations(void* Ptr);
-
 struct MemoryRegion {
   void* Ptr;
   size_t Size;

Source/Common/CMakeLists.txt

@@ -13,7 +13,8 @@ set(SRCS
 if (NOT MINGW)
   list(APPEND SRCS
     FEXServerClient.cpp
-    FileFormatCheck.cpp)
+    FileFormatCheck.cpp
+    Linux/SBRKAllocations.cpp)
 endif()
 
 add_library(${NAME} STATIC ${SRCS})

Source/Common/Linux/SBRKAllocations.cpp

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: MIT
+#include <FEXCore/fextl/fmt.h>
+#include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/MathUtils.h>
+#include <FEXCore/Utils/TypeDefines.h>
+
+#include <sys/mman.h>
+
+namespace FEX::SBRKAllocations {
+// This function disables glibc's ability to allocate memory through the `sbrk` interface.
+// This is run early in the lifecycle of FEX in order to make sure no 64-bit pointers can make it to the guest 32-bit application.
+//
+// How this works is that this allocates a single page at the current sbrk pointer (aligned upward to page size). This makes it
+// so that when the sbrk syscall is used to allocate more memory, it fails with an ENOMEM since it runs in to the allocated guard page.
+//
+// glibc notices the sbrk failure and falls back to regular mmap based allocations when this occurs. Ensuring that memory can still be allocated.
+void* DisableSBRKAllocations() {
+  void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
+  // Get the starting sbrk pointer.
+  void* StartingSBRK = sbrk(0);
+  if (StartingSBRK == INVALID_PTR) {
+    // If sbrk is already returning invalid pointers then nothing to do here.
+    return INVALID_PTR;
+  }
+
+  // Now allocate the next page after the sbrk address to ensure it can't grow.
+  // In most cases at the start of `main` this will already be page aligned, which means subsequent `sbrk`
+  // calls won't allocate any memory through that.
+  void* AlignedBRK = reinterpret_cast<void*>(FEXCore::AlignUp(reinterpret_cast<uintptr_t>(StartingSBRK), FEXCore::Utils::FEX_PAGE_SIZE));
+  void* AfterBRK =
+    ::mmap(AlignedBRK, FEXCore::Utils::FEX_PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, -1, 0);
+  if (AfterBRK == INVALID_PTR) {
+    // Couldn't allocate the page after the aligned brk? This should never happen.
+    // FEXCore::LogMan isn't configured yet so we just need to print the message.
+    fextl::fmt::print("Couldn't allocate page after SBRK.\n");
+    FEX_TRAP_EXECUTION;
+    return INVALID_PTR;
+  }
+
+  // Now that the page after sbrk is allocated, FEX needs to consume the remaining sbrk space.
+  // This will be anywhere from [0, 4096) bytes.
+  // Start allocating from 1024 byte increments just to make any steps a bit faster.
+  intptr_t IncrementAmount = 1024;
+  for (; IncrementAmount != 0; IncrementAmount >>= 1) {
+    while (sbrk(IncrementAmount) != INVALID_PTR)
+      ;
+  }
+  return AlignedBRK;
+}
+
+void ReenableSBRKAllocations(void* Ptr) {
+  const void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
+  if (Ptr != INVALID_PTR) {
+    munmap(Ptr, FEXCore::Utils::FEX_PAGE_SIZE);
+  }
+}
+} // namespace FEX::SBRKAllocations

Source/Common/Linux/SBRKAllocations.h

@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+
+namespace FEX::SBRKAllocations {
+// Disable allocations through glibc's sbrk allocation method.
+// Returns a pointer at the end of the sbrk region.
+void* DisableSBRKAllocations();
+
+// Allow sbrk again. Pass in the pointer returned by `DisableSBRKAllocations`
+void ReenableSBRKAllocations(void* Ptr);
+} // namespace FEX::SBRKAllocations

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -10,6 +10,7 @@ $end_info$
 #include "Common/FEXServerClient.h"
 #include "Common/Config.h"
 #include "Common/HostFeatures.h"
+#include "Common/Linux/SBRKAllocations.h"
 #include "PortabilityInfo.h"
 #include "ELFCodeLoader.h"
 #include "VDSO_Emulation.h"
@@ -360,7 +361,7 @@ static int StealFEXFDFromEnv(const char* Env) {
 }
 
 int main(int argc, char** argv, char** const envp) {
-  auto SBRKPointer = FEXCore::Allocator::DisableSBRKAllocations();
+  auto SBRKPointer = FEX::SBRKAllocations::DisableSBRKAllocations();
   FEXCore::Allocator::GLIBCScopedFault GLIBFaultScope;
 
   const bool ExecutedWithFD = getauxval(AT_EXECFD) != 0;
@@ -624,7 +625,7 @@ int main(int argc, char** argv, char** const envp) {
   FEXCore::Telemetry::Shutdown(Program.ProgramName);
   FEXCore::Profiler::Shutdown();
 
-  FEXCore::Allocator::ReenableSBRKAllocations(SBRKPointer);
+  FEX::SBRKAllocations::ReenableSBRKAllocations(SBRKPointer);
 
   return ProgramStatus;
 }

Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp

@@ -18,6 +18,7 @@ $end_info$
 #endif
 
 #include "Common/HostFeatures.h"
+#include "Common/Linux/SBRKAllocations.h"
 #include "HarnessHelpers.h"
 #include "TestHarnessRunner/HostRunner.h"
 
@@ -189,7 +190,7 @@ void RegisterLongJumpHandler(FEX::DummyHandlers::DummySignalDelegator* Handler)
 
 int main(int argc, char** argv, char** const envp) {
 #ifndef _WIN32
-  auto SBRKPointer = FEXCore::Allocator::DisableSBRKAllocations();
+  auto SBRKPointer = FEX::SBRKAllocations::DisableSBRKAllocations();
 #endif
   FEXCore::Allocator::GLIBCScopedFault GLIBFaultScope;
   LogMan::Throw::InstallHandler(AssertHandler);
@@ -416,7 +417,7 @@ int main(int argc, char** argv, char** const envp) {
 #ifndef _WIN32
   FEXCore::Allocator::ClearHooks();
 
-  FEXCore::Allocator::ReenableSBRKAllocations(SBRKPointer);
+  FEX::SBRKAllocations::ReenableSBRKAllocations(SBRKPointer);
 #endif
 
   return Passed ? 0 : -1;