FEX: Move SBRK handling to the frontend

This is fundamentally a frontend only problem, and also Linux only.
Moves it to the frontend where it belongs.

There's likely more things in Allocator.cpp that can be moved to the
frontend but this is the first thing.

NFC

FEXCore/Source/Utils/Allocator.cpp

@@ -72,54 +72,6 @@ int FEX_munmap(void* addr, size_t length) {
   return Result;
 }
 
-// This function disables glibc's ability to allocate memory through the `sbrk` interface.
-// This is run early in the lifecycle of FEX in order to make sure no 64-bit pointers can make it to the guest 32-bit application.
-//
-// How this works is that this allocates a single page at the current sbrk pointer (aligned upward to page size). This makes it
-// so that when the sbrk syscall is used to allocate more memory, it fails with an ENOMEM since it runs in to the allocated guard page.
-//
-// glibc notices the sbrk failure and falls back to regular mmap based allocations when this occurs. Ensuring that memory can still be allocated.
-void* DisableSBRKAllocations() {
-  void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
-  // Get the starting sbrk pointer.
-  void* StartingSBRK = sbrk(0);
-  if (StartingSBRK == INVALID_PTR) {
-    // If sbrk is already returning invalid pointers then nothing to do here.
-    return INVALID_PTR;
-  }
-
-  // Now allocate the next page after the sbrk address to ensure it can't grow.
-  // In most cases at the start of `main` this will already be page aligned, which means subsequent `sbrk`
-  // calls won't allocate any memory through that.
-  void* AlignedBRK = reinterpret_cast<void*>(FEXCore::AlignUp(reinterpret_cast<uintptr_t>(StartingSBRK), FEXCore::Utils::FEX_PAGE_SIZE));
-  void* AfterBRK =
-    ::mmap(AlignedBRK, FEXCore::Utils::FEX_PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, -1, 0);
-  if (AfterBRK == INVALID_PTR) {
-    // Couldn't allocate the page after the aligned brk? This should never happen.
-    // FEXCore::LogMan isn't configured yet so we just need to print the message.
-    fextl::fmt::print("Couldn't allocate page after SBRK.\n");
-    FEX_TRAP_EXECUTION;
-    return INVALID_PTR;
-  }
-
-  // Now that the page after sbrk is allocated, FEX needs to consume the remaining sbrk space.
-  // This will be anywhere from [0, 4096) bytes.
-  // Start allocating from 1024 byte increments just to make any steps a bit faster.
-  intptr_t IncrementAmount = 1024;
-  for (; IncrementAmount != 0; IncrementAmount >>= 1) {
-    while (sbrk(IncrementAmount) != INVALID_PTR)
-      ;
-  }
-  return AlignedBRK;
-}
-
-void ReenableSBRKAllocations(void* Ptr) {
-  const void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
-  if (Ptr != INVALID_PTR) {
-    munmap(Ptr, FEXCore::Utils::FEX_PAGE_SIZE);
-  }
-}
-
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 static void AssignHookOverrides() {

FEXCore/include/FEXCore/Utils/Allocator.h

@@ -63,13 +63,6 @@ public:
 };
 #endif
 
-// Disable allocations through glibc's sbrk allocation method.
-// Returns a pointer at the end of the sbrk region.
-FEX_DEFAULT_VISIBILITY void* DisableSBRKAllocations();
-
-// Allow sbrk again. Pass in the pointer returned by `DisableSBRKAllocations`
-FEX_DEFAULT_VISIBILITY void ReenableSBRKAllocations(void* Ptr);
-
 struct MemoryRegion {
   void* Ptr;
   size_t Size;

Source/Common/CMakeLists.txt

@@ -13,7 +13,8 @@ set(SRCS
 if (NOT MINGW)
   list(APPEND SRCS
     FEXServerClient.cpp
-    FileFormatCheck.cpp)
+    FileFormatCheck.cpp
+    Linux/SBRKAllocations.cpp)
 endif()
 
 add_library(${NAME} STATIC ${SRCS})

Source/Common/Linux/SBRKAllocations.cpp

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: MIT
+#include <FEXCore/fextl/fmt.h>
+#include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/MathUtils.h>
+#include <FEXCore/Utils/TypeDefines.h>
+
+#include <sys/mman.h>
+
+namespace FEX::SBRKAllocations {
+// This function disables glibc's ability to allocate memory through the `sbrk` interface.
+// This is run early in the lifecycle of FEX in order to make sure no 64-bit pointers can make it to the guest 32-bit application.
+//
+// How this works is that this allocates a single page at the current sbrk pointer (aligned upward to page size). This makes it
+// so that when the sbrk syscall is used to allocate more memory, it fails with an ENOMEM since it runs in to the allocated guard page.
+//
+// glibc notices the sbrk failure and falls back to regular mmap based allocations when this occurs. Ensuring that memory can still be allocated.
+void* DisableSBRKAllocations() {
+  void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
+  // Get the starting sbrk pointer.
+  void* StartingSBRK = sbrk(0);
+  if (StartingSBRK == INVALID_PTR) {
+    // If sbrk is already returning invalid pointers then nothing to do here.
+    return INVALID_PTR;
+  }
+
+  // Now allocate the next page after the sbrk address to ensure it can't grow.
+  // In most cases at the start of `main` this will already be page aligned, which means subsequent `sbrk`
+  // calls won't allocate any memory through that.
+  void* AlignedBRK = reinterpret_cast<void*>(FEXCore::AlignUp(reinterpret_cast<uintptr_t>(StartingSBRK), FEXCore::Utils::FEX_PAGE_SIZE));
+  void* AfterBRK =
+    ::mmap(AlignedBRK, FEXCore::Utils::FEX_PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, -1, 0);
+  if (AfterBRK == INVALID_PTR) {
+    // Couldn't allocate the page after the aligned brk? This should never happen.
+    // FEXCore::LogMan isn't configured yet so we just need to print the message.
+    fextl::fmt::print("Couldn't allocate page after SBRK.\n");
+    FEX_TRAP_EXECUTION;
+    return INVALID_PTR;
+  }
+
+  // Now that the page after sbrk is allocated, FEX needs to consume the remaining sbrk space.
+  // This will be anywhere from [0, 4096) bytes.
+  // Start allocating from 1024 byte increments just to make any steps a bit faster.
+  intptr_t IncrementAmount = 1024;
+  for (; IncrementAmount != 0; IncrementAmount >>= 1) {
+    while (sbrk(IncrementAmount) != INVALID_PTR)
+      ;
+  }
+  return AlignedBRK;
+}
+
+void ReenableSBRKAllocations(void* Ptr) {
+  const void* INVALID_PTR = reinterpret_cast<void*>(~0ULL);
+  if (Ptr != INVALID_PTR) {
+    munmap(Ptr, FEXCore::Utils::FEX_PAGE_SIZE);
+  }
+}
+} // namespace FEX::SBRKAllocations

Source/Common/Linux/SBRKAllocations.h

@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+
+namespace FEX::SBRKAllocations {
+// Disable allocations through glibc's sbrk allocation method.
+// Returns a pointer at the end of the sbrk region.
+void* DisableSBRKAllocations();
+
+// Allow sbrk again. Pass in the pointer returned by `DisableSBRKAllocations`
+void ReenableSBRKAllocations(void* Ptr);
+} // namespace FEX::SBRKAllocations

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -10,6 +10,7 @@ $end_info$
 #include "Common/FEXServerClient.h"
 #include "Common/Config.h"
 #include "Common/HostFeatures.h"
+#include "Common/Linux/SBRKAllocations.h"
 #include "PortabilityInfo.h"
 #include "ELFCodeLoader.h"
 #include "VDSO_Emulation.h"
@@ -360,7 +361,7 @@ static int StealFEXFDFromEnv(const char* Env) {
 }
 
 int main(int argc, char** argv, char** const envp) {
-  auto SBRKPointer = FEXCore::Allocator::DisableSBRKAllocations();
+  auto SBRKPointer = FEX::SBRKAllocations::DisableSBRKAllocations();
   FEXCore::Allocator::GLIBCScopedFault GLIBFaultScope;
 
   const bool ExecutedWithFD = getauxval(AT_EXECFD) != 0;
@@ -624,7 +625,7 @@ int main(int argc, char** argv, char** const envp) {
   FEXCore::Telemetry::Shutdown(Program.ProgramName);
   FEXCore::Profiler::Shutdown();
 
-  FEXCore::Allocator::ReenableSBRKAllocations(SBRKPointer);
+  FEX::SBRKAllocations::ReenableSBRKAllocations(SBRKPointer);
 
   return ProgramStatus;
 }

Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp

@@ -18,6 +18,7 @@ $end_info$
 #endif
 
 #include "Common/HostFeatures.h"
+#include "Common/Linux/SBRKAllocations.h"
 #include "HarnessHelpers.h"
 #include "TestHarnessRunner/HostRunner.h"
 
@@ -189,7 +190,7 @@ void RegisterLongJumpHandler(FEX::DummyHandlers::DummySignalDelegator* Handler)
 
 int main(int argc, char** argv, char** const envp) {
 #ifndef _WIN32
-  auto SBRKPointer = FEXCore::Allocator::DisableSBRKAllocations();
+  auto SBRKPointer = FEX::SBRKAllocations::DisableSBRKAllocations();
 #endif
   FEXCore::Allocator::GLIBCScopedFault GLIBFaultScope;
   LogMan::Throw::InstallHandler(AssertHandler);
@@ -416,7 +417,7 @@ int main(int argc, char** argv, char** const envp) {
 #ifndef _WIN32
   FEXCore::Allocator::ClearHooks();
 
-  FEXCore::Allocator::ReenableSBRKAllocations(SBRKPointer);
+  FEX::SBRKAllocations::ReenableSBRKAllocations(SBRKPointer);
 #endif
 
   return Passed ? 0 : -1;