FEX: Allocate a VMA allocator when running on a 48-bit VA

When running on a system with a 48-bit VA, if FEX does any allocations
between us reserving the upper 128TB and the application running, then
/technically/ we are intersecting with the application's memory region
in the lower 47-bits.

This didn't typically result in any problems due to how ASLR works, but
if we did any large allocations (like #4291 wants with 128MB VMA region)
then these typically get pushed higher in the VA space.

Again not usually a problem, but if you happen to be running an
application that is using MAP_FIXED with hardcoded addresses then this
can stomp over FEX-Emu memory causing problems.

This is what happens with Wine, it reserves the upper-32MB of its 47-bit
VA space, which is /highly/ likely to stomp on FEX memory. In-fact it
likely occurs all the time, we just got lucky with whatever it was
clobbering wasn't used at the time.

On 39-bit VA systems this isn't a problem because the mmap fails
outright with a warning message from WINE.

Because we are already reserving the upper 128TB of VA space, instead
just always enable our allocator and use the regions that were reserved.
We need to be a little bit careful to ensure we don't accidentally
allocate more memory post-reservation but that just requires a small
adjustment to our unique_ptr and constructor for the 64BitAllocator.

This means /all/ FEX-Emu allocations will be in the upper 128TB VA space
when running 64-bit applications on a 48-bit VA system. Which is kind of
nice.

Fixes WINE in #4291 when the allocator stats are bumped to 128MB per
process.

FEXCore/Source/Utils/Allocator.cpp

@@ -112,14 +112,18 @@ void ReenableSBRKAllocations(void* Ptr) {
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-void SetupHooks() {
-  Alloc64 = Alloc::OSAllocator::Create64BitAllocator();
+static void AssignHookOverrides() {
   SetJemallocMmapHook(FEX_mmap);
   SetJemallocMunmapHook(FEX_munmap);
   FEXCore::Allocator::mmap = FEX_mmap;
   FEXCore::Allocator::munmap = FEX_munmap;
 }
 
+void SetupHooks() {
+  Alloc64 = Alloc::OSAllocator::Create64BitAllocator();
+  AssignHookOverrides();
+}
+
 void ClearHooks() {
   SetJemallocMmapHook(::mmap);
   SetJemallocMunmapHook(::munmap);
@@ -300,15 +304,18 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   return Regions;
 }
 
-fextl::vector<MemoryRegion> Steal48BitVA() {
+void Setup48BitAllocatorIfExists() {
   size_t Bits = FEXCore::Allocator::DetermineVASize();
   if (Bits < 48) {
-    return {};
+    return;
   }
 
   uintptr_t Begin48BitVA = 0x0'8000'0000'0000ULL;
   uintptr_t End48BitVA = 0x1'0000'0000'0000ULL;
-  return StealMemoryRegion(Begin48BitVA, End48BitVA);
+  auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA);
+
+  Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions);
+  AssignHookOverrides();
 }
 
 void ReclaimMemoryRegion(const fextl::vector<MemoryRegion>& Regions) {

FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -7,6 +7,8 @@
 #include <FEXCore/Utils/MathUtils.h>
 #include <FEXCore/Utils/SignalScopeGuards.h>
 #include <FEXCore/Utils/TypeDefines.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/MathUtils.h>
 #include <FEXCore/fextl/sstream.h>
 #include <FEXHeaderUtils/Syscalls.h>
 #include <FEXCore/fextl/memory.h>
@@ -35,6 +37,8 @@ thread_local FEXCore::Core::InternalThreadState* TLSThread {};
 class OSAllocator_64Bit final : public Alloc::HostAllocator {
 public:
   OSAllocator_64Bit();
+  OSAllocator_64Bit(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions);
+
   virtual ~OSAllocator_64Bit();
   void* AllocateSlab(size_t Size) override {
     return nullptr;
@@ -180,7 +184,7 @@ private:
   // 32-bit old kernel workarounds
   fextl::vector<FEXCore::Allocator::MemoryRegion> Steal32BitIfOldKernel();
 
-  void AllocateMemoryRegions(const fextl::vector<FEXCore::Allocator::MemoryRegion>& Ranges);
+  void AllocateMemoryRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Ranges);
   LiveVMARegion* FindLiveRegionForAddress(uintptr_t Addr, uintptr_t AddrEnd);
 };
 
@@ -515,29 +519,41 @@ fextl::vector<FEXCore::Allocator::MemoryRegion> OSAllocator_64Bit::Steal32BitIfO
   return FEXCore::Allocator::StealMemoryRegion(LOWER_BOUND_32, UPPER_BOUND_32);
 }
 
-void OSAllocator_64Bit::AllocateMemoryRegions(const fextl::vector<FEXCore::Allocator::MemoryRegion>& Ranges) {
-  for (auto [Ptr, AllocationSize] : Ranges) {
-    if (!ObjectAlloc) {
-      auto MaxSize = std::min(size_t(64) * 1024 * 1024, AllocationSize);
+void OSAllocator_64Bit::AllocateMemoryRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Ranges) {
+  // Need to allocate the ObjectAlloc up front. Find a region that is larger than our minimum size first.
+  const size_t ObjectAllocSize = 64 * 1024 * 1024;
 
-      // Allocate up to 64 MiB the first allocation for an intrusive allocator
-      mprotect(Ptr, MaxSize, PROT_READ | PROT_WRITE);
-
-      // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure
-      ::madvise(Ptr, MaxSize, MADV_HUGEPAGE);
-
-      ObjectAlloc = new (Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(Ptr, MaxSize);
-      ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc);
-      LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);
-
-      if (AllocationSize > MaxSize) {
-        AllocationSize -= MaxSize;
-        (uint8_t*&)Ptr += MaxSize;
-      } else {
-        continue;
-      }
+  for (auto& it : Ranges) {
+    if (ObjectAllocSize > it.Size) {
+      continue;
     }
 
+    // Allocate up to 64 MiB the first allocation for an intrusive allocator
+    mprotect(it.Ptr, ObjectAllocSize, PROT_READ | PROT_WRITE);
+
+    // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure
+    ::madvise(it.Ptr, ObjectAllocSize, MADV_HUGEPAGE);
+
+    ObjectAlloc = new (it.Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(it.Ptr, ObjectAllocSize);
+    ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc);
+    LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);
+
+    if (it.Size >= ObjectAllocSize) {
+      // Modify region size
+      it.Size -= ObjectAllocSize;
+      (uint8_t*&)it.Ptr += ObjectAllocSize;
+    }
+
+    break;
+  }
+
+  if (!ObjectAlloc) {
+    ERROR_AND_DIE_FMT("Couldn't allocate object allocator!");
+  }
+
+  for (auto [Ptr, AllocationSize] : Ranges) {
+    // Skip size of zero if ObjectAllocSize matched the size of region.
+    if (AllocationSize == 0) continue;
     ReservedVMARegion* Region = ObjectAlloc->new_construct<ReservedVMARegion>();
     Region->Base = reinterpret_cast<uint64_t>(Ptr);
     Region->RegionSize = AllocationSize;
@@ -557,6 +573,10 @@ OSAllocator_64Bit::OSAllocator_64Bit() {
   FEXCore::Allocator::ReclaimMemoryRegion(LowMem);
 }
 
+OSAllocator_64Bit::OSAllocator_64Bit(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions) {
+  AllocateMemoryRegions(Regions);
+}
+
 OSAllocator_64Bit::~OSAllocator_64Bit() {
   // This needs a mutex to be thread safe
   auto lk = FEXCore::GuardSignalDeferringSectionWithFallback(AllocationMutex, TLSThread);
@@ -576,6 +596,61 @@ OSAllocator_64Bit::~OSAllocator_64Bit() {
 fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocator() {
   return fextl::make_unique<OSAllocator_64Bit>();
 }
+
+template<class T>
+struct alloc_delete : public std::default_delete<T> {
+  void operator()(T* ptr) const {
+    if (ptr) {
+      const auto size = sizeof(T);
+      const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE);
+
+      std::destroy_at(ptr);
+      ::munmap(ptr, MinPage);
+    }
+  }
+
+  template<typename U>
+  requires (std::is_base_of_v<U, T>)
+  operator fextl::default_delete<U>() {
+    return fextl::default_delete<U>();
+  }
+};
+
+template<class T, class... Args>
+requires (!std::is_array_v<T>)
+fextl::unique_ptr<T> make_alloc_unique(FEXCore::Allocator::MemoryRegion& Base, Args&&... args) {
+  const auto size = sizeof(T);
+  const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE);
+  if (Base.Size < size || MinPage != FEXCore::Utils::FEX_PAGE_SIZE) {
+    ERROR_AND_DIE_FMT("Couldn't fit allocator in to page!");
+  }
+
+  // Remove the page from the base region.
+  // Could be zero after this.
+  Base.Size -= MinPage;
+  Base.Ptr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(Base.Ptr) + MinPage);
+
+  auto ptr = ::mmap(Base.Ptr, MinPage, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+  if (ptr == MAP_FAILED) {
+    ERROR_AND_DIE_FMT("Couldn't allocate memory region");
+  }
+  auto Result = ::new (ptr) T(std::forward<Args>(args)...);
+  return fextl::unique_ptr<T, alloc_delete<T>>(Result);
+}
+
+fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocatorWithRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions) {
+  // This is a bit tricky as we can't allocate memory safely except from the Regions provided. Otherwise we might overwrite memory pages we
+  // don't own. Scan the memory regions and find the smallest one.
+  FEXCore::Allocator::MemoryRegion& Smallest = Regions[0];
+  for (auto& it : Regions) {
+    if (it.Size <= Smallest.Size) {
+      Smallest = it;
+    }
+  }
+
+  return make_alloc_unique<OSAllocator_64Bit>(Smallest, Regions);
+}
+
 } // namespace Alloc::OSAllocator
 
 namespace FEXCore::Allocator {

FEXCore/Source/Utils/Allocator/HostAllocator.h

@@ -2,9 +2,10 @@
 #pragma once
 #include <FEXCore/fextl/allocator.h>
 #include <FEXCore/fextl/memory.h>
+#include <FEXCore/fextl/vector.h>
+#include <FEXCore/Utils/Allocator.h>
 
 #include <cstddef>
-#include <cstdint>
 #include <sys/types.h>
 
 namespace FEXCore::Core {
@@ -49,4 +50,5 @@ public:
 
 namespace Alloc::OSAllocator {
 fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocator();
+fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocatorWithRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions);
 } // namespace Alloc::OSAllocator

FEXCore/include/FEXCore/Utils/Allocator.h

@@ -86,7 +86,7 @@ FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(const fextl::vector<MemoryRegion
 // AArch64 canonical addresses are only up to bits 48/52 with the remainder being other things
 // Use this to reserve the top 128TB of VA so the guest never see it
 // Returns nullptr on host VA < 48bits
-FEX_DEFAULT_VISIBILITY fextl::vector<MemoryRegion> Steal48BitVA();
+FEX_DEFAULT_VISIBILITY void Setup48BitAllocatorIfExists();
 
 #ifndef _WIN32
 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread);

Source/Tools/FEXLoader/FEXLoader.cpp

@@ -491,12 +491,11 @@ int main(int argc, char** argv, char** const envp) {
   FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_IS64BIT_MODE, Loader.Is64BitMode() ? "1" : "0");
 
   fextl::unique_ptr<FEX::HLE::MemAllocator> Allocator;
-  fextl::vector<FEXCore::Allocator::MemoryRegion> Base48Bit;
   fextl::vector<FEXCore::Allocator::MemoryRegion> Low4GB;
 
   if (Loader.Is64BitMode()) {
     // Destroy the 48th bit if it exists
-    Base48Bit = FEXCore::Allocator::Steal48BitVA();
+    FEXCore::Allocator::Setup48BitAllocatorIfExists();
   } else {
     // Reserve [0x1_0000_0000, 0x2_0000_0000).
     // Safety net if 32-bit address calculation overflows in to 64-bit range.
@@ -685,7 +684,6 @@ int main(int argc, char** argv, char** const envp) {
   LogMan::Msg::UnInstallHandler();
 
   FEXCore::Allocator::ClearHooks();
-  FEXCore::Allocator::ReclaimMemoryRegion(Base48Bit);
   FEXCore::Allocator::ReclaimMemoryRegion(Low4GB);
 
   // Allocator is now original system allocator