Switch over to rpmalloc instead of jemalloc.

rpmalloc is currently very aggressively configured which causes
significant reductions in resident memory over jemalloc.

In Bayonetta's title screen it went from 963MB down to 834MB resident.

CMakeLists.txt

@@ -20,7 +20,7 @@ option(ENABLE_ASSERTIONS "Enable debug assertions" FALSE)
 option(ENABLE_GDB_SYMBOLS "Enable GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H})
 option(ENABLE_STRICT_WERROR "Enable stricter -Werror" FALSE)
 option(ENABLE_WERROR "Enable -Werror" FALSE)
-option(ENABLE_JEMALLOC "Enable jemalloc allocator" TRUE)
+option(ENABLE_FEX_ALLOCATOR "Enable allocator for FEX" TRUE)
 option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enable jemalloc glibc allocator" TRUE)
 option(ENABLE_OFFLINE_TELEMETRY "Enable FEX offline telemetry" TRUE)
 option(ENABLE_COMPILE_TIME_TRACE "Enable time trace compile option" FALSE)
@@ -75,7 +75,7 @@ elseif (MSVC)
   message(FATAL_ERROR "FEX doesn't support MSVC! Use Clang on MinGW instead.")
 elseif (MINGW)
   message(STATUS "Building for MinGW")
-  set(ENABLE_JEMALLOC TRUE)
+  set(ENABLE_FEX_ALLOCATOR TRUE)
   set(ENABLE_JEMALLOC_GLIBC_ALLOC FALSE)
 else ()
   message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
@@ -319,12 +319,12 @@ elseif (NOT MINGW)
     " Use at your own risk!")
 endif()
 
-if (ENABLE_JEMALLOC)
-  # The jemalloc subproject that all FEXCore fextl objects allocate through.
-  add_subdirectory(External/jemalloc/)
-elseif (NOT MINGW)
-  message(STATUS
-    " jemalloc disabled!\n"
+if (ENABLE_FEX_ALLOCATOR)
+  # The rpmalloc subproject that all FEXCore fextl objects allocate through.
+  add_subdirectory(External/rpmalloc/)
+elseif (NOT MINGW_BUILD)
+  message (STATUS
+    " FEX allocator is disabled!\n"
     " This is not a recommended configuration!\n"
     " This will very explicitly break 32-bit application execution!\n"
     " Use at your own risk!")

FEXCore/Source/CMakeLists.txt

@@ -291,9 +291,9 @@ endif()
 # Meta-library to link jemalloc libraries enabled in the build configuration.
 # Only needed for targets that run emulation. For others, use JemallocDummy.
 add_library(JemallocLibs STATIC Utils/AllocatorHooks.cpp)
-if (ENABLE_JEMALLOC)
-  target_compile_definitions(JemallocLibs PRIVATE ENABLE_JEMALLOC=1 JEMALLOC_NO_RENAME=1)
-  target_link_libraries(JemallocLibs PUBLIC FEX_jemalloc)
+if (ENABLE_FEX_ALLOCATOR)
+  target_compile_definitions(JemallocLibs PRIVATE ENABLE_FEX_ALLOCATOR=1)
+  target_link_libraries(JemallocLibs PUBLIC rpmalloc)
 endif()
 if (ENABLE_JEMALLOC_GLIBC_ALLOC)
   set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)

FEXCore/Source/Utils/Allocator.cpp

@@ -81,21 +81,21 @@ int FEX_munmap(void* addr, size_t length) {
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-static void AssignHookOverrides() {
-  SetJemallocMmapHook(FEX_mmap);
-  SetJemallocMunmapHook(FEX_munmap);
+
+static void AssignHookOverrides(size_t PageSize) {
+  SetupAllocatorHooks(FEX_mmap, FEX_munmap);
   FEXCore::Allocator::mmap = FEX_mmap;
   FEXCore::Allocator::munmap = FEX_munmap;
+  InitializeAllocator(PageSize);
 }
 
-void SetupHooks() {
+void SetupHooks(size_t PageSize) {
   Alloc64 = Alloc::OSAllocator::Create64BitAllocator();
-  AssignHookOverrides();
+  AssignHookOverrides(PageSize);
 }
 
 void ClearHooks() {
-  SetJemallocMmapHook(::mmap);
-  SetJemallocMunmapHook(::munmap);
+  SetupAllocatorHooks(::mmap, ::munmap);
   FEXCore::Allocator::mmap = ::mmap;
   FEXCore::Allocator::munmap = ::munmap;
 
@@ -271,7 +271,7 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   return Regions;
 }
 
-fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists() {
+fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(size_t PageSize) {
   size_t Bits = FEXCore::Allocator::DetermineVASize();
   if (Bits < 48) {
     return {};
@@ -282,7 +282,7 @@ fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists() {
   auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA);
 
   Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions);
-  AssignHookOverrides();
+  AssignHookOverrides(PageSize);
 
   return Regions;
 }

FEXCore/Source/Utils/AllocatorHooks.cpp

@@ -1,67 +1,206 @@
 // SPDX-License-Identifier: MIT
-#ifdef ENABLE_JEMALLOC
-#include <jemalloc/jemalloc.h>
+#ifdef ENABLE_FEX_ALLOCATOR
+#include <rpmalloc/rpmalloc.h>
+#ifndef _WIN32
+#include <linux/prctl.h>
+#include <sys/prctl.h>
+#include <sys/mman.h>
+#else
+#define NTDDI_VERSION 0x0A000005
+#include <memoryapi.h>
+#endif
 #endif
 
+#include <cstdint>
 #include <malloc.h>
 #include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
 #include <unistd.h>
 
 namespace FEXCore::Allocator {
-
-#ifndef _WIN32
 using mmap_hook_type = void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset);
 using munmap_hook_type = int (*)(void* addr, size_t length);
-#endif
 
-#ifdef ENABLE_JEMALLOC
-void* malloc(size_t size) {
-  return ::je_malloc(size);
-}
-void* calloc(size_t n, size_t size) {
-  return ::je_calloc(n, size);
-}
-void* memalign(size_t align, size_t s) {
-  return ::je_memalign(align, s);
-}
-void* valloc(size_t size) {
-  return ::je_valloc(size);
-}
-int posix_memalign(void** r, size_t a, size_t s) {
-  return ::je_posix_memalign(r, a, s);
-}
-void* realloc(void* ptr, size_t size) {
-  return ::je_realloc(ptr, size);
-}
-void free(void* ptr) {
-  return ::je_free(ptr);
-}
-size_t malloc_usable_size(void* ptr) {
-  return ::je_malloc_usable_size(ptr);
-}
-void* aligned_alloc(size_t a, size_t s) {
-  return ::je_aligned_alloc(a, s);
-}
-void aligned_free(void* ptr) {
-  return ::je_free(ptr);
-}
+#ifdef ENABLE_FEX_ALLOCATOR
+typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size);
+typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size);
+extern "C" rp_mmap_hook_type rp_mmap_hook;
+extern "C" rp_munmap_hook_type rp_munmap_hook;
 
 #ifndef _WIN32
-extern "C" mmap_hook_type je___mmap_hook;
-extern "C" munmap_hook_type je___munmap_hook;
-
-void SetJemallocMmapHook(mmap_hook_type Hook) {
-  je___mmap_hook = Hook;
-}
-void SetJemallocMunmapHook(munmap_hook_type Hook) {
-  je___munmap_hook = Hook;
-}
+mmap_hook_type fex_mmap_hook = ::mmap;
+munmap_hook_type fex_munmap_hook = ::munmap;
 #endif
 
+// Assume a 64KB page size until told otherwise.
+static rpmalloc_config_t global_config {
+  .page_size = 64 * 1024,
+  // THP causes crashes for some reason.
+  .enable_huge_pages = 0,
+  .disable_decommit = 0,
+  .page_name = "FEXAllocator",
+  .huge_page_name = "FEXAllocator",
+  .unmap_on_finalize = 0,
+};
+
+void* malloc(size_t size) {
+  return ::rpmalloc(size);
+}
+void* calloc(size_t n, size_t size) {
+  return ::rpcalloc(n, size);
+}
+void* memalign(size_t align, size_t s) {
+  return ::rpmemalign(align, s);
+}
+void* valloc(size_t size) {
+  return ::rpaligned_alloc(global_config.page_size, size);
+}
+int posix_memalign(void** r, size_t a, size_t s) {
+  void* ptr;
+  auto res = ::rpposix_memalign(&ptr, a, s);
+  *r = ptr;
+  return res;
+}
+void* realloc(void* ptr, size_t size) {
+  return ::rprealloc(ptr, size);
+}
+void free(void* ptr) {
+  return ::rpfree(ptr);
+}
+size_t malloc_usable_size(void* ptr) {
+  return ::rpmalloc_usable_size(ptr);
+}
+void* aligned_alloc(size_t a, size_t s) {
+  return ::rpaligned_alloc(a, s);
+}
+void aligned_free(void* ptr) {
+  return ::rpfree(ptr);
+}
+
+void InitializeThread() {
+  rpmalloc_thread_initialize();
+}
+
+[[nodiscard]]
+constexpr uint64_t AlignUp(uint64_t value, uint64_t size) {
+  return value + (size - value % size) % size;
+}
+
+static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t* mapped_size) {
+#define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs))
+  // If the alignment is less than the operating page size then alignment is guaranteed. Just remove it.
+  if (alignment < global_config.page_size) {
+    alignment = 0;
+  }
+
+  size_t map_size = AlignUp(size + alignment, global_config.page_size);
+#ifdef _WIN32
+  auto ptr = VirtualAlloc(0, map_size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+#else
+  auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+
+  if (ptr == MAP_FAILED) {
+    ptr = nullptr;
+  } else {
+#ifndef PR_SET_VMA
+#define PR_SET_VMA 0x53564d41
+#endif
+
+#ifndef PR_SET_VMA_ANON_NAME
+#define PR_SET_VMA_ANON_NAME 0
+#endif
+    prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, map_size, global_config.page_name);
+  }
+#endif
+
+  if (ptr == nullptr) {
+    fprintf(stderr, "Failed to map VMA region.");
+    return nullptr;
+  }
+
+  if (alignment) {
+    size_t padding = ((uintptr_t)ptr & (uintptr_t)(alignment - 1));
+    if (padding) {
+      padding = alignment - padding;
+    }
+    ptr = pointer_offset(ptr, padding);
+    *offset = padding;
+  }
+  *mapped_size = map_size;
+  return ptr;
+}
+
+static void FEX_rp_memory_commit(void* address, size_t size) {
+  if (global_config.disable_decommit) {
+    return;
+  }
+
+#ifdef _WIN32
+  if (!VirtualAlloc(address, size, MEM_COMMIT, PAGE_READWRITE)) {
+#else
+  if (false) {
+#endif
+    fprintf(stderr, "Failed to commit VMA region.");
+  }
+}
+
+static void FEX_rp_memory_decommit(void* address, size_t size) {
+  if (global_config.disable_decommit) {
+    return;
+  }
+
+#ifdef _WIN32
+  if (!VirtualFree(address, size, MEM_DECOMMIT)) {
+#else
+  if (madvise(address, size, MADV_DONTNEED)) {
+#endif
+    fprintf(stderr, "Failed to decommit VMA region.");
+  }
+}
+
+static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) {
+  address = pointer_offset(address, -(int32_t)offset);
+#ifdef _WIN32
+  ::VirtualFree(address, mapped_size, MEM_RELEASE);
+#else
+  int Result = fex_munmap_hook(address, mapped_size);
+  if (Result == -1) {
+    fprintf(stderr, "Failed to unmap VMA region.");
+  }
+#endif
+#undef pointer_offset
+}
+
+void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) {
+#ifndef _WIN32
+  fex_mmap_hook = MMapHook;
+  fex_munmap_hook = MunmapHook;
+#endif
+}
+
+static rpmalloc_interface_t global_interface {
+  .memory_map = FEX_rp_mmap,
+  .memory_commit = FEX_rp_memory_commit,
+  .memory_decommit = FEX_rp_memory_decommit,
+  .memory_unmap = FEX_rp_memory_unmap,
+  .map_fail_callback = nullptr,
+  .error_callback = nullptr,
+};
+
+void InitializeAllocator(size_t PageSize) {
+  global_config.page_size = PageSize;
+  rpmalloc_initialize_config(&global_interface, &global_config);
+  rp_mmap_hook = FEX_rp_mmap;
+  rp_munmap_hook = FEX_rp_memory_unmap;
+}
+
 #elif defined(_WIN32)
 #error "Tried building _WIN32 without jemalloc"
 
 #else
+void InitializeThread() {}
+
 void* malloc(size_t size) {
   return ::malloc(size);
 }
@@ -93,8 +232,9 @@ void aligned_free(void* ptr) {
   return ::free(ptr);
 }
 
-void SetJemallocMmapHook(mmap_hook_type) {}
-void SetJemallocMunmapHook(munmap_hook_type) {}
+void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) {}
+
+void InitializeAllocator(size_t PageSize) {}
 
 #endif
 } // namespace FEXCore::Allocator

FEXCore/include/FEXCore/Utils/Allocator.h

@@ -12,7 +12,7 @@ struct InternalThreadState;
 }
 
 namespace FEXCore::Allocator {
-FEX_DEFAULT_VISIBILITY void SetupHooks();
+FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize);
 FEX_DEFAULT_VISIBILITY void ClearHooks();
 
 FEX_DEFAULT_VISIBILITY size_t DetermineVASize();
@@ -77,7 +77,7 @@ FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(const fextl::vector<MemoryRegion
 // AArch64 canonical addresses are only up to bits 48/52 with the remainder being other things
 // Use this to reserve the top 128TB of VA so the guest never see it
 // Returns nullptr on host VA < 48bits
-FEX_DEFAULT_VISIBILITY fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists();
+FEX_DEFAULT_VISIBILITY fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(size_t PageSize);
 
 #ifndef _WIN32
 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread);

FEXCore/include/FEXCore/Utils/AllocatorHooks.h

@@ -139,9 +139,11 @@ size_t malloc_usable_size(void* ptr);
 void* aligned_alloc(size_t a, size_t s);
 void aligned_free(void* ptr);
 
+FEX_DEFAULT_VISIBILITY extern void InitializeThread();
+
 #ifndef _WIN32
-void SetJemallocMmapHook(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset));
-void SetJemallocMunmapHook(int (*)(void* addr, size_t length));
+void InitializeAllocator(size_t PageSize);
+void SetupAllocatorHooks(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset), int (*)(void* addr, size_t length));
 #endif
 
 struct FEXAllocOperators {

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -145,9 +145,10 @@ void Init() {
 namespace FEX::Allocator {
 
 fextl::vector<FEXCore::Allocator::MemoryRegion> InitMemoryRegions(bool Is64Bit) {
+  const auto PageSize = sysconf(_SC_PAGESIZE);
   if (Is64Bit) {
     // Destroy the 48th bit if it exists
-    return FEXCore::Allocator::Setup48BitAllocatorIfExists();
+    return FEXCore::Allocator::Setup48BitAllocatorIfExists(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE);
   }
 
   // Reserve [0x1_0000_0000, 0x2_0000_0000).
@@ -161,8 +162,10 @@ fextl::unique_ptr<FEX::HLE::MemAllocator> InitAllocator(bool Is64Bit) {
     return {};
   }
 
+  const auto PageSize = sysconf(_SC_PAGESIZE);
+
   // Setup our userspace allocator
-  FEXCore::Allocator::SetupHooks();
+  FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE);
   auto Allocator = FEX::HLE::CreatePassthroughAllocator();
 
   // Now that the upper 32-bit address space is blocked for future allocations,

Source/Tools/FEXOfflineCompiler/Main.cpp

@@ -154,8 +154,9 @@ GenerateSingleCache(const FEXCore::ExecutableFileInfo& Binary, fextl::set<uintpt
   }
 
   if (!Is64Bit) {
+    const auto PageSize = sysconf(_SC_PAGESIZE);
     // Block upper address space
-    FEXCore::Allocator::SetupHooks();
+    FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE);
   }
 
   auto Thread = SetupCompileThread(*CTX, Is64Bit);

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp

@@ -68,6 +68,8 @@ static void* ThreadHandler(void* Data) {
     Thread->Thread->ThreadStats->TID.store(Thread->ThreadInfo.TID, std::memory_order_relaxed);
   }
 
+  FEXCore::Allocator::InitializeThread();
+
   FEX::HLE::_SyscallHandler->RegisterTLSState(Thread);
 
   // Now notify the thread that we are initialized
@@ -176,6 +178,7 @@ FEX::HLE::ThreadStateObject* CreateNewThread(FEXCore::Context::Context* CTX, FEX
 
 uint64_t HandleNewClone(FEX::HLE::ThreadStateObject* Thread, FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame,
                         FEX::HLE::clone3_args* CloneArgs) {
+  FEXCore::Allocator::InitializeThread();
   auto GuestArgs = &CloneArgs->args;
   uint64_t flags = GuestArgs->flags;
   auto NewThread = Thread;

Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp

@@ -236,7 +236,8 @@ int main(int argc, char** argv, char** const envp) {
 
   if (!Loader.Is64BitMode()) {
     // Setup our userspace allocator
-    FEXCore::Allocator::SetupHooks();
+    const auto PageSize = sysconf(_SC_PAGESIZE);
+    FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE);
     Allocator = FEX::HLE::CreatePassthroughAllocator();
   }
 #endif

Source/Windows/Common/CRT/Alloc.cpp

@@ -3,42 +3,28 @@
 #define _CRTIMP
 #include <cstdint>
 #include "../Priv.h"
-
-extern "C" {
-
-#define JEMALLOC_NOTHROW __attribute__((nothrow))
-JEMALLOC_NOTHROW extern void* je_malloc(size_t size);
-JEMALLOC_NOTHROW extern void* je_calloc(size_t n, size_t size);
-JEMALLOC_NOTHROW extern void* je_memalign(size_t align, size_t s);
-JEMALLOC_NOTHROW extern void* je_valloc(size_t size);
-JEMALLOC_NOTHROW extern int je_posix_memalign(void** r, size_t a, size_t s);
-JEMALLOC_NOTHROW extern void* je_realloc(void* ptr, size_t size);
-JEMALLOC_NOTHROW extern void je_free(void* ptr);
-JEMALLOC_NOTHROW extern size_t je_malloc_usable_size(void* ptr);
-JEMALLOC_NOTHROW extern void* je_aligned_alloc(size_t a, size_t s);
-#undef JEMALLOC_NOTHROW
-}
+#include <rpmalloc/rpmalloc.h>
 
 void* calloc(size_t NumOfElements, size_t SizeOfElements) {
-  return je_calloc(NumOfElements, SizeOfElements);
+  return ::rpcalloc(NumOfElements, SizeOfElements);
 }
 
 void free(void* Memory) {
-  je_free(Memory);
+  ::rpfree(Memory);
 }
 
 void* malloc(size_t Size) {
-  return je_malloc(Size);
+  return ::rpmalloc(Size);
 }
 
 void* realloc(void* Memory, size_t NewSize) {
-  return je_realloc(Memory, NewSize);
+  return ::rprealloc(Memory, NewSize);
 }
 
 DLLEXPORT_FUNC(void*, _aligned_malloc, (size_t Size, size_t Alignment)) {
-  return je_aligned_alloc(Alignment, Size);
+  return ::rpaligned_alloc(Alignment, Size);
 }
 
 DLLEXPORT_FUNC(void, _aligned_free, (void* Memory)) {
-  je_free(Memory);
+  ::rpfree(Memory);
 }