From 985bdf2b6c013c99be723520df9c35d23d589220 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Sat, 11 Oct 2025 13:58:13 -0700 Subject: [PATCH] Switch over to rpmalloc instead of jemalloc. rpmalloc is currently very aggressively configured which causes significant reductions in resident memory over jemalloc. In Bayonetta's title screen it went from 963MB down to 834MB resident. --- CMakeLists.txt | 12 +- FEXCore/Source/CMakeLists.txt | 6 +- FEXCore/Source/Utils/Allocator.cpp | 18 +- FEXCore/Source/Utils/AllocatorHooks.cpp | 235 ++++++++++++++---- FEXCore/include/FEXCore/Utils/Allocator.h | 4 +- .../include/FEXCore/Utils/AllocatorHooks.h | 6 +- .../Tools/FEXInterpreter/FEXInterpreter.cpp | 5 +- .../LinuxSyscalls/Syscalls/Thread.cpp | 3 + .../TestHarnessRunner/TestHarnessRunner.cpp | 3 +- Source/Windows/Common/CRT/Alloc.cpp | 28 +-- 10 files changed, 227 insertions(+), 93 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68e48349c..10f130665 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ option(ENABLE_ASSERTIONS "Enables assertions in build" FALSE) option(ENABLE_GDB_SYMBOLS "Enables GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H}) option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE) option(ENABLE_WERROR "Enables -Werror" FALSE) -option(ENABLE_JEMALLOC "Enables jemalloc allocator" TRUE) +option(ENABLE_FEX_ALLOCATOR "Enables allocator for FEX" TRUE) option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enables jemalloc glibc allocator" TRUE) option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE) option(ENABLE_COMPILE_TIME_TRACE "Enables time trace compile option" FALSE) @@ -52,7 +52,7 @@ string(FIND ${CMAKE_BASE_NAME} mingw CONTAINS_MINGW) if (NOT CONTAINS_MINGW EQUAL -1) message (STATUS "Mingw build") set (MINGW_BUILD TRUE) - set (ENABLE_JEMALLOC TRUE) + set (ENABLE_FEX_ALLOCATOR TRUE) set (ENABLE_JEMALLOC_GLIBC_ALLOC FALSE) endif() @@ -279,12 +279,12 @@ elseif (NOT MINGW_BUILD) " Use at your own risk!") endif() -if (ENABLE_JEMALLOC) - # The jemalloc subproject that all FEXCore fextl objects allocate through. - add_subdirectory(External/jemalloc/) +if (ENABLE_FEX_ALLOCATOR) + # The rpmalloc subproject that all FEXCore fextl objects allocate through. + add_subdirectory(External/rpmalloc/) elseif (NOT MINGW_BUILD) message (STATUS - " jemalloc disabled!\n" + " FEX allocator is disabled!\n" " This is not a recommended configuration!\n" " This will very explicitly break 32-bit application execution!\n" " Use at your own risk!") diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index d5f902496..3341d5b68 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -299,9 +299,9 @@ endif() # Meta-library to link jemalloc libraries enabled in the build configuration. # Only needed for targets that run emulation. For others, use JemallocDummy. add_library(JemallocLibs STATIC Utils/AllocatorHooks.cpp) -if (ENABLE_JEMALLOC) - target_compile_definitions(JemallocLibs PRIVATE ENABLE_JEMALLOC=1 JEMALLOC_NO_RENAME=1) - target_link_libraries(JemallocLibs PUBLIC FEX_jemalloc) +if (ENABLE_FEX_ALLOCATOR) + target_compile_definitions(JemallocLibs PRIVATE ENABLE_FEX_ALLOCATOR=1) + target_link_libraries(JemallocLibs PUBLIC rpmalloc) endif() if (ENABLE_JEMALLOC_GLIBC_ALLOC) set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1) diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index 134983f11..11670e1d0 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -122,21 +122,21 @@ void ReenableSBRKAllocations(void* Ptr) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -static void AssignHookOverrides() { - SetJemallocMmapHook(FEX_mmap); - SetJemallocMunmapHook(FEX_munmap); + +static void AssignHookOverrides(size_t PageSize) { + SetupAllocatorHooks(FEX_mmap, FEX_munmap); FEXCore::Allocator::mmap = FEX_mmap; FEXCore::Allocator::munmap = FEX_munmap; + InitializeAllocator(PageSize); } -void SetupHooks() { +void SetupHooks(size_t PageSize) { Alloc64 = Alloc::OSAllocator::Create64BitAllocator(); - AssignHookOverrides(); + AssignHookOverrides(PageSize); } void ClearHooks() { - SetJemallocMmapHook(::mmap); - SetJemallocMunmapHook(::munmap); + SetupAllocatorHooks(::mmap, ::munmap); FEXCore::Allocator::mmap = ::mmap; FEXCore::Allocator::munmap = ::munmap; @@ -314,7 +314,7 @@ fextl::vector StealMemoryRegion(uintptr_t Begin, uintptr_t End) { return Regions; } -fextl::vector Setup48BitAllocatorIfExists() { +fextl::vector Setup48BitAllocatorIfExists(size_t PageSize) { size_t Bits = FEXCore::Allocator::DetermineVASize(); if (Bits < 48) { return {}; @@ -325,7 +325,7 @@ fextl::vector Setup48BitAllocatorIfExists() { auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA); Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions); - AssignHookOverrides(); + AssignHookOverrides(PageSize); return Regions; } diff --git a/FEXCore/Source/Utils/AllocatorHooks.cpp b/FEXCore/Source/Utils/AllocatorHooks.cpp index 60bfabf87..b800c8a9b 100644 --- a/FEXCore/Source/Utils/AllocatorHooks.cpp +++ b/FEXCore/Source/Utils/AllocatorHooks.cpp @@ -1,66 +1,206 @@ // SPDX-License-Identifier: MIT -#ifdef ENABLE_JEMALLOC -#include +#ifdef ENABLE_FEX_ALLOCATOR +#include +#ifndef _WIN32 +#include +#include +#include +#else +#define NTDDI_VERSION 0x0A000005 +#include +#endif #endif +#include #include #include +#include +#include +#include namespace FEXCore::Allocator { - -#ifndef _WIN32 using mmap_hook_type = void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset); using munmap_hook_type = int (*)(void* addr, size_t length); -#endif -#ifdef ENABLE_JEMALLOC -void* malloc(size_t size) { - return ::je_malloc(size); -} -void* calloc(size_t n, size_t size) { - return ::je_calloc(n, size); -} -void* memalign(size_t align, size_t s) { - return ::je_memalign(align, s); -} -void* valloc(size_t size) { - return ::je_valloc(size); -} -int posix_memalign(void** r, size_t a, size_t s) { - return ::je_posix_memalign(r, a, s); -} -void* realloc(void* ptr, size_t size) { - return ::je_realloc(ptr, size); -} -void free(void* ptr) { - return ::je_free(ptr); -} -size_t malloc_usable_size(void* ptr) { - return ::je_malloc_usable_size(ptr); -} -void* aligned_alloc(size_t a, size_t s) { - return ::je_aligned_alloc(a, s); -} -void aligned_free(void* ptr) { - return ::je_free(ptr); -} +#ifdef ENABLE_FEX_ALLOCATOR +typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size); +typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size); +extern "C" rp_mmap_hook_type rp_mmap_hook; +extern "C" rp_munmap_hook_type rp_munmap_hook; #ifndef _WIN32 -extern "C" mmap_hook_type je___mmap_hook; -extern "C" munmap_hook_type je___munmap_hook; - -void SetJemallocMmapHook(mmap_hook_type Hook) { - je___mmap_hook = Hook; -} -void SetJemallocMunmapHook(munmap_hook_type Hook) { - je___munmap_hook = Hook; -} +mmap_hook_type fex_mmap_hook = ::mmap; +munmap_hook_type fex_munmap_hook = ::munmap; #endif +// Assume a 64KB page size until told otherwise. +static rpmalloc_config_t global_config { + .page_size = 64 * 1024, + // THP causes crashes for some reason. + .enable_huge_pages = 0, + .disable_decommit = 0, + .page_name = "FEXAllocator", + .huge_page_name = "FEXAllocator", + .unmap_on_finalize = 0, +}; + +void* malloc(size_t size) { + return ::rpmalloc(size); +} +void* calloc(size_t n, size_t size) { + return ::rpcalloc(n, size); +} +void* memalign(size_t align, size_t s) { + return ::rpmemalign(align, s); +} +void* valloc(size_t size) { + return ::rpaligned_alloc(global_config.page_size, size); +} +int posix_memalign(void** r, size_t a, size_t s) { + void* ptr; + auto res = ::rpposix_memalign(&ptr, a, s); + *r = ptr; + return res; +} +void* realloc(void* ptr, size_t size) { + return ::rprealloc(ptr, size); +} +void free(void* ptr) { + return ::rpfree(ptr); +} +size_t malloc_usable_size(void* ptr) { + return ::rpmalloc_usable_size(ptr); +} +void* aligned_alloc(size_t a, size_t s) { + return ::rpaligned_alloc(a, s); +} +void aligned_free(void* ptr) { + return ::rpfree(ptr); +} + +void InitializeThread() { + rpmalloc_thread_initialize(); +} + +[[nodiscard]] +constexpr uint64_t AlignUp(uint64_t value, uint64_t size) { + return value + (size - value % size) % size; +} + +static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t* mapped_size) { +#define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) + // If the alignment is less than the operating page size then alignment is guaranteed. Just remove it. + if (alignment < global_config.page_size) { + alignment = 0; + } + + size_t map_size = AlignUp(size + alignment, global_config.page_size); +#ifdef _WIN32 + auto ptr = VirtualAlloc(0, map_size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + + if (ptr == MAP_FAILED) { + ptr = nullptr; + } else { +#ifndef PR_SET_VMA +#define PR_SET_VMA 0x53564d41 +#endif + +#ifndef PR_SET_VMA_ANON_NAME +#define PR_SET_VMA_ANON_NAME 0 +#endif + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, map_size, global_config.page_name); + } +#endif + + if (ptr == nullptr) { + fprintf(stderr, "Failed to map VMA region."); + return nullptr; + } + + if (alignment) { + size_t padding = ((uintptr_t)ptr & (uintptr_t)(alignment - 1)); + if (padding) { + padding = alignment - padding; + } + ptr = pointer_offset(ptr, padding); + *offset = padding; + } + *mapped_size = map_size; + return ptr; +} + +static void FEX_rp_memory_commit(void* address, size_t size) { + if (global_config.disable_decommit) { + return; + } + +#ifdef _WIN32 + if (!VirtualAlloc(address, size, MEM_COMMIT, PAGE_READWRITE)) { +#else + if (false) { +#endif + fprintf(stderr, "Failed to commit VMA region."); + } +} + +static void FEX_rp_memory_decommit(void* address, size_t size) { + if (global_config.disable_decommit) { + return; + } + +#ifdef _WIN32 + if (!VirtualFree(address, size, MEM_DECOMMIT)) { +#else + if (madvise(address, size, MADV_DONTNEED)) { +#endif + fprintf(stderr, "Failed to decommit VMA region."); + } +} + +static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) { + address = pointer_offset(address, -(int32_t)offset); +#ifdef _WIN32 + ::VirtualFree(address, mapped_size, MEM_RELEASE); +#else + int Result = fex_munmap_hook(address, mapped_size); + if (Result == -1) { + fprintf(stderr, "Failed to unmap VMA region."); + } +#endif +#undef pointer_offset +} + +void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) { +#ifndef _WIN32 + fex_mmap_hook = MMapHook; + fex_munmap_hook = MunmapHook; +#endif +} + +static rpmalloc_interface_t global_interface { + .memory_map = FEX_rp_mmap, + .memory_commit = FEX_rp_memory_commit, + .memory_decommit = FEX_rp_memory_decommit, + .memory_unmap = FEX_rp_memory_unmap, + .map_fail_callback = nullptr, + .error_callback = nullptr, +}; + +void InitializeAllocator(size_t PageSize) { + global_config.page_size = PageSize; + rpmalloc_initialize_config(&global_interface, &global_config); + rp_mmap_hook = FEX_rp_mmap; + rp_munmap_hook = FEX_rp_memory_unmap; +} + #elif defined(_WIN32) #error "Tried building _WIN32 without jemalloc" #else +void InitializeThread() {} + void* malloc(size_t size) { return ::malloc(size); } @@ -92,8 +232,9 @@ void aligned_free(void* ptr) { return ::free(ptr); } -void SetJemallocMmapHook(mmap_hook_type) {} -void SetJemallocMunmapHook(munmap_hook_type) {} +void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) {} + +void InitializeAllocator(size_t PageSize) {} #endif } // namespace FEXCore::Allocator diff --git a/FEXCore/include/FEXCore/Utils/Allocator.h b/FEXCore/include/FEXCore/Utils/Allocator.h index fedf45257..534d8f25e 100644 --- a/FEXCore/include/FEXCore/Utils/Allocator.h +++ b/FEXCore/include/FEXCore/Utils/Allocator.h @@ -12,7 +12,7 @@ struct InternalThreadState; } namespace FEXCore::Allocator { -FEX_DEFAULT_VISIBILITY void SetupHooks(); +FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize); FEX_DEFAULT_VISIBILITY void ClearHooks(); FEX_DEFAULT_VISIBILITY size_t DetermineVASize(); @@ -84,7 +84,7 @@ FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(const fextl::vector Setup48BitAllocatorIfExists(); +FEX_DEFAULT_VISIBILITY fextl::vector Setup48BitAllocatorIfExists(size_t PageSize); #ifndef _WIN32 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread); diff --git a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h index 2143ed64b..845262dc0 100644 --- a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h +++ b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h @@ -139,9 +139,11 @@ size_t malloc_usable_size(void* ptr); void* aligned_alloc(size_t a, size_t s); void aligned_free(void* ptr); +FEX_DEFAULT_VISIBILITY extern void InitializeThread(); + #ifndef _WIN32 -void SetJemallocMmapHook(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset)); -void SetJemallocMunmapHook(int (*)(void* addr, size_t length)); +void InitializeAllocator(size_t PageSize); +void SetupAllocatorHooks(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset), int (*)(void* addr, size_t length)); #endif struct FEXAllocOperators { diff --git a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp index cfedcb016..c4b31e4ef 100644 --- a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp +++ b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp @@ -422,9 +422,10 @@ int main(int argc, char** argv, char** const envp) { fextl::vector Base48Bit; fextl::vector Low4GB; + const auto PageSize = sysconf(_SC_PAGESIZE); if (Loader.Is64BitMode()) { // Destroy the 48th bit if it exists - Base48Bit = FEXCore::Allocator::Setup48BitAllocatorIfExists(); + Base48Bit = FEXCore::Allocator::Setup48BitAllocatorIfExists(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); } else { // Reserve [0x1_0000_0000, 0x2_0000_0000). // Safety net if 32-bit address calculation overflows in to 64-bit range. @@ -432,7 +433,7 @@ int main(int argc, char** argv, char** const envp) { Low4GB = FEXCore::Allocator::StealMemoryRegion(First64BitAddr, First64BitAddr + First64BitAddr); // Setup our userspace allocator - FEXCore::Allocator::SetupHooks(); + FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); Allocator = FEX::HLE::CreatePassthroughAllocator(); // Now that the upper 32-bit address space is blocked for future allocations, diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp index c99f7b408..6f572c37d 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp @@ -68,6 +68,8 @@ static void* ThreadHandler(void* Data) { Thread->Thread->ThreadStats->TID.store(Thread->ThreadInfo.TID, std::memory_order_relaxed); } + FEXCore::Allocator::InitializeThread(); + FEX::HLE::_SyscallHandler->RegisterTLSState(Thread); // Now notify the thread that we are initialized @@ -176,6 +178,7 @@ FEX::HLE::ThreadStateObject* CreateNewThread(FEXCore::Context::Context* CTX, FEX uint64_t HandleNewClone(FEX::HLE::ThreadStateObject* Thread, FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame, FEX::HLE::clone3_args* CloneArgs) { + FEXCore::Allocator::InitializeThread(); auto GuestArgs = &CloneArgs->args; uint64_t flags = GuestArgs->flags; auto NewThread = Thread; diff --git a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp index 33ff0be7b..cc2e64e76 100644 --- a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp +++ b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp @@ -235,7 +235,8 @@ int main(int argc, char** argv, char** const envp) { if (!Loader.Is64BitMode()) { // Setup our userspace allocator - FEXCore::Allocator::SetupHooks(); + const auto PageSize = sysconf(_SC_PAGESIZE); + FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); Allocator = FEX::HLE::CreatePassthroughAllocator(); } #endif diff --git a/Source/Windows/Common/CRT/Alloc.cpp b/Source/Windows/Common/CRT/Alloc.cpp index 3900537a8..3d5f60de8 100644 --- a/Source/Windows/Common/CRT/Alloc.cpp +++ b/Source/Windows/Common/CRT/Alloc.cpp @@ -3,42 +3,28 @@ #define _CRTIMP #include #include "../Priv.h" - -extern "C" { - -#define JEMALLOC_NOTHROW __attribute__((nothrow)) -JEMALLOC_NOTHROW extern void* je_malloc(size_t size); -JEMALLOC_NOTHROW extern void* je_calloc(size_t n, size_t size); -JEMALLOC_NOTHROW extern void* je_memalign(size_t align, size_t s); -JEMALLOC_NOTHROW extern void* je_valloc(size_t size); -JEMALLOC_NOTHROW extern int je_posix_memalign(void** r, size_t a, size_t s); -JEMALLOC_NOTHROW extern void* je_realloc(void* ptr, size_t size); -JEMALLOC_NOTHROW extern void je_free(void* ptr); -JEMALLOC_NOTHROW extern size_t je_malloc_usable_size(void* ptr); -JEMALLOC_NOTHROW extern void* je_aligned_alloc(size_t a, size_t s); -#undef JEMALLOC_NOTHROW -} +#include void* calloc(size_t NumOfElements, size_t SizeOfElements) { - return je_calloc(NumOfElements, SizeOfElements); + return ::rpcalloc(NumOfElements, SizeOfElements); } void free(void* Memory) { - je_free(Memory); + ::rpfree(Memory); } void* malloc(size_t Size) { - return je_malloc(Size); + return ::rpmalloc(Size); } void* realloc(void* Memory, size_t NewSize) { - return je_realloc(Memory, NewSize); + return ::rprealloc(Memory, NewSize); } DLLEXPORT_FUNC(void*, _aligned_malloc, (size_t Size, size_t Alignment)) { - return je_aligned_alloc(Alignment, Size); + return ::rpaligned_alloc(Alignment, Size); } DLLEXPORT_FUNC(void, _aligned_free, (void* Memory)) { - je_free(Memory); + ::rpfree(Memory); }