diff --git a/CMakeLists.txt b/CMakeLists.txt index a0e7484cb..c7453c5a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ option(ENABLE_ASSERTIONS "Enable debug assertions" FALSE) option(ENABLE_GDB_SYMBOLS "Enable GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H}) option(ENABLE_STRICT_WERROR "Enable stricter -Werror" FALSE) option(ENABLE_WERROR "Enable -Werror" FALSE) -option(ENABLE_JEMALLOC "Enable jemalloc allocator" TRUE) +option(ENABLE_FEX_ALLOCATOR "Enable allocator for FEX" TRUE) option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enable jemalloc glibc allocator" TRUE) option(ENABLE_OFFLINE_TELEMETRY "Enable FEX offline telemetry" TRUE) option(ENABLE_COMPILE_TIME_TRACE "Enable time trace compile option" FALSE) @@ -75,7 +75,7 @@ elseif (MSVC) message(FATAL_ERROR "FEX doesn't support MSVC! Use Clang on MinGW instead.") elseif (MINGW) message(STATUS "Building for MinGW") - set(ENABLE_JEMALLOC TRUE) + set(ENABLE_FEX_ALLOCATOR TRUE) set(ENABLE_JEMALLOC_GLIBC_ALLOC FALSE) else () message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}") @@ -319,12 +319,12 @@ elseif (NOT MINGW) " Use at your own risk!") endif() -if (ENABLE_JEMALLOC) - # The jemalloc subproject that all FEXCore fextl objects allocate through. - add_subdirectory(External/jemalloc/) -elseif (NOT MINGW) - message(STATUS - " jemalloc disabled!\n" +if (ENABLE_FEX_ALLOCATOR) + # The rpmalloc subproject that all FEXCore fextl objects allocate through. + add_subdirectory(External/rpmalloc/) +elseif (NOT MINGW_BUILD) + message (STATUS + " FEX allocator is disabled!\n" " This is not a recommended configuration!\n" " This will very explicitly break 32-bit application execution!\n" " Use at your own risk!") diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index e851d85df..2307973a8 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -291,9 +291,9 @@ endif() # Meta-library to link jemalloc libraries enabled in the build configuration. # Only needed for targets that run emulation. For others, use JemallocDummy. add_library(JemallocLibs STATIC Utils/AllocatorHooks.cpp) -if (ENABLE_JEMALLOC) - target_compile_definitions(JemallocLibs PRIVATE ENABLE_JEMALLOC=1 JEMALLOC_NO_RENAME=1) - target_link_libraries(JemallocLibs PUBLIC FEX_jemalloc) +if (ENABLE_FEX_ALLOCATOR) + target_compile_definitions(JemallocLibs PRIVATE ENABLE_FEX_ALLOCATOR=1) + target_link_libraries(JemallocLibs PUBLIC rpmalloc) endif() if (ENABLE_JEMALLOC_GLIBC_ALLOC) set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1) diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index 1b5f4d317..bbf9e09c1 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -81,21 +81,21 @@ int FEX_munmap(void* addr, size_t length) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -static void AssignHookOverrides() { - SetJemallocMmapHook(FEX_mmap); - SetJemallocMunmapHook(FEX_munmap); + +static void AssignHookOverrides(size_t PageSize) { + SetupAllocatorHooks(FEX_mmap, FEX_munmap); FEXCore::Allocator::mmap = FEX_mmap; FEXCore::Allocator::munmap = FEX_munmap; + InitializeAllocator(PageSize); } -void SetupHooks() { +void SetupHooks(size_t PageSize) { Alloc64 = Alloc::OSAllocator::Create64BitAllocator(); - AssignHookOverrides(); + AssignHookOverrides(PageSize); } void ClearHooks() { - SetJemallocMmapHook(::mmap); - SetJemallocMunmapHook(::munmap); + SetupAllocatorHooks(::mmap, ::munmap); FEXCore::Allocator::mmap = ::mmap; FEXCore::Allocator::munmap = ::munmap; @@ -271,7 +271,7 @@ fextl::vector StealMemoryRegion(uintptr_t Begin, uintptr_t End) { return Regions; } -fextl::vector Setup48BitAllocatorIfExists() { +fextl::vector Setup48BitAllocatorIfExists(size_t PageSize) { size_t Bits = FEXCore::Allocator::DetermineVASize(); if (Bits < 48) { return {}; @@ -282,7 +282,7 @@ fextl::vector Setup48BitAllocatorIfExists() { auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA); Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions); - AssignHookOverrides(); + AssignHookOverrides(PageSize); return Regions; } diff --git a/FEXCore/Source/Utils/AllocatorHooks.cpp b/FEXCore/Source/Utils/AllocatorHooks.cpp index cd0171b8a..b800c8a9b 100644 --- a/FEXCore/Source/Utils/AllocatorHooks.cpp +++ b/FEXCore/Source/Utils/AllocatorHooks.cpp @@ -1,67 +1,206 @@ // SPDX-License-Identifier: MIT -#ifdef ENABLE_JEMALLOC -#include +#ifdef ENABLE_FEX_ALLOCATOR +#include +#ifndef _WIN32 +#include +#include +#include +#else +#define NTDDI_VERSION 0x0A000005 +#include +#endif #endif +#include #include #include +#include +#include #include namespace FEXCore::Allocator { - -#ifndef _WIN32 using mmap_hook_type = void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset); using munmap_hook_type = int (*)(void* addr, size_t length); -#endif -#ifdef ENABLE_JEMALLOC -void* malloc(size_t size) { - return ::je_malloc(size); -} -void* calloc(size_t n, size_t size) { - return ::je_calloc(n, size); -} -void* memalign(size_t align, size_t s) { - return ::je_memalign(align, s); -} -void* valloc(size_t size) { - return ::je_valloc(size); -} -int posix_memalign(void** r, size_t a, size_t s) { - return ::je_posix_memalign(r, a, s); -} -void* realloc(void* ptr, size_t size) { - return ::je_realloc(ptr, size); -} -void free(void* ptr) { - return ::je_free(ptr); -} -size_t malloc_usable_size(void* ptr) { - return ::je_malloc_usable_size(ptr); -} -void* aligned_alloc(size_t a, size_t s) { - return ::je_aligned_alloc(a, s); -} -void aligned_free(void* ptr) { - return ::je_free(ptr); -} +#ifdef ENABLE_FEX_ALLOCATOR +typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size); +typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size); +extern "C" rp_mmap_hook_type rp_mmap_hook; +extern "C" rp_munmap_hook_type rp_munmap_hook; #ifndef _WIN32 -extern "C" mmap_hook_type je___mmap_hook; -extern "C" munmap_hook_type je___munmap_hook; - -void SetJemallocMmapHook(mmap_hook_type Hook) { - je___mmap_hook = Hook; -} -void SetJemallocMunmapHook(munmap_hook_type Hook) { - je___munmap_hook = Hook; -} +mmap_hook_type fex_mmap_hook = ::mmap; +munmap_hook_type fex_munmap_hook = ::munmap; #endif +// Assume a 64KB page size until told otherwise. +static rpmalloc_config_t global_config { + .page_size = 64 * 1024, + // THP causes crashes for some reason. + .enable_huge_pages = 0, + .disable_decommit = 0, + .page_name = "FEXAllocator", + .huge_page_name = "FEXAllocator", + .unmap_on_finalize = 0, +}; + +void* malloc(size_t size) { + return ::rpmalloc(size); +} +void* calloc(size_t n, size_t size) { + return ::rpcalloc(n, size); +} +void* memalign(size_t align, size_t s) { + return ::rpmemalign(align, s); +} +void* valloc(size_t size) { + return ::rpaligned_alloc(global_config.page_size, size); +} +int posix_memalign(void** r, size_t a, size_t s) { + void* ptr; + auto res = ::rpposix_memalign(&ptr, a, s); + *r = ptr; + return res; +} +void* realloc(void* ptr, size_t size) { + return ::rprealloc(ptr, size); +} +void free(void* ptr) { + return ::rpfree(ptr); +} +size_t malloc_usable_size(void* ptr) { + return ::rpmalloc_usable_size(ptr); +} +void* aligned_alloc(size_t a, size_t s) { + return ::rpaligned_alloc(a, s); +} +void aligned_free(void* ptr) { + return ::rpfree(ptr); +} + +void InitializeThread() { + rpmalloc_thread_initialize(); +} + +[[nodiscard]] +constexpr uint64_t AlignUp(uint64_t value, uint64_t size) { + return value + (size - value % size) % size; +} + +static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t* mapped_size) { +#define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) + // If the alignment is less than the operating page size then alignment is guaranteed. Just remove it. + if (alignment < global_config.page_size) { + alignment = 0; + } + + size_t map_size = AlignUp(size + alignment, global_config.page_size); +#ifdef _WIN32 + auto ptr = VirtualAlloc(0, map_size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + + if (ptr == MAP_FAILED) { + ptr = nullptr; + } else { +#ifndef PR_SET_VMA +#define PR_SET_VMA 0x53564d41 +#endif + +#ifndef PR_SET_VMA_ANON_NAME +#define PR_SET_VMA_ANON_NAME 0 +#endif + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, map_size, global_config.page_name); + } +#endif + + if (ptr == nullptr) { + fprintf(stderr, "Failed to map VMA region."); + return nullptr; + } + + if (alignment) { + size_t padding = ((uintptr_t)ptr & (uintptr_t)(alignment - 1)); + if (padding) { + padding = alignment - padding; + } + ptr = pointer_offset(ptr, padding); + *offset = padding; + } + *mapped_size = map_size; + return ptr; +} + +static void FEX_rp_memory_commit(void* address, size_t size) { + if (global_config.disable_decommit) { + return; + } + +#ifdef _WIN32 + if (!VirtualAlloc(address, size, MEM_COMMIT, PAGE_READWRITE)) { +#else + if (false) { +#endif + fprintf(stderr, "Failed to commit VMA region."); + } +} + +static void FEX_rp_memory_decommit(void* address, size_t size) { + if (global_config.disable_decommit) { + return; + } + +#ifdef _WIN32 + if (!VirtualFree(address, size, MEM_DECOMMIT)) { +#else + if (madvise(address, size, MADV_DONTNEED)) { +#endif + fprintf(stderr, "Failed to decommit VMA region."); + } +} + +static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) { + address = pointer_offset(address, -(int32_t)offset); +#ifdef _WIN32 + ::VirtualFree(address, mapped_size, MEM_RELEASE); +#else + int Result = fex_munmap_hook(address, mapped_size); + if (Result == -1) { + fprintf(stderr, "Failed to unmap VMA region."); + } +#endif +#undef pointer_offset +} + +void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) { +#ifndef _WIN32 + fex_mmap_hook = MMapHook; + fex_munmap_hook = MunmapHook; +#endif +} + +static rpmalloc_interface_t global_interface { + .memory_map = FEX_rp_mmap, + .memory_commit = FEX_rp_memory_commit, + .memory_decommit = FEX_rp_memory_decommit, + .memory_unmap = FEX_rp_memory_unmap, + .map_fail_callback = nullptr, + .error_callback = nullptr, +}; + +void InitializeAllocator(size_t PageSize) { + global_config.page_size = PageSize; + rpmalloc_initialize_config(&global_interface, &global_config); + rp_mmap_hook = FEX_rp_mmap; + rp_munmap_hook = FEX_rp_memory_unmap; +} + #elif defined(_WIN32) #error "Tried building _WIN32 without jemalloc" #else +void InitializeThread() {} + void* malloc(size_t size) { return ::malloc(size); } @@ -93,8 +232,9 @@ void aligned_free(void* ptr) { return ::free(ptr); } -void SetJemallocMmapHook(mmap_hook_type) {} -void SetJemallocMunmapHook(munmap_hook_type) {} +void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) {} + +void InitializeAllocator(size_t PageSize) {} #endif } // namespace FEXCore::Allocator diff --git a/FEXCore/include/FEXCore/Utils/Allocator.h b/FEXCore/include/FEXCore/Utils/Allocator.h index cd4fbd1fd..1fec1b15d 100644 --- a/FEXCore/include/FEXCore/Utils/Allocator.h +++ b/FEXCore/include/FEXCore/Utils/Allocator.h @@ -12,7 +12,7 @@ struct InternalThreadState; } namespace FEXCore::Allocator { -FEX_DEFAULT_VISIBILITY void SetupHooks(); +FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize); FEX_DEFAULT_VISIBILITY void ClearHooks(); FEX_DEFAULT_VISIBILITY size_t DetermineVASize(); @@ -77,7 +77,7 @@ FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(const fextl::vector Setup48BitAllocatorIfExists(); +FEX_DEFAULT_VISIBILITY fextl::vector Setup48BitAllocatorIfExists(size_t PageSize); #ifndef _WIN32 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread); diff --git a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h index 718287b53..7188a872a 100644 --- a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h +++ b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h @@ -139,9 +139,11 @@ size_t malloc_usable_size(void* ptr); void* aligned_alloc(size_t a, size_t s); void aligned_free(void* ptr); +FEX_DEFAULT_VISIBILITY extern void InitializeThread(); + #ifndef _WIN32 -void SetJemallocMmapHook(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset)); -void SetJemallocMunmapHook(int (*)(void* addr, size_t length)); +void InitializeAllocator(size_t PageSize); +void SetupAllocatorHooks(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset), int (*)(void* addr, size_t length)); #endif struct FEXAllocOperators { diff --git a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp index 71bd91575..f5113c8c4 100644 --- a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp +++ b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp @@ -145,9 +145,10 @@ void Init() { namespace FEX::Allocator { fextl::vector InitMemoryRegions(bool Is64Bit) { + const auto PageSize = sysconf(_SC_PAGESIZE); if (Is64Bit) { // Destroy the 48th bit if it exists - return FEXCore::Allocator::Setup48BitAllocatorIfExists(); + return FEXCore::Allocator::Setup48BitAllocatorIfExists(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); } // Reserve [0x1_0000_0000, 0x2_0000_0000). @@ -161,8 +162,10 @@ fextl::unique_ptr InitAllocator(bool Is64Bit) { return {}; } + const auto PageSize = sysconf(_SC_PAGESIZE); + // Setup our userspace allocator - FEXCore::Allocator::SetupHooks(); + FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); auto Allocator = FEX::HLE::CreatePassthroughAllocator(); // Now that the upper 32-bit address space is blocked for future allocations, diff --git a/Source/Tools/FEXOfflineCompiler/Main.cpp b/Source/Tools/FEXOfflineCompiler/Main.cpp index 1f1a66aa1..da2f68e11 100644 --- a/Source/Tools/FEXOfflineCompiler/Main.cpp +++ b/Source/Tools/FEXOfflineCompiler/Main.cpp @@ -154,8 +154,9 @@ GenerateSingleCache(const FEXCore::ExecutableFileInfo& Binary, fextl::set 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); } auto Thread = SetupCompileThread(*CTX, Is64Bit); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp index 47e1d0c97..aa13b9cd9 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp @@ -68,6 +68,8 @@ static void* ThreadHandler(void* Data) { Thread->Thread->ThreadStats->TID.store(Thread->ThreadInfo.TID, std::memory_order_relaxed); } + FEXCore::Allocator::InitializeThread(); + FEX::HLE::_SyscallHandler->RegisterTLSState(Thread); // Now notify the thread that we are initialized @@ -176,6 +178,7 @@ FEX::HLE::ThreadStateObject* CreateNewThread(FEXCore::Context::Context* CTX, FEX uint64_t HandleNewClone(FEX::HLE::ThreadStateObject* Thread, FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame, FEX::HLE::clone3_args* CloneArgs) { + FEXCore::Allocator::InitializeThread(); auto GuestArgs = &CloneArgs->args; uint64_t flags = GuestArgs->flags; auto NewThread = Thread; diff --git a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp index b8426fe9f..4a0f85115 100644 --- a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp +++ b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp @@ -236,7 +236,8 @@ int main(int argc, char** argv, char** const envp) { if (!Loader.Is64BitMode()) { // Setup our userspace allocator - FEXCore::Allocator::SetupHooks(); + const auto PageSize = sysconf(_SC_PAGESIZE); + FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); Allocator = FEX::HLE::CreatePassthroughAllocator(); } #endif diff --git a/Source/Windows/Common/CRT/Alloc.cpp b/Source/Windows/Common/CRT/Alloc.cpp index 3900537a8..3d5f60de8 100644 --- a/Source/Windows/Common/CRT/Alloc.cpp +++ b/Source/Windows/Common/CRT/Alloc.cpp @@ -3,42 +3,28 @@ #define _CRTIMP #include #include "../Priv.h" - -extern "C" { - -#define JEMALLOC_NOTHROW __attribute__((nothrow)) -JEMALLOC_NOTHROW extern void* je_malloc(size_t size); -JEMALLOC_NOTHROW extern void* je_calloc(size_t n, size_t size); -JEMALLOC_NOTHROW extern void* je_memalign(size_t align, size_t s); -JEMALLOC_NOTHROW extern void* je_valloc(size_t size); -JEMALLOC_NOTHROW extern int je_posix_memalign(void** r, size_t a, size_t s); -JEMALLOC_NOTHROW extern void* je_realloc(void* ptr, size_t size); -JEMALLOC_NOTHROW extern void je_free(void* ptr); -JEMALLOC_NOTHROW extern size_t je_malloc_usable_size(void* ptr); -JEMALLOC_NOTHROW extern void* je_aligned_alloc(size_t a, size_t s); -#undef JEMALLOC_NOTHROW -} +#include void* calloc(size_t NumOfElements, size_t SizeOfElements) { - return je_calloc(NumOfElements, SizeOfElements); + return ::rpcalloc(NumOfElements, SizeOfElements); } void free(void* Memory) { - je_free(Memory); + ::rpfree(Memory); } void* malloc(size_t Size) { - return je_malloc(Size); + return ::rpmalloc(Size); } void* realloc(void* Memory, size_t NewSize) { - return je_realloc(Memory, NewSize); + return ::rprealloc(Memory, NewSize); } DLLEXPORT_FUNC(void*, _aligned_malloc, (size_t Size, size_t Alignment)) { - return je_aligned_alloc(Alignment, Size); + return ::rpaligned_alloc(Alignment, Size); } DLLEXPORT_FUNC(void, _aligned_free, (void* Memory)) { - je_free(Memory); + ::rpfree(Memory); }