diff --git a/.gitmodules b/.gitmodules index 6d41ac236..35c6317dc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,6 +17,9 @@ shallow = true path = External/fex-gcc-target-tests-bins url = https://github.com/FEX-Emu/fex-gcc-target-tests-bins.git +[submodule "External/jemalloc"] + path = External/jemalloc + url = https://github.com/FEX-Emu/jemalloc.git [submodule "External/fmt"] path = External/fmt url = https://github.com/fmtlib/fmt.git @@ -46,6 +49,3 @@ [submodule "External/range-v3"] path = External/range-v3 url = https://github.com/ericniebler/range-v3.git -[submodule "External/rpmalloc"] - path = External/rpmalloc - url = https://github.com/FEX-Emu/rpmalloc.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 10f130665..68e48349c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ option(ENABLE_ASSERTIONS "Enables assertions in build" FALSE) option(ENABLE_GDB_SYMBOLS "Enables GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H}) option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE) option(ENABLE_WERROR "Enables -Werror" FALSE) -option(ENABLE_FEX_ALLOCATOR "Enables allocator for FEX" TRUE) +option(ENABLE_JEMALLOC "Enables jemalloc allocator" TRUE) option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enables jemalloc glibc allocator" TRUE) option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE) option(ENABLE_COMPILE_TIME_TRACE "Enables time trace compile option" FALSE) @@ -52,7 +52,7 @@ string(FIND ${CMAKE_BASE_NAME} mingw CONTAINS_MINGW) if (NOT CONTAINS_MINGW EQUAL -1) message (STATUS "Mingw build") set (MINGW_BUILD TRUE) - set (ENABLE_FEX_ALLOCATOR TRUE) + set (ENABLE_JEMALLOC TRUE) set (ENABLE_JEMALLOC_GLIBC_ALLOC FALSE) endif() @@ -279,12 +279,12 @@ elseif (NOT MINGW_BUILD) " Use at your own risk!") endif() -if (ENABLE_FEX_ALLOCATOR) - # The rpmalloc subproject that all FEXCore fextl objects allocate through. - add_subdirectory(External/rpmalloc/) +if (ENABLE_JEMALLOC) + # The jemalloc subproject that all FEXCore fextl objects allocate through. + add_subdirectory(External/jemalloc/) elseif (NOT MINGW_BUILD) message (STATUS - " FEX allocator is disabled!\n" + " jemalloc disabled!\n" " This is not a recommended configuration!\n" " This will very explicitly break 32-bit application execution!\n" " Use at your own risk!") diff --git a/External/jemalloc b/External/jemalloc new file mode 160000 index 000000000..ce2459301 --- /dev/null +++ b/External/jemalloc @@ -0,0 +1 @@ +Subproject commit ce24593018ca5d5af7e5661ceda9744e02b59f8f diff --git a/External/rpmalloc b/External/rpmalloc deleted file mode 160000 index f1b76e16a..000000000 --- a/External/rpmalloc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f1b76e16a354713b7e91162ef9b091f7826ab701 diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index 3341d5b68..d5f902496 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -299,9 +299,9 @@ endif() # Meta-library to link jemalloc libraries enabled in the build configuration. # Only needed for targets that run emulation. For others, use JemallocDummy. add_library(JemallocLibs STATIC Utils/AllocatorHooks.cpp) -if (ENABLE_FEX_ALLOCATOR) - target_compile_definitions(JemallocLibs PRIVATE ENABLE_FEX_ALLOCATOR=1) - target_link_libraries(JemallocLibs PUBLIC rpmalloc) +if (ENABLE_JEMALLOC) + target_compile_definitions(JemallocLibs PRIVATE ENABLE_JEMALLOC=1 JEMALLOC_NO_RENAME=1) + target_link_libraries(JemallocLibs PUBLIC FEX_jemalloc) endif() if (ENABLE_JEMALLOC_GLIBC_ALLOC) set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1) diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index 11670e1d0..134983f11 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -122,21 +122,21 @@ void ReenableSBRKAllocations(void* Ptr) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" - -static void AssignHookOverrides(size_t PageSize) { - SetupAllocatorHooks(FEX_mmap, FEX_munmap); +static void AssignHookOverrides() { + SetJemallocMmapHook(FEX_mmap); + SetJemallocMunmapHook(FEX_munmap); FEXCore::Allocator::mmap = FEX_mmap; FEXCore::Allocator::munmap = FEX_munmap; - InitializeAllocator(PageSize); } -void SetupHooks(size_t PageSize) { +void SetupHooks() { Alloc64 = Alloc::OSAllocator::Create64BitAllocator(); - AssignHookOverrides(PageSize); + AssignHookOverrides(); } void ClearHooks() { - SetupAllocatorHooks(::mmap, ::munmap); + SetJemallocMmapHook(::mmap); + SetJemallocMunmapHook(::munmap); FEXCore::Allocator::mmap = ::mmap; FEXCore::Allocator::munmap = ::munmap; @@ -314,7 +314,7 @@ fextl::vector StealMemoryRegion(uintptr_t Begin, uintptr_t End) { return Regions; } -fextl::vector Setup48BitAllocatorIfExists(size_t PageSize) { +fextl::vector Setup48BitAllocatorIfExists() { size_t Bits = FEXCore::Allocator::DetermineVASize(); if (Bits < 48) { return {}; @@ -325,7 +325,7 @@ fextl::vector Setup48BitAllocatorIfExists(size_t PageSize) { auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA); Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions); - AssignHookOverrides(PageSize); + AssignHookOverrides(); return Regions; } diff --git a/FEXCore/Source/Utils/AllocatorHooks.cpp b/FEXCore/Source/Utils/AllocatorHooks.cpp index 87e38d51f..60bfabf87 100644 --- a/FEXCore/Source/Utils/AllocatorHooks.cpp +++ b/FEXCore/Source/Utils/AllocatorHooks.cpp @@ -1,175 +1,59 @@ // SPDX-License-Identifier: MIT -#ifdef ENABLE_FEX_ALLOCATOR -#include -#ifndef _WIN32 -#include -#include -#include -#else -#define NTDDI_VERSION 0x0A000005 -#include -#endif +#ifdef ENABLE_JEMALLOC +#include #endif -#include #include #include -#include -#include -#include namespace FEXCore::Allocator { + +#ifndef _WIN32 using mmap_hook_type = void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset); using munmap_hook_type = int (*)(void* addr, size_t length); - -#ifdef ENABLE_FEX_ALLOCATOR -typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size); -typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size); -extern "C" rp_mmap_hook_type rp_mmap_hook; -extern "C" rp_munmap_hook_type rp_munmap_hook; - -#ifndef _WIN32 -mmap_hook_type fex_mmap_hook = ::mmap; -munmap_hook_type fex_munmap_hook = ::munmap; #endif -// Assume a 64KB page size until told otherwise. -static rpmalloc_config_t global_config { - .page_size = 64 * 1024, - // THP causes crashes for some reason. - .enable_huge_pages = 0, - .disable_decommit = 0, - .page_name = "FEXAllocator", - .huge_page_name = "FEXAllocator", - .unmap_on_finalize = 0, -}; - +#ifdef ENABLE_JEMALLOC void* malloc(size_t size) { - return ::rpmalloc(size); + return ::je_malloc(size); } void* calloc(size_t n, size_t size) { - return ::rpcalloc(n, size); + return ::je_calloc(n, size); } void* memalign(size_t align, size_t s) { - return ::rpmemalign(align, s); + return ::je_memalign(align, s); } void* valloc(size_t size) { - return ::rpaligned_alloc(global_config.page_size, size); + return ::je_valloc(size); } int posix_memalign(void** r, size_t a, size_t s) { - void* ptr; - auto res = ::rpposix_memalign(&ptr, a, s); - *r = ptr; - return res; + return ::je_posix_memalign(r, a, s); } void* realloc(void* ptr, size_t size) { - return ::rprealloc(ptr, size); + return ::je_realloc(ptr, size); } void free(void* ptr) { - return ::rpfree(ptr); + return ::je_free(ptr); } size_t malloc_usable_size(void* ptr) { - return ::rpmalloc_usable_size(ptr); + return ::je_malloc_usable_size(ptr); } void* aligned_alloc(size_t a, size_t s) { - return ::rpaligned_alloc(a, s); + return ::je_aligned_alloc(a, s); } void aligned_free(void* ptr) { - return ::rpfree(ptr); -} - -void InitializeThread() { - rpmalloc_thread_initialize(); + return ::je_free(ptr); } #ifndef _WIN32 -[[nodiscard]] -constexpr uint64_t AlignUp(uint64_t value, uint64_t size) { - return value + (size - value % size) % size; +extern "C" mmap_hook_type je___mmap_hook; +extern "C" munmap_hook_type je___munmap_hook; + +void SetJemallocMmapHook(mmap_hook_type Hook) { + je___mmap_hook = Hook; } - -static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t* mapped_size) { -#define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) - // If the alignment is less than the operating page size then alignment is guaranteed. Just remove it. - if (alignment < global_config.page_size) { - alignment = 0; - } - - size_t map_size = AlignUp(size + alignment, global_config.page_size); - auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - - if (ptr == MAP_FAILED) { - ptr = nullptr; - } else { -#ifndef PR_SET_VMA -#define PR_SET_VMA 0x53564d41 -#endif - -#ifndef PR_SET_VMA_ANON_NAME -#define PR_SET_VMA_ANON_NAME 0 -#endif - prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, map_size, global_config.page_name); - } - - if (ptr == nullptr) { - fprintf(stderr, "Failed to map VMA region."); - return nullptr; - } - - if (alignment) { - size_t padding = ((uintptr_t)ptr & (uintptr_t)(alignment - 1)); - if (padding) { - padding = alignment - padding; - } - ptr = pointer_offset(ptr, padding); - *offset = padding; - } - *mapped_size = map_size; - return ptr; -} - -static void FEX_rp_memory_commit(void* address, size_t size) { - // NOP-implementation. -} - -static void FEX_rp_memory_decommit(void* address, size_t size) { - if (global_config.disable_decommit) { - return; - } - - if (madvise(address, size, MADV_DONTNEED)) { - fprintf(stderr, "Failed to decommit VMA region."); - } -} - -static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) { - address = pointer_offset(address, -(int32_t)offset); - int Result = fex_munmap_hook(address, mapped_size); - if (Result == -1) { - fprintf(stderr, "Failed to unmap VMA region."); - } -#undef pointer_offset -} - -void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) { - fex_mmap_hook = MMapHook; - fex_munmap_hook = MunmapHook; -} - -static rpmalloc_interface_t global_interface { - .memory_map = FEX_rp_mmap, - .memory_commit = FEX_rp_memory_commit, - .memory_decommit = FEX_rp_memory_decommit, - .memory_unmap = FEX_rp_memory_unmap, - .map_fail_callback = nullptr, - .error_callback = nullptr, -}; - -void InitializeAllocator(size_t PageSize) { - global_config.page_size = PageSize; - rpmalloc_initialize_config(&global_interface, &global_config); - rp_mmap_hook = FEX_rp_mmap; - rp_munmap_hook = FEX_rp_memory_unmap; +void SetJemallocMunmapHook(munmap_hook_type Hook) { + je___munmap_hook = Hook; } #endif @@ -177,8 +61,6 @@ void InitializeAllocator(size_t PageSize) { #error "Tried building _WIN32 without jemalloc" #else -void InitializeThread() {} - void* malloc(size_t size) { return ::malloc(size); } @@ -210,9 +92,8 @@ void aligned_free(void* ptr) { return ::free(ptr); } -void SetupAllocatorHooks(mmap_hook_type MMapHook, munmap_hook_type MunmapHook) {} - -void InitializeAllocator(size_t PageSize) {} +void SetJemallocMmapHook(mmap_hook_type) {} +void SetJemallocMunmapHook(munmap_hook_type) {} #endif } // namespace FEXCore::Allocator diff --git a/FEXCore/include/FEXCore/Utils/Allocator.h b/FEXCore/include/FEXCore/Utils/Allocator.h index 534d8f25e..fedf45257 100644 --- a/FEXCore/include/FEXCore/Utils/Allocator.h +++ b/FEXCore/include/FEXCore/Utils/Allocator.h @@ -12,7 +12,7 @@ struct InternalThreadState; } namespace FEXCore::Allocator { -FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize); +FEX_DEFAULT_VISIBILITY void SetupHooks(); FEX_DEFAULT_VISIBILITY void ClearHooks(); FEX_DEFAULT_VISIBILITY size_t DetermineVASize(); @@ -84,7 +84,7 @@ FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(const fextl::vector Setup48BitAllocatorIfExists(size_t PageSize); +FEX_DEFAULT_VISIBILITY fextl::vector Setup48BitAllocatorIfExists(); #ifndef _WIN32 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread); diff --git a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h index 845262dc0..2143ed64b 100644 --- a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h +++ b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h @@ -139,11 +139,9 @@ size_t malloc_usable_size(void* ptr); void* aligned_alloc(size_t a, size_t s); void aligned_free(void* ptr); -FEX_DEFAULT_VISIBILITY extern void InitializeThread(); - #ifndef _WIN32 -void InitializeAllocator(size_t PageSize); -void SetupAllocatorHooks(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset), int (*)(void* addr, size_t length)); +void SetJemallocMmapHook(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset)); +void SetJemallocMunmapHook(int (*)(void* addr, size_t length)); #endif struct FEXAllocOperators { diff --git a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp index c4b31e4ef..cfedcb016 100644 --- a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp +++ b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp @@ -422,10 +422,9 @@ int main(int argc, char** argv, char** const envp) { fextl::vector Base48Bit; fextl::vector Low4GB; - const auto PageSize = sysconf(_SC_PAGESIZE); if (Loader.Is64BitMode()) { // Destroy the 48th bit if it exists - Base48Bit = FEXCore::Allocator::Setup48BitAllocatorIfExists(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); + Base48Bit = FEXCore::Allocator::Setup48BitAllocatorIfExists(); } else { // Reserve [0x1_0000_0000, 0x2_0000_0000). // Safety net if 32-bit address calculation overflows in to 64-bit range. @@ -433,7 +432,7 @@ int main(int argc, char** argv, char** const envp) { Low4GB = FEXCore::Allocator::StealMemoryRegion(First64BitAddr, First64BitAddr + First64BitAddr); // Setup our userspace allocator - FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); + FEXCore::Allocator::SetupHooks(); Allocator = FEX::HLE::CreatePassthroughAllocator(); // Now that the upper 32-bit address space is blocked for future allocations, diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp index 6f572c37d..c99f7b408 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp @@ -68,8 +68,6 @@ static void* ThreadHandler(void* Data) { Thread->Thread->ThreadStats->TID.store(Thread->ThreadInfo.TID, std::memory_order_relaxed); } - FEXCore::Allocator::InitializeThread(); - FEX::HLE::_SyscallHandler->RegisterTLSState(Thread); // Now notify the thread that we are initialized @@ -178,7 +176,6 @@ FEX::HLE::ThreadStateObject* CreateNewThread(FEXCore::Context::Context* CTX, FEX uint64_t HandleNewClone(FEX::HLE::ThreadStateObject* Thread, FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame, FEX::HLE::clone3_args* CloneArgs) { - FEXCore::Allocator::InitializeThread(); auto GuestArgs = &CloneArgs->args; uint64_t flags = GuestArgs->flags; auto NewThread = Thread; diff --git a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp index cc2e64e76..33ff0be7b 100644 --- a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp +++ b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp @@ -235,8 +235,7 @@ int main(int argc, char** argv, char** const envp) { if (!Loader.Is64BitMode()) { // Setup our userspace allocator - const auto PageSize = sysconf(_SC_PAGESIZE); - FEXCore::Allocator::SetupHooks(PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE); + FEXCore::Allocator::SetupHooks(); Allocator = FEX::HLE::CreatePassthroughAllocator(); } #endif diff --git a/Source/Windows/ARM64EC/Module.cpp b/Source/Windows/ARM64EC/Module.cpp index 972320edb..8b072b99f 100644 --- a/Source/Windows/ARM64EC/Module.cpp +++ b/Source/Windows/ARM64EC/Module.cpp @@ -1049,7 +1049,6 @@ NTSTATUS ThreadTerm(HANDLE Thread, LONG ExitCode) { if (ThreadTID == GetCurrentThreadId()) { FEX::Windows::DeinitCRTThread(); } - return STATUS_SUCCESS; } diff --git a/Source/Windows/Common/CRT/Alloc.cpp b/Source/Windows/Common/CRT/Alloc.cpp index 3d5f60de8..3900537a8 100644 --- a/Source/Windows/Common/CRT/Alloc.cpp +++ b/Source/Windows/Common/CRT/Alloc.cpp @@ -3,28 +3,42 @@ #define _CRTIMP #include #include "../Priv.h" -#include + +extern "C" { + +#define JEMALLOC_NOTHROW __attribute__((nothrow)) +JEMALLOC_NOTHROW extern void* je_malloc(size_t size); +JEMALLOC_NOTHROW extern void* je_calloc(size_t n, size_t size); +JEMALLOC_NOTHROW extern void* je_memalign(size_t align, size_t s); +JEMALLOC_NOTHROW extern void* je_valloc(size_t size); +JEMALLOC_NOTHROW extern int je_posix_memalign(void** r, size_t a, size_t s); +JEMALLOC_NOTHROW extern void* je_realloc(void* ptr, size_t size); +JEMALLOC_NOTHROW extern void je_free(void* ptr); +JEMALLOC_NOTHROW extern size_t je_malloc_usable_size(void* ptr); +JEMALLOC_NOTHROW extern void* je_aligned_alloc(size_t a, size_t s); +#undef JEMALLOC_NOTHROW +} void* calloc(size_t NumOfElements, size_t SizeOfElements) { - return ::rpcalloc(NumOfElements, SizeOfElements); + return je_calloc(NumOfElements, SizeOfElements); } void free(void* Memory) { - ::rpfree(Memory); + je_free(Memory); } void* malloc(size_t Size) { - return ::rpmalloc(Size); + return je_malloc(Size); } void* realloc(void* Memory, size_t NewSize) { - return ::rprealloc(Memory, NewSize); + return je_realloc(Memory, NewSize); } DLLEXPORT_FUNC(void*, _aligned_malloc, (size_t Size, size_t Alignment)) { - return ::rpaligned_alloc(Alignment, Size); + return je_aligned_alloc(Alignment, Size); } DLLEXPORT_FUNC(void, _aligned_free, (void* Memory)) { - ::rpfree(Memory); + je_free(Memory); } diff --git a/Source/Windows/Common/CRT/CRT.cpp b/Source/Windows/Common/CRT/CRT.cpp index addea79d8..bf87f1b0f 100644 --- a/Source/Windows/Common/CRT/CRT.cpp +++ b/Source/Windows/Common/CRT/CRT.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include "CRT.h" extern "C" { @@ -40,8 +39,6 @@ void RunFuncArray(TFuncIt Begin, TFuncIt End, TArgs... Args) { namespace FEX::Windows { void InitCRTProcess() { - rpmalloc_initialize(nullptr); - auto GNUCtorBegin = &__CTOR_LIST__[1]; auto GNUCtorEnd = GNUCtorBegin; while (*GNUCtorEnd != nullptr) { @@ -56,12 +53,10 @@ void InitCRTProcess() { } void InitCRTThread() { - rpmalloc_thread_initialize(); RunFuncArray(&XLA, &XLZ, nullptr, DLL_THREAD_ATTACH, nullptr); } void DeinitCRTThread() { RunFuncArray(&XLA, &XLZ, nullptr, DLL_THREAD_DETACH, nullptr); - rpmalloc_thread_finalize(); } } // namespace FEX::Windows