diff --git a/.gitmodules b/.gitmodules index f31933488..5e88b57bf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -53,3 +53,6 @@ shallow = true path = External/Vulkan-Headers url = https://github.com/KhronosGroup/Vulkan-Headers.git +[submodule "External/jemalloc_glibc"] + path = External/jemalloc_glibc + url = https://github.com/FEX-Emu/jemalloc.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 9080872ec..9f0721d39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ option(ENABLE_VISUAL_DEBUGGER "Enables the visual debugger for compiling" FALSE) option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE) option(ENABLE_WERROR "Enables -Werror" FALSE) option(ENABLE_JEMALLOC "Enables jemalloc allocator" TRUE) +option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enables jemalloc glibc allocator" TRUE) option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE) option(ENABLE_COMPILE_TIME_TRACE "Enables time trace compile option" FALSE) option(ENABLE_LIBCXX "Enables LLVM libc++" FALSE) @@ -57,6 +58,10 @@ if (ENABLE_FEXCORE_PROFILER) endif() endif() +if (ENABLE_JEMALLOC_GLIBC_ALLOC AND ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT) + message(FATAL_ERROR "Can't have both glibc fault allocator and jemalloc glibc allocator enabled at the same time") +endif() + if (ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT) add_definitions(-DGLIBC_ALLOCATOR_FAULT=1) endif() @@ -190,7 +195,22 @@ if (ENABLE_TSAN) link_libraries(-fno-omit-frame-pointer -fsanitize=thread) endif() +if (ENABLE_JEMALLOC_GLIBC_ALLOC) + # The glibc jemalloc subproject which hooks the glibc allocator. + # Required for thunks to work. + # All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator. + add_definitions(-DENABLE_JEMALLOC_GLIBC=1) + add_subdirectory(External/jemalloc_glibc/) +else() + message (STATUS + " jemalloc glibc allocator disabled!\n" + " This is not a recommended configuration!\n" + " This will very explicitly break thunk execution!\n" + " Use at your own risk!") +endif() + if (ENABLE_JEMALLOC) + # The jemalloc subproject that all FEXCore fextl objects allocate through. add_definitions(-DENABLE_JEMALLOC=1) add_subdirectory(External/jemalloc/) include_directories(External/jemalloc/pregen/include/) diff --git a/External/FEXCore/Source/CMakeLists.txt b/External/FEXCore/Source/CMakeLists.txt index 882ffb5a8..d534350c6 100644 --- a/External/FEXCore/Source/CMakeLists.txt +++ b/External/FEXCore/Source/CMakeLists.txt @@ -233,6 +233,10 @@ if (ENABLE_JEMALLOC) list (APPEND LIBS FEX_jemalloc) endif() +if (ENABLE_JEMALLOC_GLIBC_ALLOC) + list (APPEND LIBS FEX_jemalloc_glibc) +endif() + # Generate config configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/Interface/Config/Config.json.in diff --git a/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp b/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp index bdfc2ab34..45d530f3e 100644 --- a/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp +++ b/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp @@ -29,8 +29,14 @@ $end_info$ #include #include -#ifdef ENABLE_JEMALLOC -#include "jemalloc/jemalloc.h" +#ifdef ENABLE_JEMALLOC_GLIBC +extern "C" { +// jemalloc defines nothrow on its internal C function signatures. +#define JEMALLOC_NOTHROW __attribute__((nothrow)) +// Forward declare jemalloc functions because we can't include the headers from the glibc jemalloc project. +// This is because we can't simultaneously set up include paths for both of our internal jemalloc modules. +FEX_DEFAULT_VISIBILITY JEMALLOC_NOTHROW extern int glibc_je_is_known_allocation(void *ptr); +} #endif struct LoadlibArgs { @@ -251,13 +257,13 @@ namespace FEXCore { * and host heap pointers. */ static void IsHostHeapAllocation(void* ArgsRV) { -#ifdef ENABLE_JEMALLOC +#ifdef ENABLE_JEMALLOC_GLIBC struct ArgsRV_t { void* ptr; bool rv; } *args = reinterpret_cast(ArgsRV); - args->rv = je_is_known_allocation(args->ptr); + args->rv = glibc_je_is_known_allocation(args->ptr); #else // Thunks usage without jemalloc isn't supported ERROR_AND_DIE_FMT("Unsupported: Thunks querying for host heap allocation information"); diff --git a/External/FEXCore/Source/Utils/Allocator.cpp b/External/FEXCore/Source/Utils/Allocator.cpp index fcf7c3c11..67f938b54 100644 --- a/External/FEXCore/Source/Utils/Allocator.cpp +++ b/External/FEXCore/Source/Utils/Allocator.cpp @@ -29,8 +29,8 @@ extern "C" { typedef int (*munmap_hook_type)(void *addr, size_t length); #ifdef ENABLE_JEMALLOC - extern mmap_hook_type __mmap_hook; - extern munmap_hook_type __munmap_hook; + extern mmap_hook_type je___mmap_hook; + extern munmap_hook_type je___munmap_hook; #endif } @@ -122,8 +122,8 @@ namespace FEXCore::Allocator { void SetupHooks() { Alloc64 = Alloc::OSAllocator::Create64BitAllocator(); #ifdef ENABLE_JEMALLOC - __mmap_hook = FEX_mmap; - __munmap_hook = FEX_munmap; + je___mmap_hook = FEX_mmap; + je___munmap_hook = FEX_munmap; #endif FEXCore::Allocator::mmap = FEX_mmap; FEXCore::Allocator::munmap = FEX_munmap; @@ -131,8 +131,8 @@ namespace FEXCore::Allocator { void ClearHooks() { #ifdef ENABLE_JEMALLOC - __mmap_hook = ::mmap; - __munmap_hook = ::munmap; + je___mmap_hook = ::mmap; + je___munmap_hook = ::munmap; #endif FEXCore::Allocator::mmap = ::mmap; FEXCore::Allocator::munmap = ::munmap; diff --git a/External/jemalloc b/External/jemalloc index ac8870dd6..19196715f 160000 --- a/External/jemalloc +++ b/External/jemalloc @@ -1 +1 @@ -Subproject commit ac8870dd6bc0573392686680b0e20b050244bf73 +Subproject commit 19196715f12207455e32b4912618f0db8d4af4bb diff --git a/External/jemalloc_glibc b/External/jemalloc_glibc new file mode 160000 index 000000000..79e02853e --- /dev/null +++ b/External/jemalloc_glibc @@ -0,0 +1 @@ +Subproject commit 79e02853ed2f62fae182af2374274b66dda757e3 diff --git a/docs/jemalloc_usage.md b/docs/jemalloc_usage.md new file mode 100644 index 000000000..ab27c1345 --- /dev/null +++ b/docs/jemalloc_usage.md @@ -0,0 +1,35 @@ +# JEMalloc usage +FEX-Emu uses two different jemalloc heap allocators at once, each for different purposes: +- jemalloc: The primary heap allocator (to keep FEX's internal allocations out of the 32-bit address space used by guest applications) +- jemalloc_glibc: The second heap allocator (to add allocation introspection features used by thunks) + +## jemalloc - primary heap allocator +This allocator overrides `mmap` and `munmap` by forwarding them to FEXCore's internal VMA region allocator. + +All of FEXCore's `fextl::` namespaced objects allocate memory with this method. + +### FEXCore internal VMA region allocator +When running a 32-bit guest application, the VMA region allocator allocates from memory *above* the 4GB of virtual address space reserved for the +32-bit application. + +This ensures that all of FEX's allocations stay out of the lower 32-bit 4GB VA space, since games would quickly run out of virtual address space +otherwise. + +When running a 64-bit guest application, this VMA region allocator is disabled and passes through to mmap and munmap in the host kernel. + +## jemalloc_glibc - secondary heap allocator +This heap allocator replaces the host glibc's allocator using weak symbol overriding. It adds introspection features used for thunking, but has no +functional differences otherwise: All memory is allocated in the 4GB of 32-bit address space. The FEXCore VMA region allocator is explicitly **not** +involved hence. + +All native shared libraries use this allocator including the host-side of thunks. + +Internally, all allocations that go through this heap allocator use the kernel mmap and munmap interface. + +### Thunks +Thunks may allocate memory either through the guest-side (on the guest glibc heap) or the host-side (on the `jemalloc_glibc` heap). To properly free +this memory, FEX must be able to determine which heap allocator it belongs to. + +`glibc` provides no public interface to do this, but FEX's `jemalloc` fork does. The `is_known_allocation` function is used by FEXCore to query +whether a given pointer originated from the `jemalloc_glibc` allocator. This enables FEXCore to determine the appropriate heap for freeing the +pointer.