Merge pull request #2591 from Sonicadvance1/new_jemalloc

Add in jemalloc glibc hooking again

.gitmodules

@@ -53,3 +53,6 @@
 	shallow = true
 	path = External/Vulkan-Headers
 	url = https://github.com/KhronosGroup/Vulkan-Headers.git
+[submodule "External/jemalloc_glibc"]
+	path = External/jemalloc_glibc
+	url = https://github.com/FEX-Emu/jemalloc.git

CMakeLists.txt

@@ -22,6 +22,7 @@ option(ENABLE_VISUAL_DEBUGGER "Enables the visual debugger for compiling" FALSE)
 option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE)
 option(ENABLE_WERROR "Enables -Werror" FALSE)
 option(ENABLE_JEMALLOC "Enables jemalloc allocator" TRUE)
+option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enables jemalloc glibc allocator" TRUE)
 option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE)
 option(ENABLE_COMPILE_TIME_TRACE "Enables time trace compile option" FALSE)
 option(ENABLE_LIBCXX "Enables LLVM libc++" FALSE)
@@ -57,6 +58,10 @@ if (ENABLE_FEXCORE_PROFILER)
   endif()
 endif()
 
+if (ENABLE_JEMALLOC_GLIBC_ALLOC AND ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)
+  message(FATAL_ERROR "Can't have both glibc fault allocator and jemalloc glibc allocator enabled at the same time")
+endif()
+
 if (ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)
   add_definitions(-DGLIBC_ALLOCATOR_FAULT=1)
 endif()
@@ -190,7 +195,22 @@ if (ENABLE_TSAN)
   link_libraries(-fno-omit-frame-pointer -fsanitize=thread)
 endif()
 
+if (ENABLE_JEMALLOC_GLIBC_ALLOC)
+  # The glibc jemalloc subproject which hooks the glibc allocator.
+  # Required for thunks to work.
+  # All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator.
+  add_definitions(-DENABLE_JEMALLOC_GLIBC=1)
+  add_subdirectory(External/jemalloc_glibc/)
+else()
+  message (STATUS
+    " jemalloc glibc allocator disabled!\n"
+    " This is not a recommended configuration!\n"
+    " This will very explicitly break thunk execution!\n"
+    " Use at your own risk!")
+endif()
+
 if (ENABLE_JEMALLOC)
+  # The jemalloc subproject that all FEXCore fextl objects allocate through.
   add_definitions(-DENABLE_JEMALLOC=1)
   add_subdirectory(External/jemalloc/)
   include_directories(External/jemalloc/pregen/include/)

External/FEXCore/Source/CMakeLists.txt

@@ -233,6 +233,10 @@ if (ENABLE_JEMALLOC)
   list (APPEND LIBS FEX_jemalloc)
 endif()
 
+if (ENABLE_JEMALLOC_GLIBC_ALLOC)
+  list (APPEND LIBS FEX_jemalloc_glibc)
+endif()
+
 # Generate config
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/Interface/Config/Config.json.in

External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp

@@ -29,8 +29,14 @@ $end_info$
 #include <stdint.h>
 #include <utility>
 
-#ifdef ENABLE_JEMALLOC
-#include "jemalloc/jemalloc.h"
+#ifdef ENABLE_JEMALLOC_GLIBC
+extern "C" {
+// jemalloc defines nothrow on its internal C function signatures.
+#define JEMALLOC_NOTHROW __attribute__((nothrow))
+// Forward declare jemalloc functions because we can't include the headers from the glibc jemalloc project.
+// This is because we can't simultaneously set up include paths for both of our internal jemalloc modules.
+FEX_DEFAULT_VISIBILITY JEMALLOC_NOTHROW extern int glibc_je_is_known_allocation(void *ptr);
+}
 #endif
 
 struct LoadlibArgs {
@@ -251,13 +257,13 @@ namespace FEXCore {
          * and host heap pointers.
          */
         static void IsHostHeapAllocation(void* ArgsRV) {
-#ifdef ENABLE_JEMALLOC
+#ifdef ENABLE_JEMALLOC_GLIBC
             struct ArgsRV_t {
                 void* ptr;
                 bool rv;
             } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
 
-            args->rv = je_is_known_allocation(args->ptr);
+            args->rv = glibc_je_is_known_allocation(args->ptr);
 #else
             // Thunks usage without jemalloc isn't supported
             ERROR_AND_DIE_FMT("Unsupported: Thunks querying for host heap allocation information");

External/FEXCore/Source/Utils/Allocator.cpp

@@ -29,8 +29,8 @@ extern "C" {
   typedef int (*munmap_hook_type)(void *addr, size_t length);
 
 #ifdef ENABLE_JEMALLOC
-  extern mmap_hook_type __mmap_hook;
-  extern munmap_hook_type __munmap_hook;
+  extern mmap_hook_type je___mmap_hook;
+  extern munmap_hook_type je___munmap_hook;
 #endif
 }
 
@@ -122,8 +122,8 @@ namespace FEXCore::Allocator {
   void SetupHooks() {
     Alloc64 = Alloc::OSAllocator::Create64BitAllocator();
 #ifdef ENABLE_JEMALLOC
-    __mmap_hook   = FEX_mmap;
-    __munmap_hook = FEX_munmap;
+    je___mmap_hook   = FEX_mmap;
+    je___munmap_hook = FEX_munmap;
 #endif
     FEXCore::Allocator::mmap = FEX_mmap;
     FEXCore::Allocator::munmap = FEX_munmap;
@@ -131,8 +131,8 @@ namespace FEXCore::Allocator {
 
   void ClearHooks() {
 #ifdef ENABLE_JEMALLOC
-    __mmap_hook   = ::mmap;
-    __munmap_hook = ::munmap;
+    je___mmap_hook   = ::mmap;
+    je___munmap_hook = ::munmap;
 #endif
     FEXCore::Allocator::mmap = ::mmap;
     FEXCore::Allocator::munmap = ::munmap;

docs/jemalloc_usage.md

@@ -0,0 +1,35 @@
+# JEMalloc usage
+FEX-Emu uses two different jemalloc heap allocators at once, each for different purposes:
+- jemalloc: The primary heap allocator (to keep FEX's internal allocations out of the 32-bit address space used by guest applications)
+- jemalloc_glibc: The second heap allocator (to add allocation introspection features used by thunks)
+
+## jemalloc - primary heap allocator
+This allocator overrides `mmap` and `munmap` by forwarding them to FEXCore's internal VMA region allocator.
+
+All of FEXCore's `fextl::` namespaced objects allocate memory with this method.
+
+### FEXCore internal VMA region allocator
+When running a 32-bit guest application, the VMA region allocator allocates from memory *above* the 4GB of virtual address space reserved for the
+32-bit application.
+
+This ensures that all of FEX's allocations stay out of the lower 32-bit 4GB VA space, since games would quickly run out of virtual address space
+otherwise.
+
+When running a 64-bit guest application, this VMA region allocator is disabled and passes through to mmap and munmap in the host kernel.
+
+## jemalloc_glibc - secondary heap allocator
+This heap allocator replaces the host glibc's allocator using weak symbol overriding. It adds introspection features used for thunking, but has no
+functional differences otherwise: All memory is allocated in the 4GB of 32-bit address space. The FEXCore VMA region allocator is explicitly **not**
+involved hence.
+
+All native shared libraries use this allocator including the host-side of thunks.
+
+Internally, all allocations that go through this heap allocator use the kernel mmap and munmap interface.
+
+### Thunks
+Thunks may allocate memory either through the guest-side (on the guest glibc heap) or the host-side (on the `jemalloc_glibc` heap). To properly free
+this memory, FEX must be able to determine which heap allocator it belongs to.
+
+`glibc` provides no public interface to do this, but FEX's `jemalloc` fork does. The `is_known_allocation` function is used by FEXCore to query
+whether a given pointer originated from the `jemalloc_glibc` allocator. This enables FEXCore to determine the appropriate heap for freeing the
+pointer.