Add in jemalloc glibc hooking again

We still need to hook glibc for thunks to work with
`IsHostHeapAllocation`.
So now we link in two jemalloc allocators in different namespaces.

As usual we have multiple heap allocators that we need to be careful about.

1. jemalloc with `je_` namespace.
  - This is FEX's regular heap allocator and what gets used for all the
    fextl objects.
  - This allocator is the one that the FEX mmap/munmap hooks hook in to
     - This mmap hooking gives this allocator the full 48-bit VA even in
       32-bit space.

2. jemalloc with `glibc_je_` namespace.
  - This is the allocator that overrides the glibc allocator routines
  - This is the allocator that thunks will use.
  - This is what `IsHostHeapAllocation` will check for.

3. Guest glibc allocator
  - We don't touch this one. But it is distinct from the host side
    allocators.
  - The guest side of thunks will use this heap allocator.

4. Host glibc allocator
  - #2 replaces this one unless explicitly disabled.
  - Always expected to override the allocator, so this configuration
    isn't expected.

Already tested this with Dota Underlords to ensure this works with
thunks.

.gitmodules

@@ -53,3 +53,6 @@
 	shallow = true
 	path = External/Vulkan-Headers
 	url = https://github.com/KhronosGroup/Vulkan-Headers.git
+[submodule "External/jemalloc_glibc"]
+	path = External/jemalloc_glibc
+	url = https://github.com/FEX-Emu/jemalloc.git

CMakeLists.txt

@@ -22,6 +22,7 @@ option(ENABLE_VISUAL_DEBUGGER "Enables the visual debugger for compiling" FALSE)
 option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE)
 option(ENABLE_WERROR "Enables -Werror" FALSE)
 option(ENABLE_JEMALLOC "Enables jemalloc allocator" TRUE)
+option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enables jemalloc glibc allocator" TRUE)
 option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE)
 option(ENABLE_COMPILE_TIME_TRACE "Enables time trace compile option" FALSE)
 option(ENABLE_LIBCXX "Enables LLVM libc++" FALSE)
@@ -57,6 +58,10 @@ if (ENABLE_FEXCORE_PROFILER)
   endif()
 endif()
 
+if (ENABLE_JEMALLOC_GLIBC_ALLOC AND ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)
+  message(FATAL_ERROR "Can't have both glibc fault allocator and jemalloc glibc allocator enabled at the same time")
+endif()
+
 if (ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)
   add_definitions(-DGLIBC_ALLOCATOR_FAULT=1)
 endif()
@@ -190,7 +195,22 @@ if (ENABLE_TSAN)
   link_libraries(-fno-omit-frame-pointer -fsanitize=thread)
 endif()
 
+if (ENABLE_JEMALLOC_GLIBC_ALLOC)
+  # The glibc jemalloc subproject which hooks the glibc allocator.
+  # Required for thunks to work.
+  # All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator.
+  add_definitions(-DENABLE_JEMALLOC_GLIBC=1)
+  add_subdirectory(External/jemalloc_glibc/)
+else()
+  message (STATUS
+    " jemalloc glibc allocator disabled!\n"
+    " This is not a recommended configuration!\n"
+    " This will very explicitly break thunk execution!\n"
+    " Use at your own risk!")
+endif()
+
 if (ENABLE_JEMALLOC)
+  # The jemalloc subproject that all FEXCore fextl objects allocate through.
   add_definitions(-DENABLE_JEMALLOC=1)
   add_subdirectory(External/jemalloc/)
   include_directories(External/jemalloc/pregen/include/)

External/FEXCore/Source/CMakeLists.txt

@@ -233,6 +233,10 @@ if (ENABLE_JEMALLOC)
   list (APPEND LIBS FEX_jemalloc)
 endif()
 
+if (ENABLE_JEMALLOC_GLIBC_ALLOC)
+  list (APPEND LIBS FEX_jemalloc_glibc)
+endif()
+
 # Generate config
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/Interface/Config/Config.json.in

External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp

@@ -29,8 +29,14 @@ $end_info$
 #include <stdint.h>
 #include <utility>
 
-#ifdef ENABLE_JEMALLOC
-#include "jemalloc/jemalloc.h"
+#ifdef ENABLE_JEMALLOC_GLIBC
+extern "C" {
+// jemalloc defines nothrow on its internal C function signatures.
+#define JEMALLOC_NOTHROW __attribute__((nothrow))
+// Forward declare jemalloc functions because we can't include the headers from the glibc jemalloc project.
+// This is because we can't simultaneously set up include paths for both of our internal jemalloc modules.
+FEX_DEFAULT_VISIBILITY JEMALLOC_NOTHROW extern int glibc_je_is_known_allocation(void *ptr);
+}
 #endif
 
 struct LoadlibArgs {
@@ -251,13 +257,13 @@ namespace FEXCore {
          * and host heap pointers.
          */
         static void IsHostHeapAllocation(void* ArgsRV) {
-#ifdef ENABLE_JEMALLOC
+#ifdef ENABLE_JEMALLOC_GLIBC
             struct ArgsRV_t {
                 void* ptr;
                 bool rv;
             } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
 
-            args->rv = je_is_known_allocation(args->ptr);
+            args->rv = glibc_je_is_known_allocation(args->ptr);
 #else
             // Thunks usage without jemalloc isn't supported
             ERROR_AND_DIE_FMT("Unsupported: Thunks querying for host heap allocation information");