diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index 58311a374..468f92c95 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -112,14 +112,18 @@ void ReenableSBRKAllocations(void* Ptr) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -void SetupHooks() { - Alloc64 = Alloc::OSAllocator::Create64BitAllocator(); +static void AssignHookOverrides() { SetJemallocMmapHook(FEX_mmap); SetJemallocMunmapHook(FEX_munmap); FEXCore::Allocator::mmap = FEX_mmap; FEXCore::Allocator::munmap = FEX_munmap; } +void SetupHooks() { + Alloc64 = Alloc::OSAllocator::Create64BitAllocator(); + AssignHookOverrides(); +} + void ClearHooks() { SetJemallocMmapHook(::mmap); SetJemallocMunmapHook(::munmap); @@ -300,15 +304,18 @@ fextl::vector StealMemoryRegion(uintptr_t Begin, uintptr_t End) { return Regions; } -fextl::vector Steal48BitVA() { +void Setup48BitAllocatorIfExists() { size_t Bits = FEXCore::Allocator::DetermineVASize(); if (Bits < 48) { - return {}; + return; } uintptr_t Begin48BitVA = 0x0'8000'0000'0000ULL; uintptr_t End48BitVA = 0x1'0000'0000'0000ULL; - return StealMemoryRegion(Begin48BitVA, End48BitVA); + auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA); + + Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions); + AssignHookOverrides(); } void ReclaimMemoryRegion(const fextl::vector& Regions) { diff --git a/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp b/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp index 60d6545c0..a34071502 100644 --- a/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp +++ b/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -35,6 +37,8 @@ thread_local FEXCore::Core::InternalThreadState* TLSThread {}; class OSAllocator_64Bit final : public Alloc::HostAllocator { public: OSAllocator_64Bit(); + OSAllocator_64Bit(fextl::vector& Regions); + virtual ~OSAllocator_64Bit(); void* AllocateSlab(size_t Size) override { return nullptr; @@ -180,7 +184,7 @@ private: // 32-bit old kernel workarounds fextl::vector Steal32BitIfOldKernel(); - void AllocateMemoryRegions(const fextl::vector& Ranges); + void AllocateMemoryRegions(fextl::vector& Ranges); LiveVMARegion* FindLiveRegionForAddress(uintptr_t Addr, uintptr_t AddrEnd); }; @@ -515,29 +519,41 @@ fextl::vector OSAllocator_64Bit::Steal32BitIfO return FEXCore::Allocator::StealMemoryRegion(LOWER_BOUND_32, UPPER_BOUND_32); } -void OSAllocator_64Bit::AllocateMemoryRegions(const fextl::vector& Ranges) { - for (auto [Ptr, AllocationSize] : Ranges) { - if (!ObjectAlloc) { - auto MaxSize = std::min(size_t(64) * 1024 * 1024, AllocationSize); +void OSAllocator_64Bit::AllocateMemoryRegions(fextl::vector& Ranges) { + // Need to allocate the ObjectAlloc up front. Find a region that is larger than our minimum size first. + const size_t ObjectAllocSize = 64 * 1024 * 1024; - // Allocate up to 64 MiB the first allocation for an intrusive allocator - mprotect(Ptr, MaxSize, PROT_READ | PROT_WRITE); - - // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure - ::madvise(Ptr, MaxSize, MADV_HUGEPAGE); - - ObjectAlloc = new (Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(Ptr, MaxSize); - ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc); - LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc); - - if (AllocationSize > MaxSize) { - AllocationSize -= MaxSize; - (uint8_t*&)Ptr += MaxSize; - } else { - continue; - } + for (auto& it : Ranges) { + if (ObjectAllocSize > it.Size) { + continue; } + // Allocate up to 64 MiB the first allocation for an intrusive allocator + mprotect(it.Ptr, ObjectAllocSize, PROT_READ | PROT_WRITE); + + // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure + ::madvise(it.Ptr, ObjectAllocSize, MADV_HUGEPAGE); + + ObjectAlloc = new (it.Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(it.Ptr, ObjectAllocSize); + ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc); + LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc); + + if (it.Size >= ObjectAllocSize) { + // Modify region size + it.Size -= ObjectAllocSize; + (uint8_t*&)it.Ptr += ObjectAllocSize; + } + + break; + } + + if (!ObjectAlloc) { + ERROR_AND_DIE_FMT("Couldn't allocate object allocator!"); + } + + for (auto [Ptr, AllocationSize] : Ranges) { + // Skip size of zero if ObjectAllocSize matched the size of region. + if (AllocationSize == 0) continue; ReservedVMARegion* Region = ObjectAlloc->new_construct(); Region->Base = reinterpret_cast(Ptr); Region->RegionSize = AllocationSize; @@ -557,6 +573,10 @@ OSAllocator_64Bit::OSAllocator_64Bit() { FEXCore::Allocator::ReclaimMemoryRegion(LowMem); } +OSAllocator_64Bit::OSAllocator_64Bit(fextl::vector& Regions) { + AllocateMemoryRegions(Regions); +} + OSAllocator_64Bit::~OSAllocator_64Bit() { // This needs a mutex to be thread safe auto lk = FEXCore::GuardSignalDeferringSectionWithFallback(AllocationMutex, TLSThread); @@ -576,6 +596,61 @@ OSAllocator_64Bit::~OSAllocator_64Bit() { fextl::unique_ptr Create64BitAllocator() { return fextl::make_unique(); } + +template +struct alloc_delete : public std::default_delete { + void operator()(T* ptr) const { + if (ptr) { + const auto size = sizeof(T); + const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE); + + std::destroy_at(ptr); + ::munmap(ptr, MinPage); + } + } + + template + requires (std::is_base_of_v) + operator fextl::default_delete() { + return fextl::default_delete(); + } +}; + +template +requires (!std::is_array_v) +fextl::unique_ptr make_alloc_unique(FEXCore::Allocator::MemoryRegion& Base, Args&&... args) { + const auto size = sizeof(T); + const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE); + if (Base.Size < size || MinPage != FEXCore::Utils::FEX_PAGE_SIZE) { + ERROR_AND_DIE_FMT("Couldn't fit allocator in to page!"); + } + + // Remove the page from the base region. + // Could be zero after this. + Base.Size -= MinPage; + Base.Ptr = reinterpret_cast(reinterpret_cast(Base.Ptr) + MinPage); + + auto ptr = ::mmap(Base.Ptr, MinPage, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (ptr == MAP_FAILED) { + ERROR_AND_DIE_FMT("Couldn't allocate memory region"); + } + auto Result = ::new (ptr) T(std::forward(args)...); + return fextl::unique_ptr>(Result); +} + +fextl::unique_ptr Create64BitAllocatorWithRegions(fextl::vector& Regions) { + // This is a bit tricky as we can't allocate memory safely except from the Regions provided. Otherwise we might overwrite memory pages we + // don't own. Scan the memory regions and find the smallest one. + FEXCore::Allocator::MemoryRegion& Smallest = Regions[0]; + for (auto& it : Regions) { + if (it.Size <= Smallest.Size) { + Smallest = it; + } + } + + return make_alloc_unique(Smallest, Regions); +} + } // namespace Alloc::OSAllocator namespace FEXCore::Allocator { diff --git a/FEXCore/Source/Utils/Allocator/HostAllocator.h b/FEXCore/Source/Utils/Allocator/HostAllocator.h index 445722de7..a3e464892 100644 --- a/FEXCore/Source/Utils/Allocator/HostAllocator.h +++ b/FEXCore/Source/Utils/Allocator/HostAllocator.h @@ -2,9 +2,10 @@ #pragma once #include #include +#include +#include #include -#include #include namespace FEXCore::Core { @@ -49,4 +50,5 @@ public: namespace Alloc::OSAllocator { fextl::unique_ptr Create64BitAllocator(); +fextl::unique_ptr Create64BitAllocatorWithRegions(fextl::vector& Regions); } // namespace Alloc::OSAllocator diff --git a/FEXCore/include/FEXCore/Utils/Allocator.h b/FEXCore/include/FEXCore/Utils/Allocator.h index 795aff1aa..efbb70f7c 100644 --- a/FEXCore/include/FEXCore/Utils/Allocator.h +++ b/FEXCore/include/FEXCore/Utils/Allocator.h @@ -86,7 +86,7 @@ FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(const fextl::vector Steal48BitVA(); +FEX_DEFAULT_VISIBILITY void Setup48BitAllocatorIfExists(); #ifndef _WIN32 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread); diff --git a/Source/Tools/FEXLoader/FEXLoader.cpp b/Source/Tools/FEXLoader/FEXLoader.cpp index dd0aec576..48bc61808 100644 --- a/Source/Tools/FEXLoader/FEXLoader.cpp +++ b/Source/Tools/FEXLoader/FEXLoader.cpp @@ -491,12 +491,11 @@ int main(int argc, char** argv, char** const envp) { FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_IS64BIT_MODE, Loader.Is64BitMode() ? "1" : "0"); fextl::unique_ptr Allocator; - fextl::vector Base48Bit; fextl::vector Low4GB; if (Loader.Is64BitMode()) { // Destroy the 48th bit if it exists - Base48Bit = FEXCore::Allocator::Steal48BitVA(); + FEXCore::Allocator::Setup48BitAllocatorIfExists(); } else { // Reserve [0x1_0000_0000, 0x2_0000_0000). // Safety net if 32-bit address calculation overflows in to 64-bit range. @@ -685,7 +684,6 @@ int main(int argc, char** argv, char** const envp) { LogMan::Msg::UnInstallHandler(); FEXCore::Allocator::ClearHooks(); - FEXCore::Allocator::ReclaimMemoryRegion(Base48Bit); FEXCore::Allocator::ReclaimMemoryRegion(Low4GB); // Allocator is now original system allocator