From d3cfdcb431642e038536847aa09976d61ea0e217 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Thu, 12 Mar 2026 16:40:33 -0700 Subject: [PATCH] Win32: Enable support for virtual naming and THP control Allows WTF to work (mostly) with Wine by letting us VirtualName things, and also allows madvise control of THP, which significantly cuts back memory usage. This works around the problem of Wine not giving us control of this by using raw syscalls when wine is detected. Based on top of #5362 so the THP disable controls are in. --- FEXCore/Source/Interface/Core/LookupCache.cpp | 2 +- FEXCore/Source/Utils/Allocator.cpp | 18 +- FEXCore/Source/Utils/Allocator.h | 2 + FEXCore/include/FEXCore/Utils/Allocator.h | 3 - .../include/FEXCore/Utils/AllocatorHooks.h | 23 +- Source/Windows/ARM64EC/Module.cpp | 3 + Source/Windows/Common/Allocator.cpp | 243 ++++++++++++++++++ Source/Windows/Common/Allocator.h | 7 + Source/Windows/Common/CMakeLists.txt | 1 + Source/Windows/Common/CallRetStack.h | 5 + Source/Windows/WOW64/Module.cpp | 3 + 11 files changed, 298 insertions(+), 12 deletions(-) create mode 100644 Source/Windows/Common/Allocator.cpp create mode 100644 Source/Windows/Common/Allocator.h diff --git a/FEXCore/Source/Interface/Core/LookupCache.cpp b/FEXCore/Source/Interface/Core/LookupCache.cpp index bc2b62ba5..aaa0c2424 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.cpp +++ b/FEXCore/Source/Interface/Core/LookupCache.cpp @@ -42,7 +42,7 @@ LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX) LOGMAN_THROW_A_FMT(PagePointer != -1ULL, "Failed to allocate PagePointer"); // Disable THP on the Lookup cache. - FEXCore::Allocator::VirtualTHPControl(reinterpret_cast(PagePointer), TotalCacheSize, FEXCore::Allocator::THPControl::Disable); + FEXCore::Allocator::VirtualTHPControl(reinterpret_cast(PagePointer), TotalCacheSize, FEXCore::Allocator::THPControl::Disable); FEXCore::Allocator::VirtualName("FEXMem_Lookup", reinterpret_cast(PagePointer), ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE); diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index bbf9e09c1..5cad9c014 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT #include "Utils/Allocator/HostAllocator.h" +#include "Utils/Allocator.h" #include #include #include @@ -32,8 +33,8 @@ std::pmr::memory_resource* get_default_resource() { } } // namespace fextl::pmr -#ifndef _WIN32 namespace FEXCore::Allocator { +#ifndef _WIN32 MMAP_Hook mmap {::mmap}; MUNMAP_Hook munmap {::munmap}; @@ -304,5 +305,18 @@ void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child) { Alloc64->UnlockAfterFork(Thread, Child); } } -} // namespace FEXCore::Allocator +#else + +void VirtualNameNOP(const char*, const void*, size_t) {} +void VirtualTHPNOP(const void* Ptr, size_t Size, THPControl Control) {} + +VirtualNamePtr VirtualName {VirtualNameNOP}; +VirtualTHPPtr VirtualTHPControl {VirtualTHPNOP}; + +void SetupHooks(size_t PageSize, HookPtrs Ptrs) { + VirtualName = Ptrs.VirtualName; + VirtualTHPControl = Ptrs.VirtualTHPControl; +} + #endif +} // namespace FEXCore::Allocator diff --git a/FEXCore/Source/Utils/Allocator.h b/FEXCore/Source/Utils/Allocator.h index a30605a80..b006c670b 100644 --- a/FEXCore/Source/Utils/Allocator.h +++ b/FEXCore/Source/Utils/Allocator.h @@ -1,11 +1,13 @@ // SPDX-License-Identifier: MIT #pragma once +#include namespace FEXCore::Core { struct InternalThreadState; } namespace FEXCore::Allocator { +void InitializeAllocator(size_t PageSize); void LockBeforeFork(FEXCore::Core::InternalThreadState* Thread); void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child); } // namespace FEXCore::Allocator diff --git a/FEXCore/include/FEXCore/Utils/Allocator.h b/FEXCore/include/FEXCore/Utils/Allocator.h index 1fec1b15d..c9444251e 100644 --- a/FEXCore/include/FEXCore/Utils/Allocator.h +++ b/FEXCore/include/FEXCore/Utils/Allocator.h @@ -12,9 +12,6 @@ struct InternalThreadState; } namespace FEXCore::Allocator { -FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize); -FEX_DEFAULT_VISIBILITY void ClearHooks(); - FEX_DEFAULT_VISIBILITY size_t DetermineVASize(); #ifdef GLIBC_ALLOCATOR_FAULT diff --git a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h index 193d116ec..48bbe5e3e 100644 --- a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h +++ b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h @@ -32,6 +32,19 @@ enum class THPControl { Disable, }; +#ifndef _WIN32 +FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize); +#else +using VirtualNamePtr = void (*)(const char*, const void*, size_t); +using VirtualTHPPtr = void (*)(const void*, size_t, THPControl); +struct HookPtrs { + VirtualNamePtr VirtualName; + VirtualTHPPtr VirtualTHPControl; +}; +FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize, HookPtrs Ptrs); +#endif +FEX_DEFAULT_VISIBILITY void ClearHooks(); + #ifdef _WIN32 inline void* VirtualAlloc(void* Base, size_t Size, bool Execute = false, bool Commit = true) { // Allocate top-down to avoid polluting the lower VA space, as even on 64-bit some programs (i.e. LuaJIT) require allocations below 4GB. @@ -87,9 +100,8 @@ inline bool VirtualProtect(void* Ptr, size_t Size, ProtectOptions options) { return ::VirtualProtect(Ptr, Size, prot, nullptr) == 0; } -inline void VirtualName(const char*, void*, size_t) {} -inline void VirtualTHPControl(void* Ptr, size_t Size, THPControl Control) {} - +FEX_DEFAULT_VISIBILITY extern VirtualNamePtr VirtualName; +FEX_DEFAULT_VISIBILITY extern VirtualTHPPtr VirtualTHPControl; #else using MMAP_Hook = void* (*)(void*, size_t, int, int, int, off_t); using MUNMAP_Hook = int (*)(void*, size_t); @@ -129,8 +141,8 @@ inline bool VirtualProtect(void* Ptr, size_t Size, ProtectOptions options) { return ::mprotect(Ptr, Size, prot) == 0; } -inline void VirtualTHPControl(void* Ptr, size_t Size, THPControl Control) { - ::madvise(Ptr, Size, Control == THPControl::Enable ? MADV_HUGEPAGE : MADV_NOHUGEPAGE); +inline void VirtualTHPControl(const void* Ptr, size_t Size, THPControl Control) { + ::madvise(const_cast(Ptr), Size, Control == THPControl::Enable ? MADV_HUGEPAGE : MADV_NOHUGEPAGE); } #endif @@ -152,7 +164,6 @@ void aligned_free(void* ptr); FEX_DEFAULT_VISIBILITY extern void InitializeThread(); #ifndef _WIN32 -void InitializeAllocator(size_t PageSize); void SetupAllocatorHooks(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset), int (*)(void* addr, size_t length)); #endif diff --git a/Source/Windows/ARM64EC/Module.cpp b/Source/Windows/ARM64EC/Module.cpp index 2384efaff..28c25cfc7 100644 --- a/Source/Windows/ARM64EC/Module.cpp +++ b/Source/Windows/ARM64EC/Module.cpp @@ -27,6 +27,7 @@ $end_info$ #include #include +#include "Windows/Common/Allocator.h" #include "Common/CallRetStack.h" #include "Common/JITGuardPage.h" #include "Common/Config.h" @@ -593,6 +594,8 @@ NTSTATUS ProcessInit() { const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version"); OvercommitTracker.emplace(IsWine); + FEX::Windows::Allocator::SetupHooks(NtDll); + { auto HostFeatures = FEX::Windows::CPUFeatures::FetchHostFeatures(IsWine); CTX = FEXCore::Context::Context::CreateNewContext(HostFeatures); diff --git a/Source/Windows/Common/Allocator.cpp b/Source/Windows/Common/Allocator.cpp new file mode 100644 index 000000000..0643e062f --- /dev/null +++ b/Source/Windows/Common/Allocator.cpp @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: MIT +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FEX::Windows::Allocator { +#define PR_SET_VMA 0x53564d41 +#define PR_SET_VMA_ANON_NAME 0 + +#define MADV_HUGEPAGE 14 +#define MADV_NOHUGEPAGE 15 + +namespace Trampoline { + struct madvise_data { + const void* addr; + size_t size; + int advise; + }; + + struct prctl_data { + int op; + uint64_t attr; + const void* addr; + size_t size; + const char* name; + uint64_t ret; + }; + + __attribute__((naked)) uint64_t wine_prctl(prctl_data* d) { + asm volatile( + R"( + .globl wine_prctl_begin + wine_prctl_begin: + mov x19, x0; + mov x8, 167; // prctl + ldr x0, [x19]; // op + ldp x1, x2, [x19, %[attr_offset]]; // {attr, addr} + ldp x3, x4, [x19, %[size_offset]]; // {size, name} + svc #0; + str x0, [x19, %[ret_offset]]; + // Tell wine it was all groovy. + mov x0, 0; + ret; + + .globl wine_prctl_end + wine_prctl_end: + )" + : + : [attr_offset] "i"(offsetof(prctl_data, attr)), [size_offset] "i"(offsetof(prctl_data, size)), [ret_offset] "i"(offsetof(prctl_data, ret)) + : "memory"); + }; + + __attribute__((naked)) uint64_t wine_madvise(madvise_data* d) { + asm volatile(R"( + .globl wine_madvise_begin + wine_madvise_begin: + mov x8, 233; // madvise + ldr x2, [x0, %[advise_offset]]; // advise + ldp x0, x1, [x0]; // {addr, size} + svc #0; + // Tell wine it was all groovy. + mov x0, 0; + ret; + + .globl wine_madvise_end + wine_madvise_end: + )" ::[advise_offset] "i"(offsetof(madvise_data, advise)) + : "memory"); + } + + extern "C" uint64_t wine_madvise_begin; + extern "C" uint64_t wine_madvise_end; + + void* const wine_madvise_begin_loc = &wine_madvise_begin; + void* const wine_madvise_end_loc = &wine_madvise_end; + + extern "C" uint64_t wine_prctl_begin; + extern "C" uint64_t wine_prctl_end; + + void* const wine_prctl_begin_loc = &wine_prctl_begin; + void* const wine_prctl_end_loc = &wine_prctl_end; + + extern NTSTATUS(WINAPI* __wine_unix_call_dispatcher)(uint64_t, unsigned int, void*); + decltype(__wine_unix_call_dispatcher) WineUnixCall; + + enum unix_function_indexes { + INDEX_PRCTL = 0, + INDEX_MADVISE = 1, + INDEX_MAX, + }; + static std::array unix_functions {}; + + static uint64_t wine_prctl_trampoline(int op, uint64_t attr, const void* addr, size_t size, const char* name) { + prctl_data d { + .op = op, + .attr = attr, + .addr = addr, + .size = size, + .name = name, + }; + WineUnixCall(reinterpret_cast(unix_functions.data()), INDEX_PRCTL, &d); + return d.ret; + } + + static uint64_t wine_madvise_trampoline(const void* addr, size_t size, int advice) { + madvise_data d { + .addr = addr, + .size = size, + .advise = advice, + }; + WineUnixCall(reinterpret_cast(unix_functions.data()), INDEX_MADVISE, &d); + return 0; + } + + void VirtualName(const char* Name, const void* Ptr, size_t Size) { + static bool Supports {true}; + if (Supports) { + auto Result = wine_prctl_trampoline(PR_SET_VMA, PR_SET_VMA_ANON_NAME, Ptr, Size, Name); + if (Result != 0) { + // Disable any additional attempts. + Supports = false; + } + } + } + + void VirtualTHPControl(const void* Ptr, size_t Size, FEXCore::Allocator::THPControl Control) { + wine_madvise_trampoline(Ptr, Size, Control == FEXCore::Allocator::THPControl::Disable ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); + } +} // namespace Trampoline + +// This code path will eventually crash once Wine and the kernel implements `userspace syscall dispatch`. +// FEX will need to switch over to using WINE's unixlib syscall approach then. +// See `SetupHooks` for why unixlib doesn't work today. +namespace Illegal { + __attribute__((naked)) uint64_t prctl(int op, uint64_t attr, const void* addr, size_t size, const char* Name) { + asm volatile(R"( + mov x8, 167; // prctl + svc #0; + ret; + )" :: + : "memory"); + } + + __attribute__((naked)) uint64_t madvise(const void* addr, size_t size, int advice) { + asm volatile(R"( + mov x8, 233; // madvise + svc #0; + ret; + )" :: + : "memory"); + } + + void VirtualName(const char* Name, const void* Ptr, size_t Size) { + static bool Supports {true}; + if (Supports) { + auto Result = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, Ptr, Size, Name); + if (Result != 0) { + // Disable any additional attempts. + Supports = false; + } + } + } + + void VirtualTHPControl(const void* Ptr, size_t Size, FEXCore::Allocator::THPControl Control) { + madvise(Ptr, Size, Control == FEXCore::Allocator::THPControl::Disable ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); + } +} // namespace Illegal + +void SetupHooks(HMODULE ntdll) { + // If this symbol doesn't exist, then we aren't running under WINE. + const auto Sym = GetProcAddress(ntdll, "__wine_unix_call_dispatcher"); + + if (!Sym) { + return; + } + + FEXCore::Allocator::HookPtrs Ptrs {}; + + // Wine will soon require us to use unixlib for calling helper routines that use Linux syscalls. + // It needs this for `userspace syscall dispatch` to capture rogue applications doing raw Windows syscalls. + // If FEX doesn't use unixlib, then when WINE and a kernel implements this, then our "illegal" path will start crashing. + // + // This code currently conflicts with FEX's `Call Checker` since the latter captures uses of `unixlib`. + // We hence have to stick to the `Illegal::` functions until a workaround is implemented. + if constexpr (false) { + // NTSTATUS __wine_unix_call_dispatcher( unixlib_handle_t, unsigned int, void * ); + // - unixlib_handle_t is just an array of functions + // - uint32_t is just an index in to that + // - void* is the user provided pointer, gets loaded in to x0 in for the unix_function called. + // - Return value - SUCCESS or other error. + Trampoline::WineUnixCall = *reinterpret_cast(Sym); + + // This code must be copied over to allocated memory from top down allocations apparently. + auto Code = reinterpret_cast( + ::VirtualAlloc(nullptr, FEXCore::Utils::FEX_PAGE_SIZE, MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN, PAGE_EXECUTE_READWRITE)); + if (!Code) { + return; + } + + size_t CurrentOffset {}; + const size_t prctl_size = + reinterpret_cast(Trampoline::wine_prctl_end_loc) - reinterpret_cast(Trampoline::wine_prctl_begin_loc); + const size_t madvise_size = + reinterpret_cast(Trampoline::wine_madvise_end_loc) - reinterpret_cast(Trampoline::wine_madvise_begin_loc); + + // Copy prctl. + memcpy(Code + CurrentOffset, Trampoline::wine_prctl_begin_loc, prctl_size); + Trampoline::unix_functions[Trampoline::INDEX_PRCTL] = Code + CurrentOffset; + CurrentOffset += prctl_size; + + // Copy madvise. + memcpy(Code + CurrentOffset, Trampoline::wine_madvise_begin_loc, madvise_size); + Trampoline::unix_functions[Trampoline::INDEX_MADVISE] = Code + CurrentOffset; + + // Protect the page now. + FEXCore::Allocator::VirtualProtect(Code, FEXCore::Utils::FEX_PAGE_SIZE, + FEXCore::Allocator::ProtectOptions::Read | FEXCore::Allocator::ProtectOptions::Exec); + + Ptrs = { + .VirtualName = Trampoline::VirtualName, + .VirtualTHPControl = Trampoline::VirtualTHPControl, + }; + } else { + Ptrs = { + .VirtualName = Illegal::VirtualName, + .VirtualTHPControl = Illegal::VirtualTHPControl, + }; + } + + SYSTEM_INFO system_info {}; + GetSystemInfo(&system_info); + FEXCore::Allocator::SetupHooks(system_info.dwPageSize, Ptrs); +} +} // namespace FEX::Windows::Allocator diff --git a/Source/Windows/Common/Allocator.h b/Source/Windows/Common/Allocator.h new file mode 100644 index 000000000..082ebd1cf --- /dev/null +++ b/Source/Windows/Common/Allocator.h @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +#pragma once +#include + +namespace FEX::Windows::Allocator { +void SetupHooks(HMODULE ntdll); +} diff --git a/Source/Windows/Common/CMakeLists.txt b/Source/Windows/Common/CMakeLists.txt index 194b9767b..d7e2da9c7 100644 --- a/Source/Windows/Common/CMakeLists.txt +++ b/Source/Windows/Common/CMakeLists.txt @@ -7,6 +7,7 @@ target_compile_options(CommonWindowsRuntime PRIVATE -Wno-inconsistent-dllimport) target_include_directories(CommonWindowsRuntime PRIVATE "${CMAKE_SOURCE_DIR}/Source/Windows/include/") add_library(CommonWindows STATIC + Allocator.cpp CPUFeatures.cpp SHMStats.cpp InvalidationTracker.cpp diff --git a/Source/Windows/Common/CallRetStack.h b/Source/Windows/Common/CallRetStack.h index 1aa252608..d2ddee3e0 100644 --- a/Source/Windows/Common/CallRetStack.h +++ b/Source/Windows/Common/CallRetStack.h @@ -25,6 +25,11 @@ void InitializeThread(FEXCore::Core::InternalThreadState* Thread) { const void* CallRetStackAlloc = ::VirtualAlloc( nullptr, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE + 2 * FEXCore::Utils::FEX_PAGE_SIZE, MEM_RESERVE, PAGE_NOACCESS); + FEXCore::Allocator::VirtualName("FEXMem_CallRetStacks", CallRetStackAlloc, + FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE + 2 * FEXCore::Utils::FEX_PAGE_SIZE); + FEXCore::Allocator::VirtualTHPControl(CallRetStackAlloc, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE + 2 * FEXCore::Utils::FEX_PAGE_SIZE, + FEXCore::Allocator::THPControl::Disable); + Thread->CallRetStackBase = reinterpret_cast(reinterpret_cast(CallRetStackAlloc) + FEXCore::Utils::FEX_PAGE_SIZE); ::VirtualAlloc(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE, MEM_COMMIT, PAGE_READWRITE); diff --git a/Source/Windows/WOW64/Module.cpp b/Source/Windows/WOW64/Module.cpp index 8a9e69ef6..e4772ffea 100644 --- a/Source/Windows/WOW64/Module.cpp +++ b/Source/Windows/WOW64/Module.cpp @@ -28,6 +28,7 @@ $end_info$ #include #include +#include "Windows/Common/Allocator.h" #include "Common/CallRetStack.h" #include "Common/JITGuardPage.h" #include "Common/Config.h" @@ -524,6 +525,8 @@ void BTCpuProcessInit() { const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version"); OvercommitTracker.emplace(IsWine); + FEX::Windows::Allocator::SetupHooks(NtDll); + { auto HostFeatures = FEX::Windows::CPUFeatures::FetchHostFeatures(IsWine); // AVX is unsupported for WOW64