diff --git a/FEXCore/Source/Interface/Core/LookupCache.cpp b/FEXCore/Source/Interface/Core/LookupCache.cpp index bc2b62ba5..aaa0c2424 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.cpp +++ b/FEXCore/Source/Interface/Core/LookupCache.cpp @@ -42,7 +42,7 @@ LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX) LOGMAN_THROW_A_FMT(PagePointer != -1ULL, "Failed to allocate PagePointer"); // Disable THP on the Lookup cache. - FEXCore::Allocator::VirtualTHPControl(reinterpret_cast(PagePointer), TotalCacheSize, FEXCore::Allocator::THPControl::Disable); + FEXCore::Allocator::VirtualTHPControl(reinterpret_cast(PagePointer), TotalCacheSize, FEXCore::Allocator::THPControl::Disable); FEXCore::Allocator::VirtualName("FEXMem_Lookup", reinterpret_cast(PagePointer), ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE); diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index bbf9e09c1..5cad9c014 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT #include "Utils/Allocator/HostAllocator.h" +#include "Utils/Allocator.h" #include #include #include @@ -32,8 +33,8 @@ std::pmr::memory_resource* get_default_resource() { } } // namespace fextl::pmr -#ifndef _WIN32 namespace FEXCore::Allocator { +#ifndef _WIN32 MMAP_Hook mmap {::mmap}; MUNMAP_Hook munmap {::munmap}; @@ -304,5 +305,18 @@ void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child) { Alloc64->UnlockAfterFork(Thread, Child); } } -} // namespace FEXCore::Allocator +#else + +void VirtualNameNOP(const char*, const void*, size_t) {} +void VirtualTHPNOP(const void* Ptr, size_t Size, THPControl Control) {} + +VirtualNamePtr VirtualName {VirtualNameNOP}; +VirtualTHPPtr VirtualTHPControl {VirtualTHPNOP}; + +void SetupHooks(size_t PageSize, HookPtrs Ptrs) { + VirtualName = Ptrs.VirtualName; + VirtualTHPControl = Ptrs.VirtualTHPControl; +} + #endif +} // namespace FEXCore::Allocator diff --git a/FEXCore/Source/Utils/Allocator.h b/FEXCore/Source/Utils/Allocator.h index a30605a80..b006c670b 100644 --- a/FEXCore/Source/Utils/Allocator.h +++ b/FEXCore/Source/Utils/Allocator.h @@ -1,11 +1,13 @@ // SPDX-License-Identifier: MIT #pragma once +#include namespace FEXCore::Core { struct InternalThreadState; } namespace FEXCore::Allocator { +void InitializeAllocator(size_t PageSize); void LockBeforeFork(FEXCore::Core::InternalThreadState* Thread); void UnlockAfterFork(FEXCore::Core::InternalThreadState* Thread, bool Child); } // namespace FEXCore::Allocator diff --git a/FEXCore/include/FEXCore/Utils/Allocator.h b/FEXCore/include/FEXCore/Utils/Allocator.h index 1fec1b15d..c9444251e 100644 --- a/FEXCore/include/FEXCore/Utils/Allocator.h +++ b/FEXCore/include/FEXCore/Utils/Allocator.h @@ -12,9 +12,6 @@ struct InternalThreadState; } namespace FEXCore::Allocator { -FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize); -FEX_DEFAULT_VISIBILITY void ClearHooks(); - FEX_DEFAULT_VISIBILITY size_t DetermineVASize(); #ifdef GLIBC_ALLOCATOR_FAULT diff --git a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h index 193d116ec..48bbe5e3e 100644 --- a/FEXCore/include/FEXCore/Utils/AllocatorHooks.h +++ b/FEXCore/include/FEXCore/Utils/AllocatorHooks.h @@ -32,6 +32,19 @@ enum class THPControl { Disable, }; +#ifndef _WIN32 +FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize); +#else +using VirtualNamePtr = void (*)(const char*, const void*, size_t); +using VirtualTHPPtr = void (*)(const void*, size_t, THPControl); +struct HookPtrs { + VirtualNamePtr VirtualName; + VirtualTHPPtr VirtualTHPControl; +}; +FEX_DEFAULT_VISIBILITY void SetupHooks(size_t PageSize, HookPtrs Ptrs); +#endif +FEX_DEFAULT_VISIBILITY void ClearHooks(); + #ifdef _WIN32 inline void* VirtualAlloc(void* Base, size_t Size, bool Execute = false, bool Commit = true) { // Allocate top-down to avoid polluting the lower VA space, as even on 64-bit some programs (i.e. LuaJIT) require allocations below 4GB. @@ -87,9 +100,8 @@ inline bool VirtualProtect(void* Ptr, size_t Size, ProtectOptions options) { return ::VirtualProtect(Ptr, Size, prot, nullptr) == 0; } -inline void VirtualName(const char*, void*, size_t) {} -inline void VirtualTHPControl(void* Ptr, size_t Size, THPControl Control) {} - +FEX_DEFAULT_VISIBILITY extern VirtualNamePtr VirtualName; +FEX_DEFAULT_VISIBILITY extern VirtualTHPPtr VirtualTHPControl; #else using MMAP_Hook = void* (*)(void*, size_t, int, int, int, off_t); using MUNMAP_Hook = int (*)(void*, size_t); @@ -129,8 +141,8 @@ inline bool VirtualProtect(void* Ptr, size_t Size, ProtectOptions options) { return ::mprotect(Ptr, Size, prot) == 0; } -inline void VirtualTHPControl(void* Ptr, size_t Size, THPControl Control) { - ::madvise(Ptr, Size, Control == THPControl::Enable ? MADV_HUGEPAGE : MADV_NOHUGEPAGE); +inline void VirtualTHPControl(const void* Ptr, size_t Size, THPControl Control) { + ::madvise(const_cast(Ptr), Size, Control == THPControl::Enable ? MADV_HUGEPAGE : MADV_NOHUGEPAGE); } #endif @@ -152,7 +164,6 @@ void aligned_free(void* ptr); FEX_DEFAULT_VISIBILITY extern void InitializeThread(); #ifndef _WIN32 -void InitializeAllocator(size_t PageSize); void SetupAllocatorHooks(void* (*)(void* addr, size_t length, int prot, int flags, int fd, off_t offset), int (*)(void* addr, size_t length)); #endif diff --git a/Source/Windows/ARM64EC/Module.cpp b/Source/Windows/ARM64EC/Module.cpp index 2384efaff..28c25cfc7 100644 --- a/Source/Windows/ARM64EC/Module.cpp +++ b/Source/Windows/ARM64EC/Module.cpp @@ -27,6 +27,7 @@ $end_info$ #include #include +#include "Windows/Common/Allocator.h" #include "Common/CallRetStack.h" #include "Common/JITGuardPage.h" #include "Common/Config.h" @@ -593,6 +594,8 @@ NTSTATUS ProcessInit() { const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version"); OvercommitTracker.emplace(IsWine); + FEX::Windows::Allocator::SetupHooks(NtDll); + { auto HostFeatures = FEX::Windows::CPUFeatures::FetchHostFeatures(IsWine); CTX = FEXCore::Context::Context::CreateNewContext(HostFeatures); diff --git a/Source/Windows/Common/Allocator.cpp b/Source/Windows/Common/Allocator.cpp new file mode 100644 index 000000000..0643e062f --- /dev/null +++ b/Source/Windows/Common/Allocator.cpp @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: MIT +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FEX::Windows::Allocator { +#define PR_SET_VMA 0x53564d41 +#define PR_SET_VMA_ANON_NAME 0 + +#define MADV_HUGEPAGE 14 +#define MADV_NOHUGEPAGE 15 + +namespace Trampoline { + struct madvise_data { + const void* addr; + size_t size; + int advise; + }; + + struct prctl_data { + int op; + uint64_t attr; + const void* addr; + size_t size; + const char* name; + uint64_t ret; + }; + + __attribute__((naked)) uint64_t wine_prctl(prctl_data* d) { + asm volatile( + R"( + .globl wine_prctl_begin + wine_prctl_begin: + mov x19, x0; + mov x8, 167; // prctl + ldr x0, [x19]; // op + ldp x1, x2, [x19, %[attr_offset]]; // {attr, addr} + ldp x3, x4, [x19, %[size_offset]]; // {size, name} + svc #0; + str x0, [x19, %[ret_offset]]; + // Tell wine it was all groovy. + mov x0, 0; + ret; + + .globl wine_prctl_end + wine_prctl_end: + )" + : + : [attr_offset] "i"(offsetof(prctl_data, attr)), [size_offset] "i"(offsetof(prctl_data, size)), [ret_offset] "i"(offsetof(prctl_data, ret)) + : "memory"); + }; + + __attribute__((naked)) uint64_t wine_madvise(madvise_data* d) { + asm volatile(R"( + .globl wine_madvise_begin + wine_madvise_begin: + mov x8, 233; // madvise + ldr x2, [x0, %[advise_offset]]; // advise + ldp x0, x1, [x0]; // {addr, size} + svc #0; + // Tell wine it was all groovy. + mov x0, 0; + ret; + + .globl wine_madvise_end + wine_madvise_end: + )" ::[advise_offset] "i"(offsetof(madvise_data, advise)) + : "memory"); + } + + extern "C" uint64_t wine_madvise_begin; + extern "C" uint64_t wine_madvise_end; + + void* const wine_madvise_begin_loc = &wine_madvise_begin; + void* const wine_madvise_end_loc = &wine_madvise_end; + + extern "C" uint64_t wine_prctl_begin; + extern "C" uint64_t wine_prctl_end; + + void* const wine_prctl_begin_loc = &wine_prctl_begin; + void* const wine_prctl_end_loc = &wine_prctl_end; + + extern NTSTATUS(WINAPI* __wine_unix_call_dispatcher)(uint64_t, unsigned int, void*); + decltype(__wine_unix_call_dispatcher) WineUnixCall; + + enum unix_function_indexes { + INDEX_PRCTL = 0, + INDEX_MADVISE = 1, + INDEX_MAX, + }; + static std::array unix_functions {}; + + static uint64_t wine_prctl_trampoline(int op, uint64_t attr, const void* addr, size_t size, const char* name) { + prctl_data d { + .op = op, + .attr = attr, + .addr = addr, + .size = size, + .name = name, + }; + WineUnixCall(reinterpret_cast(unix_functions.data()), INDEX_PRCTL, &d); + return d.ret; + } + + static uint64_t wine_madvise_trampoline(const void* addr, size_t size, int advice) { + madvise_data d { + .addr = addr, + .size = size, + .advise = advice, + }; + WineUnixCall(reinterpret_cast(unix_functions.data()), INDEX_MADVISE, &d); + return 0; + } + + void VirtualName(const char* Name, const void* Ptr, size_t Size) { + static bool Supports {true}; + if (Supports) { + auto Result = wine_prctl_trampoline(PR_SET_VMA, PR_SET_VMA_ANON_NAME, Ptr, Size, Name); + if (Result != 0) { + // Disable any additional attempts. + Supports = false; + } + } + } + + void VirtualTHPControl(const void* Ptr, size_t Size, FEXCore::Allocator::THPControl Control) { + wine_madvise_trampoline(Ptr, Size, Control == FEXCore::Allocator::THPControl::Disable ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); + } +} // namespace Trampoline + +// This code path will eventually crash once Wine and the kernel implements `userspace syscall dispatch`. +// FEX will need to switch over to using WINE's unixlib syscall approach then. +// See `SetupHooks` for why unixlib doesn't work today. +namespace Illegal { + __attribute__((naked)) uint64_t prctl(int op, uint64_t attr, const void* addr, size_t size, const char* Name) { + asm volatile(R"( + mov x8, 167; // prctl + svc #0; + ret; + )" :: + : "memory"); + } + + __attribute__((naked)) uint64_t madvise(const void* addr, size_t size, int advice) { + asm volatile(R"( + mov x8, 233; // madvise + svc #0; + ret; + )" :: + : "memory"); + } + + void VirtualName(const char* Name, const void* Ptr, size_t Size) { + static bool Supports {true}; + if (Supports) { + auto Result = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, Ptr, Size, Name); + if (Result != 0) { + // Disable any additional attempts. + Supports = false; + } + } + } + + void VirtualTHPControl(const void* Ptr, size_t Size, FEXCore::Allocator::THPControl Control) { + madvise(Ptr, Size, Control == FEXCore::Allocator::THPControl::Disable ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); + } +} // namespace Illegal + +void SetupHooks(HMODULE ntdll) { + // If this symbol doesn't exist, then we aren't running under WINE. + const auto Sym = GetProcAddress(ntdll, "__wine_unix_call_dispatcher"); + + if (!Sym) { + return; + } + + FEXCore::Allocator::HookPtrs Ptrs {}; + + // Wine will soon require us to use unixlib for calling helper routines that use Linux syscalls. + // It needs this for `userspace syscall dispatch` to capture rogue applications doing raw Windows syscalls. + // If FEX doesn't use unixlib, then when WINE and a kernel implements this, then our "illegal" path will start crashing. + // + // This code currently conflicts with FEX's `Call Checker` since the latter captures uses of `unixlib`. + // We hence have to stick to the `Illegal::` functions until a workaround is implemented. + if constexpr (false) { + // NTSTATUS __wine_unix_call_dispatcher( unixlib_handle_t, unsigned int, void * ); + // - unixlib_handle_t is just an array of functions + // - uint32_t is just an index in to that + // - void* is the user provided pointer, gets loaded in to x0 in for the unix_function called. + // - Return value - SUCCESS or other error. + Trampoline::WineUnixCall = *reinterpret_cast(Sym); + + // This code must be copied over to allocated memory from top down allocations apparently. + auto Code = reinterpret_cast( + ::VirtualAlloc(nullptr, FEXCore::Utils::FEX_PAGE_SIZE, MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN, PAGE_EXECUTE_READWRITE)); + if (!Code) { + return; + } + + size_t CurrentOffset {}; + const size_t prctl_size = + reinterpret_cast(Trampoline::wine_prctl_end_loc) - reinterpret_cast(Trampoline::wine_prctl_begin_loc); + const size_t madvise_size = + reinterpret_cast(Trampoline::wine_madvise_end_loc) - reinterpret_cast(Trampoline::wine_madvise_begin_loc); + + // Copy prctl. + memcpy(Code + CurrentOffset, Trampoline::wine_prctl_begin_loc, prctl_size); + Trampoline::unix_functions[Trampoline::INDEX_PRCTL] = Code + CurrentOffset; + CurrentOffset += prctl_size; + + // Copy madvise. + memcpy(Code + CurrentOffset, Trampoline::wine_madvise_begin_loc, madvise_size); + Trampoline::unix_functions[Trampoline::INDEX_MADVISE] = Code + CurrentOffset; + + // Protect the page now. + FEXCore::Allocator::VirtualProtect(Code, FEXCore::Utils::FEX_PAGE_SIZE, + FEXCore::Allocator::ProtectOptions::Read | FEXCore::Allocator::ProtectOptions::Exec); + + Ptrs = { + .VirtualName = Trampoline::VirtualName, + .VirtualTHPControl = Trampoline::VirtualTHPControl, + }; + } else { + Ptrs = { + .VirtualName = Illegal::VirtualName, + .VirtualTHPControl = Illegal::VirtualTHPControl, + }; + } + + SYSTEM_INFO system_info {}; + GetSystemInfo(&system_info); + FEXCore::Allocator::SetupHooks(system_info.dwPageSize, Ptrs); +} +} // namespace FEX::Windows::Allocator diff --git a/Source/Windows/Common/Allocator.h b/Source/Windows/Common/Allocator.h new file mode 100644 index 000000000..082ebd1cf --- /dev/null +++ b/Source/Windows/Common/Allocator.h @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +#pragma once +#include + +namespace FEX::Windows::Allocator { +void SetupHooks(HMODULE ntdll); +} diff --git a/Source/Windows/Common/CMakeLists.txt b/Source/Windows/Common/CMakeLists.txt index 194b9767b..d7e2da9c7 100644 --- a/Source/Windows/Common/CMakeLists.txt +++ b/Source/Windows/Common/CMakeLists.txt @@ -7,6 +7,7 @@ target_compile_options(CommonWindowsRuntime PRIVATE -Wno-inconsistent-dllimport) target_include_directories(CommonWindowsRuntime PRIVATE "${CMAKE_SOURCE_DIR}/Source/Windows/include/") add_library(CommonWindows STATIC + Allocator.cpp CPUFeatures.cpp SHMStats.cpp InvalidationTracker.cpp diff --git a/Source/Windows/Common/CallRetStack.h b/Source/Windows/Common/CallRetStack.h index 1aa252608..d2ddee3e0 100644 --- a/Source/Windows/Common/CallRetStack.h +++ b/Source/Windows/Common/CallRetStack.h @@ -25,6 +25,11 @@ void InitializeThread(FEXCore::Core::InternalThreadState* Thread) { const void* CallRetStackAlloc = ::VirtualAlloc( nullptr, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE + 2 * FEXCore::Utils::FEX_PAGE_SIZE, MEM_RESERVE, PAGE_NOACCESS); + FEXCore::Allocator::VirtualName("FEXMem_CallRetStacks", CallRetStackAlloc, + FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE + 2 * FEXCore::Utils::FEX_PAGE_SIZE); + FEXCore::Allocator::VirtualTHPControl(CallRetStackAlloc, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE + 2 * FEXCore::Utils::FEX_PAGE_SIZE, + FEXCore::Allocator::THPControl::Disable); + Thread->CallRetStackBase = reinterpret_cast(reinterpret_cast(CallRetStackAlloc) + FEXCore::Utils::FEX_PAGE_SIZE); ::VirtualAlloc(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE, MEM_COMMIT, PAGE_READWRITE); diff --git a/Source/Windows/WOW64/Module.cpp b/Source/Windows/WOW64/Module.cpp index 8a9e69ef6..e4772ffea 100644 --- a/Source/Windows/WOW64/Module.cpp +++ b/Source/Windows/WOW64/Module.cpp @@ -28,6 +28,7 @@ $end_info$ #include #include +#include "Windows/Common/Allocator.h" #include "Common/CallRetStack.h" #include "Common/JITGuardPage.h" #include "Common/Config.h" @@ -524,6 +525,8 @@ void BTCpuProcessInit() { const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version"); OvercommitTracker.emplace(IsWine); + FEX::Windows::Allocator::SetupHooks(NtDll); + { auto HostFeatures = FEX::Windows::CPUFeatures::FetchHostFeatures(IsWine); // AVX is unsupported for WOW64