steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-07 23:00:19 +02:00
Code Issues Packages Projects Releases 67 Wiki Activity
Files
e80e2bdafec5d9f4114710d24184cfbe71141a7e
FEX-Emu--FEX/Source/Tests/LinuxSyscalls/LinuxAllocator.cpp
T
Ryan Houdek 26c3ebd6a7 Linux: Fixes 32-bit allocator range scanning 
32-bit allocations were scanning for available pages by shifting by the
length of allocation. This is incorrect since large allocations would
then only scan through the region in very large chunks. Especially if
MAP_32BIT was present. Instead scan by page size to ensure better fit.

This fixes X-Plane 11.

Also ensure we don't try to munmap a range unless the result is
MAP_FAILED, noticed we were trying to munmap ~0ULL
2022-06-12 17:53:12 -07:00

615 lines
18 KiB
C++
Raw Blame History

#include "Tests/LinuxSyscalls/LinuxAllocator.h"
#include "Tests/LinuxSyscalls/Syscalls.h"
#include <FEXCore/Utils/MathUtils.h>
#include <FEXHeaderUtils/Syscalls.h>
#include <FEXHeaderUtils/TypeDefines.h>
#include <bitset>
#include <map>
#include <linux/mman.h>
#include <unistd.h>
#include <sys/user.h>
#include <sys/mman.h>
#include <sys/shm.h>
#ifndef MREMAP_DONTUNMAP
#define MREMAP_DONTUNMAP 4
#endif
namespace FEX::HLE {
class MemAllocator32Bit final : public FEX::HLE::MemAllocator {
private:
static constexpr uint64_t BASE_KEY = 16;
const uint64_t TOP_KEY = 0xFFFF'F000ULL >> FHU::FEX_PAGE_SHIFT;
const uint64_t TOP_KEY32BIT = 0x7FFF'F000ULL >> FHU::FEX_PAGE_SHIFT;
public:
MemAllocator32Bit() {
// First 16 pages are taken by the Linux kernel
for (size_t i = 0; i < 16; ++i) {
MappedPages.set(i);
}
// Take the top page as well
MappedPages.set(TOP_KEY);
if (SearchDown) {
LastScanLocation = TOP_KEY;
LastKeyLocation = TOP_KEY;
LastKeyLocation32Bit = TOP_KEY32BIT;
FindPageRangePtr = &MemAllocator32Bit::FindPageRange_TopDown;
}
else {
LastScanLocation = BASE_KEY;
LastKeyLocation = BASE_KEY;
FindPageRangePtr = &MemAllocator32Bit::FindPageRange;
}
}
void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override;
int munmap(void *addr, size_t length) override;
void *mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) override;
uint64_t shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) override;
uint64_t shmdt(const void* shmaddr) override;
static constexpr bool SearchDown = true;
// PageAddr is a page already shifted to page index
// PagesLength is the number of pages
void SetUsedPages(uint64_t PageAddr, size_t PagesLength) {
// Set the range as mapped
for (size_t i = 0; i < PagesLength; ++i) {
MappedPages.set(PageAddr + i);
}
}
// PageAddr is a page already shifted to page index
// PagesLength is the number of pages
void SetFreePages(uint64_t PageAddr, size_t PagesLength) {
// Set the range as unused
for (size_t i = 0; i < PagesLength; ++i) {
MappedPages.reset(PageAddr + i);
}
}
private:
// Set that contains 4k mapped pages
// This is the full 32bit memory range
std::bitset<0x10'0000> MappedPages;
std::map<uint32_t, int> PageToShm{};
uint64_t LastScanLocation{};
uint64_t LastKeyLocation{};
uint64_t LastKeyLocation32Bit{};
std::mutex AllocMutex{};
uint64_t FindPageRange(uint64_t Start, size_t Pages) const;
uint64_t FindPageRange_TopDown(uint64_t Start, size_t Pages) const;
using FindHandler = uint64_t(MemAllocator32Bit::*)(uint64_t Start, size_t Pages) const;
FindHandler FindPageRangePtr{};
};
uint64_t MemAllocator32Bit::FindPageRange(uint64_t Start, size_t Pages) const {
// Linear range scan
while (Start != TOP_KEY) {
bool Free = true;
if ((Start + Pages) > TOP_KEY) {
return 0;
}
uint64_t Offset = 0;
for (; Offset < Pages; ++Offset) {
if (MappedPages.test(Start + Offset)) {
Free = false;
break;
}
}
if (Free) {
return Start;
}
Start += Offset + 1;
}
return 0;
}
uint64_t MemAllocator32Bit::FindPageRange_TopDown(uint64_t Start, size_t Pages) const {
// Linear range scan
while (Start >= BASE_KEY &&
Start <= TOP_KEY) {
bool Free = true;
uint64_t Offset = 0;
for (; Offset < Pages; ++Offset) {
if (MappedPages.test(Start - Offset)) {
Free = false;
break;
}
}
if (Free) {
return Start - Offset;
}
Start -= Offset + 1;
}
return 0;
}
void *MemAllocator32Bit::mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
std::scoped_lock<std::mutex> lk{AllocMutex};
size_t PagesLength = FEXCore::AlignUp(length, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
uintptr_t Addr = reinterpret_cast<uintptr_t>(addr);
uintptr_t PageAddr = Addr >> FHU::FEX_PAGE_SHIFT;
// Define MAP_FIXED_NOREPLACE ourselves to ensure we always parse this flag
constexpr int FEX_MAP_FIXED_NOREPLACE = 0x100000;
bool Fixed = ((flags & MAP_FIXED) ||
(flags & FEX_MAP_FIXED_NOREPLACE));
// Both Addr and length must be page aligned
if (Addr & ~FHU::FEX_PAGE_MASK) {
return reinterpret_cast<void*>(-EINVAL);
}
// If we do have an fd then offset must be page aligned
if (fd != -1 &&
offset & ~FHU::FEX_PAGE_MASK) {
return reinterpret_cast<void*>(-EINVAL);
}
if (Addr + length > std::numeric_limits<uint32_t>::max()) {
return reinterpret_cast<void*>(-EOVERFLOW);
}
// Check reserved range
if (Fixed && PageAddr < 16) {
return reinterpret_cast<void*>(-EINVAL);
}
if (!Fixed) {
// If we aren't mapping fixed the ignore the address input
Addr = 0;
PageAddr = 0;
}
bool Map32Bit = flags & FEX::HLE::X86_64_MAP_32BIT;
// Remove the MAP_32BIT flag if it exists now
flags &= ~FEX::HLE::X86_64_MAP_32BIT;
auto AllocateNoHint = [&]() -> void*{
bool Wrapped = false;
uint64_t BottomPage = Map32Bit && (LastScanLocation >= LastKeyLocation32Bit) ? LastKeyLocation32Bit : LastScanLocation;
restart:
{
// Linear range scan
uint64_t LowerPage = (this->*FindPageRangePtr)(BottomPage, PagesLength);
if (LowerPage == 0) {
// Try again but this time from the start
BottomPage = Map32Bit ? LastKeyLocation32Bit : LastKeyLocation;
LowerPage = (this->*FindPageRangePtr)(BottomPage, PagesLength);
}
uint64_t UpperPage = LowerPage + PagesLength;
if (LowerPage == 0) {
return reinterpret_cast<void*>(-ENOMEM);
}
{
// Try and map the range
void *MappedPtr = ::mmap(
reinterpret_cast<void*>(LowerPage << FHU::FEX_PAGE_SHIFT),
length,
prot,
flags | FEX_MAP_FIXED_NOREPLACE,
fd,
offset);
if (MappedPtr == MAP_FAILED &&
errno != EEXIST) {
return reinterpret_cast<void*>(-errno);
}
else if (MappedPtr == MAP_FAILED ||
MappedPtr >= reinterpret_cast<void*>(TOP_KEY << FHU::FEX_PAGE_SHIFT)) {
// Handles the case where MAP_FIXED_NOREPLACE failed with MAP_FAILED
// or if the host system's kernel isn't new enough then it returns the wrong pointer
if (MappedPtr != MAP_FAILED &&
MappedPtr >= reinterpret_cast<void*>(TOP_KEY << FHU::FEX_PAGE_SHIFT)) {
// Make sure to munmap this so we don't leak memory
::munmap(MappedPtr, length);
}
if (UpperPage == TOP_KEY) {
BottomPage = BASE_KEY;
Wrapped = true;
goto restart;
}
else if (Wrapped &&
LowerPage >= LastScanLocation) {
// We linear scanned the entire memory range. Give up
return (void*)(uintptr_t)-errno;
}
else {
// Try again
if (SearchDown) {
--BottomPage;
}
else {
++BottomPage;
}
goto restart;
}
}
else {
if (SearchDown) {
LastScanLocation = LowerPage;
}
else {
LastScanLocation = UpperPage;
}
SetUsedPages(LowerPage, PagesLength);
return MappedPtr;
}
}
}
};
// Find a region that fits our address
if (Addr == 0) {
return AllocateNoHint();
}
else {
void *MappedPtr = ::mmap(
reinterpret_cast<void*>(PageAddr << FHU::FEX_PAGE_SHIFT),
PagesLength << FHU::FEX_PAGE_SHIFT,
prot,
flags,
fd,
offset);
if (MappedPtr >= reinterpret_cast<void*>(TOP_KEY << FHU::FEX_PAGE_SHIFT) &&
(flags & FEX_MAP_FIXED_NOREPLACE)) {
// Handles the case where MAP_FIXED_NOREPLACE isn't handled by the host system's
// kernel and returns the wrong pointer
// Make sure to munmap this so we don't leak memory
::munmap(MappedPtr, length);
return reinterpret_cast<void*>(-EEXIST);
}
else if (MappedPtr != MAP_FAILED) {
SetUsedPages(PageAddr, PagesLength);
return MappedPtr;
}
else {
return reinterpret_cast<void*>(-errno);
}
}
return 0;
}
int MemAllocator32Bit::munmap(void *addr, size_t length) {
std::scoped_lock<std::mutex> lk{AllocMutex};
size_t PagesLength = FEXCore::AlignUp(length, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
uintptr_t Addr = reinterpret_cast<uintptr_t>(addr);
uintptr_t PageAddr = Addr >> FHU::FEX_PAGE_SHIFT;
uintptr_t PageEnd = PageAddr + PagesLength;
// Both Addr and length must be page aligned
if (Addr & ~FHU::FEX_PAGE_MASK) {
return -EINVAL;
}
if (length & ~FHU::FEX_PAGE_MASK) {
return -EINVAL;
}
if (Addr + length > std::numeric_limits<uint32_t>::max()) {
return -EOVERFLOW;
}
// Check reserved range
if (PageAddr < 16) {
// Return success for these
return 0;
}
while (PageAddr != PageEnd) {
// Always pass to munmap, it may be something allocated we aren't tracking
int Result = ::munmap(reinterpret_cast<void*>(PageAddr << FHU::FEX_PAGE_SHIFT), FHU::FEX_PAGE_SIZE);
if (Result != 0) {
return -errno;
}
if (MappedPages.test(PageAddr)) {
MappedPages.reset(PageAddr);
}
++PageAddr;
}
return 0;
}
void *MemAllocator32Bit::mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) {
size_t OldPagesLength = FEXCore::AlignUp(old_size, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
size_t NewPagesLength = FEXCore::AlignUp(new_size, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
{
std::scoped_lock<std::mutex> lk{AllocMutex};
if (flags & MREMAP_FIXED) {
void *MappedPtr = ::mremap(old_address, old_size, new_size, flags, new_address);
if (MappedPtr != MAP_FAILED) {
if (!(flags & MREMAP_DONTUNMAP)) {
// Unmap the old location
uintptr_t OldAddr = reinterpret_cast<uintptr_t>(old_address);
SetFreePages(OldAddr >> FHU::FEX_PAGE_SHIFT, OldPagesLength);
}
// Map the new pages
uintptr_t NewAddr = reinterpret_cast<uintptr_t>(MappedPtr);
SetUsedPages(NewAddr >> FHU::FEX_PAGE_SHIFT, NewPagesLength);
}
else {
return reinterpret_cast<void*>(-errno);
}
}
else {
uintptr_t OldAddr = reinterpret_cast<uintptr_t>(old_address);
uintptr_t OldPageAddr = OldAddr >> FHU::FEX_PAGE_SHIFT;
if (NewPagesLength < OldPagesLength) {
void *MappedPtr = ::mremap(old_address, old_size, new_size, flags & ~MREMAP_MAYMOVE);
if (MappedPtr != MAP_FAILED) {
// Clear the pages that we just shrunk
size_t NewPagesLength = FEXCore::AlignUp(new_size, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
uintptr_t NewPageAddr = reinterpret_cast<uintptr_t>(MappedPtr) >> FHU::FEX_PAGE_SHIFT;
SetFreePages(NewPageAddr + NewPagesLength, OldPagesLength - NewPagesLength);
return MappedPtr;
}
else {
return reinterpret_cast<void*>(-errno);
}
}
else {
// Scan the region forward from our first region's endd to see if it can be extended
bool CanExtend{true};
for (size_t i = OldPagesLength; i < NewPagesLength; ++i) {
if (MappedPages[OldPageAddr + i]) {
CanExtend = false;
break;
}
}
if (CanExtend) {
void *MappedPtr = ::mremap(old_address, old_size, new_size, flags & ~MREMAP_MAYMOVE);
if (MappedPtr != MAP_FAILED) {
// Map the new pages
size_t NewPagesLength = FEXCore::AlignUp(new_size, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
uintptr_t NewAddr = reinterpret_cast<uintptr_t>(MappedPtr);
SetUsedPages(NewAddr >> FHU::FEX_PAGE_SHIFT, NewPagesLength);
return MappedPtr;
}
else if (!(flags & MREMAP_MAYMOVE)) {
// We have one more chance if MAYMOVE is specified
return reinterpret_cast<void*>(-errno);
}
}
}
}
}
// Flags can not contain MREMAP_FIXED at this point
// Flags might contain MREMAP_MAYMOVE and/or MREMAP_DONTUNMAP
// New Size is >= old size
// First, try and allocate a region the size of the new size
void *MappedPtr = this->mmap(nullptr, new_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
std::scoped_lock<std::mutex> lk{AllocMutex};
if (FEX::HLE::HasSyscallError(MappedPtr)) {
// Couldn't find a region that fit our space
return MappedPtr;
}
// Good news, we found a region
// This will overwrite the previous mmap if it succeeds
MappedPtr = ::mremap(old_address, old_size, new_size, flags | MREMAP_FIXED | MREMAP_MAYMOVE, MappedPtr);
if (MappedPtr != MAP_FAILED) {
if (!(flags & MREMAP_DONTUNMAP) &&
MappedPtr != old_address) {
// If we have both MREMAP_DONTUNMAP not set and the new pointer is at a new location
// Make sure to clear the old mapping
uintptr_t OldAddr = reinterpret_cast<uintptr_t>(old_address);
SetFreePages(OldAddr >> FHU::FEX_PAGE_SHIFT , OldPagesLength);
}
// Map the new pages
size_t NewPagesLength = FEXCore::AlignUp(new_size, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
uintptr_t NewAddr = reinterpret_cast<uintptr_t>(MappedPtr);
SetUsedPages(NewAddr >> FHU::FEX_PAGE_SHIFT, NewPagesLength);
return MappedPtr;
}
// Failed
return reinterpret_cast<void*>(-errno);
}
uint64_t MemAllocator32Bit::shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) {
std::scoped_lock<std::mutex> lk{AllocMutex};
if (shmaddr != nullptr) {
// shmaddr must be valid
uint64_t Result = reinterpret_cast<uint64_t>(::shmat(shmid, shmaddr, shmflg));
if (Result != -1) {
uint32_t SmallRet = Result >> 32;
if (!(SmallRet == 0 ||
SmallRet == ~0U)) {
LOGMAN_MSG_A_FMT("Syscall returning something with data in the upper 32bits! BUG!");
return -ENOMEM;
}
uintptr_t NewAddr = reinterpret_cast<uintptr_t>(Result);
uintptr_t NewPageAddr = NewAddr >> FHU::FEX_PAGE_SHIFT;
// Add to the map
PageToShm[NewPageAddr] = shmid;
*ResultAddress = Result;
// We must get the shm size and track it
struct shmid_ds buf{};
if (shmctl(shmid, IPC_STAT, &buf) == 0) {
// Map the new pages
size_t NewPagesLength = buf.shm_segsz >> FHU::FEX_PAGE_SHIFT;
SetUsedPages(NewPageAddr, NewPagesLength);
}
// Zero on working result
Result = 0;
}
else {
Result = -errno;
}
return Result;
}
else {
// We must get the shm size and track it
struct shmid_ds buf{};
uint64_t PagesLength{};
if (shmctl(shmid, IPC_STAT, &buf) == 0) {
PagesLength = FEXCore::AlignUp(buf.shm_segsz, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
}
else {
return -EINVAL;
}
bool Wrapped = false;
uint64_t BottomPage = LastScanLocation;
restart:
{
// Linear range scan
uint64_t LowerPage = (this->*FindPageRangePtr)(BottomPage, PagesLength);
if (LowerPage == 0) {
// Try again but this time from the start
BottomPage = LastKeyLocation;
LowerPage = (this->*FindPageRangePtr)(BottomPage, PagesLength);
}
uint64_t UpperPage = LowerPage + PagesLength;
if (LowerPage == 0) {
return -ENOMEM;
}
{
// Try and map the range
void *MappedPtr = ::shmat(
shmid,
reinterpret_cast<const void*>(LowerPage << FHU::FEX_PAGE_SHIFT),
shmflg);
if (MappedPtr == MAP_FAILED) {
if (UpperPage == TOP_KEY) {
BottomPage = LastKeyLocation;
Wrapped = true;
goto restart;
}
else if (Wrapped &&
LowerPage >= LastScanLocation) {
// We linear scanned the entire memory range. Give up
return -errno;
}
else {
// Try again
BottomPage += PagesLength;
goto restart;
}
}
else {
if (SearchDown) {
LastScanLocation = LowerPage;
}
else {
LastScanLocation = UpperPage;
}
// Set the range as mapped
SetUsedPages(LowerPage, PagesLength);
*ResultAddress = reinterpret_cast<uint64_t>(MappedPtr);
// Add to the map
PageToShm[LowerPage] = shmid;
// Zero on working result
return 0;
}
}
}
}
}
uint64_t MemAllocator32Bit::shmdt(const void* shmaddr) {
std::scoped_lock<std::mutex> lk{AllocMutex};
uint32_t AddrPage = reinterpret_cast<uint64_t>(shmaddr) >> FHU::FEX_PAGE_SHIFT;
auto it = PageToShm.find(AddrPage);
if (it == PageToShm.end()) {
// Page wasn't mapped
return -EINVAL;
}
uint64_t Result = ::shmdt(shmaddr);
PageToShm.erase(it);
SYSCALL_ERRNO();
}
class MemAllocatorPassThrough final : public FEX::HLE::MemAllocator {
public:
void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override {
uint64_t Result = (uint64_t)::mmap(addr, length, prot, flags, fd, offset);
if (Result == ~0ULL) {
return reinterpret_cast<void*>(-errno);
}
return reinterpret_cast<void*>(Result);
}
int munmap(void *addr, size_t length) override {
uint64_t Result = (uint64_t)::munmap(addr, length);
SYSCALL_ERRNO();
}
void *mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) override {
uint64_t Result = (uint64_t)::mremap(old_address, old_size, new_size, flags, new_address);
if (Result == ~0ULL) {
return reinterpret_cast<void*>(-errno);
}
return reinterpret_cast<void*>(Result);
}
uint64_t shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) override {
uint64_t Result = (uint64_t)::shmat(shmid, reinterpret_cast<const void*>(shmaddr), shmflg);
if (Result != ~0ULL) {
*ResultAddress = Result;
Result = 0;
}
SYSCALL_ERRNO();
}
uint64_t shmdt(const void* shmaddr) override {
uint64_t Result = ::shmdt(shmaddr);
SYSCALL_ERRNO();
}
};
std::unique_ptr<FEX::HLE::MemAllocator> Create32BitAllocator() {
return std::make_unique<MemAllocator32Bit>();
}
std::unique_ptr<FEX::HLE::MemAllocator> CreatePassthroughAllocator() {
return std::make_unique<MemAllocatorPassThrough>();
}
}
Reference in new issue View Git Blame Copy Permalink