// SPDX-License-Identifier: MIT /* $info$ category: LinuxSyscalls ~ Linux syscall emulation, marshaling and passthrough tags: LinuxSyscalls|common desc: SMC/MMan Tracking $end_info$ */ #include "Common/FDUtils.h" #include #include #include #include #include "LinuxSyscalls/Syscalls.h" #include "LinuxSyscalls/SignalDelegator.h" #include #include #include #include #include namespace FEX::HLE { // SMC interactions bool SyscallHandler::HandleSegfault(FEXCore::Core::InternalThreadState* Thread, int Signal, void* info, void* ucontext) { const auto FaultAddress = (uintptr_t)((siginfo_t*)info)->si_addr; auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromFEXCoreThread(Thread); auto CallRetStackInfo = ThreadObject->GetCallRetStackInfo(); if (FaultAddress >= CallRetStackInfo.AllocationBase && FaultAddress < CallRetStackInfo.AllocationEnd) { // Reset REG_CALLRET_SP to the default location to allow for underflows/overflows ArchHelpers::Context::SetArmReg(ucontext, 25, CallRetStackInfo.DefaultLocation); return true; } { // Can't use the deferred signal lock in the SIGSEGV handler. auto lk = FEXCore::MaskSignalsAndLockMutex(_SyscallHandler->VMATracking.Mutex); auto VMATracking = &_SyscallHandler->VMATracking; // If the write spans two pages, they will be flushed one at a time (generating two faults) auto Entry = VMATracking->FindVMAEntry(FaultAddress); // If an untracked address, or the mapping wasn't writable, it can't be handled here if (Entry == VMATracking->VMAs.end() || !Entry->second.Prot.Writable) { return false; } auto FaultBase = FEXCore::AlignDown(FaultAddress, FEXCore::Utils::FEX_PAGE_SIZE); auto UnprotectRegionCallback = [](uintptr_t Start, uintptr_t Length) { auto rv = mprotect((void*)Start, Length, PROT_READ | PROT_WRITE); LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", Start, Length); }; if (Entry->second.Flags.Shared) { LOGMAN_THROW_A_FMT(Entry->second.Resource, "VMA tracking error"); auto Offset = FaultBase - Entry->first + Entry->second.Offset; auto VMA = Entry->second.Resource->FirstVMA; LOGMAN_THROW_A_FMT(VMA, "VMA tracking error"); // Flush all mirrors, remap the page writable as needed do { if (VMA->Offset <= Offset && (VMA->Offset + VMA->Length) > Offset) { auto FaultBaseMirrored = Offset - VMA->Offset + VMA->Base; if (VMA->Prot.Writable) { _SyscallHandler->TM.InvalidateGuestCodeRange(Thread, FaultBaseMirrored, FEXCore::Utils::FEX_PAGE_SIZE, UnprotectRegionCallback); } else { _SyscallHandler->TM.InvalidateGuestCodeRange(Thread, FaultBaseMirrored, FEXCore::Utils::FEX_PAGE_SIZE); } } } while ((VMA = VMA->ResourceNextVMA)); } else { _SyscallHandler->TM.InvalidateGuestCodeRange(Thread, FaultBase, FEXCore::Utils::FEX_PAGE_SIZE, UnprotectRegionCallback); } FEXCORE_PROFILE_INSTANT_INCREMENT(Thread, AccumulatedSMCCount, 1); auto CTX = Thread->CTX; if (CTX->IsAddressInCodeBuffer(Thread, ArchHelpers::Context::GetPc(ucontext)) && !CTX->IsCurrentBlockSingleInst(Thread) && CTX->IsAddressInCurrentBlock(Thread, FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE)) { // If we are not in a single-instruction block, and the SMC write address could intersect with the current block, // reconstruct the context and repeat the faulting instruction as a single-instruction block so any SMC it performs // is immediately picked up. ThreadObject->SignalInfo.Delegator->SpillSRA(Thread, ucontext, Thread->CurrentFrame->InSyscallInfo & 0xFFFF); // Adjust context to return to the dispatcher, reloading SRA from thread state const auto& Config = ThreadObject->SignalInfo.Delegator->GetConfig(); ArchHelpers::Context::SetPc(ucontext, Config.AbsoluteLoopTopAddressFillSRA); ArchHelpers::Context::SetArmReg(ucontext, 1, 1); // Set ENTRY_FILL_SRA_SINGLE_INST_REG to force a single step } return true; } } void SyscallHandler::MarkGuestExecutableRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) { const auto Base = Start & FEXCore::Utils::FEX_PAGE_MASK; const auto Top = FEXCore::AlignUp(Start + Length, FEXCore::Utils::FEX_PAGE_SIZE); { if (SMCChecks != FEXCore::Config::CONFIG_SMC_MTRACK) { return; } auto lk = FEXCore::GuardSignalDeferringSection(VMATracking.Mutex, Thread); // Find the first mapping at or after the range ends, or ::end(). // Top points to the address after the end of the range auto Mapping = VMATracking.VMAs.lower_bound(Top); while (Mapping != VMATracking.VMAs.begin()) { Mapping--; const auto MapBase = Mapping->first; const auto MapTop = MapBase + Mapping->second.Length; if (MapTop <= Base) { // Mapping ends before the Range start, exit break; } else { const auto ProtectBase = std::max(MapBase, Base); const auto ProtectSize = std::min(MapTop, Top) - ProtectBase; if (Mapping->second.Flags.Shared) { LOGMAN_THROW_A_FMT(Mapping->second.Resource, "VMA tracking error"); const auto OffsetBase = ProtectBase - Mapping->first + Mapping->second.Offset; const auto OffsetTop = OffsetBase + ProtectSize; auto VMA = Mapping->second.Resource->FirstVMA; LOGMAN_THROW_A_FMT(VMA, "VMA tracking error"); do { auto VMAOffsetBase = VMA->Offset; auto VMAOffsetTop = VMA->Offset + VMA->Length; auto VMABase = VMA->Base; if (VMA->Prot.Writable && VMAOffsetBase < OffsetTop && VMAOffsetTop > OffsetBase) { const auto MirroredBase = std::max(VMAOffsetBase, OffsetBase); const auto MirroredSize = std::min(OffsetTop, VMAOffsetTop) - MirroredBase; auto rv = mprotect((void*)(MirroredBase - VMAOffsetBase + VMABase), MirroredSize, PROT_READ); LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", MirroredBase, MirroredSize); } } while ((VMA = VMA->ResourceNextVMA)); } else if (Mapping->second.Prot.Writable) { int rv = mprotect((void*)ProtectBase, ProtectSize, PROT_READ); LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", ProtectBase, ProtectSize); } } } } } // Used for AOT FEXCore::HLE::AOTIRCacheEntryLookupResult SyscallHandler::LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestAddr) { auto lk = FEXCore::GuardSignalDeferringSection(VMATracking.Mutex, Thread); // Get the first mapping after GuestAddr, or end // GuestAddr is inclusive // If the write spans two pages, they will be flushed one at a time (generating two faults) auto Entry = VMATracking.FindVMAEntry(GuestAddr); if (Entry == VMATracking.VMAs.end()) { return {nullptr, 0}; } return {Entry->second.Resource ? Entry->second.Resource->AOTIRCacheEntry : nullptr, Entry->second.Base - Entry->second.Offset}; } FEXCore::HLE::ExecutableRangeInfo SyscallHandler::QueryGuestExecutableRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Address) { auto lk = FEXCore::GuardSignalDeferringSection(VMATracking.Mutex, Thread); auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromFEXCoreThread(Thread); auto Entry = VMATracking.FindVMAEntry(Address); if (Entry == VMATracking.VMAs.end() || (!Entry->second.Prot.Executable && (!(ThreadObject->persona & READ_IMPLIES_EXEC) || !Entry->second.Prot.Readable))) { return {0, 0, false}; } return {Entry->first, Entry->second.Length, Entry->second.Prot.Writable}; } void* SyscallHandler::GuestMmap(bool Is64Bit, FEXCore::Core::InternalThreadState* Thread, void* addr, size_t length, int prot, int flags, int fd, off_t offset) { LOGMAN_THROW_A_FMT(Is64Bit || (length >> 32) == 0, "values must fit to 32 bits"); uint64_t Result {}; size_t Size = FEXCore::AlignUp(length, FEXCore::Utils::FEX_PAGE_SIZE); if (flags & MAP_SHARED) { CTX->MarkMemoryShared(Thread); } { // NOTE: Frontend calls this with a nullptr Thread during initialization, but // providing this code with a valid Thread object earlier would allow // us to be more optimal by using GuardSignalDeferringSection instead auto lk = FEXCore::GuardSignalDeferringSectionWithFallback(VMATracking.Mutex, Thread); bool Map32Bit = !Is64Bit || (flags & FEX::HLE::X86_64_MAP_32BIT); if (Map32Bit) { Result = (uint64_t)Get32BitAllocator()->Mmap((void*)addr, length, prot, flags, fd, offset); if (FEX::HLE::HasSyscallError(Result)) { return reinterpret_cast(Result); } LOGMAN_THROW_A_FMT(Is64Bit || (Result >> 32) == 0 || (Result >> 32) == 0xFFFFFFFF, "values must fit to 32 bits"); } else { Result = reinterpret_cast(::mmap(reinterpret_cast(addr), length, prot, flags, fd, offset)); if (Result == ~0ULL) { return reinterpret_cast(-errno); } } FEX::HLE::_SyscallHandler->TrackMmap(Thread, Result, length, prot, flags, fd, offset); } FEX::HLE::_SyscallHandler->InvalidateCodeRangeIfNecessary(Thread, Result, Size); return reinterpret_cast(Result); } uint64_t SyscallHandler::GuestMunmap(bool Is64Bit, FEXCore::Core::InternalThreadState* Thread, void* addr, uint64_t length) { LOGMAN_THROW_A_FMT(Is64Bit || (reinterpret_cast(addr) >> 32) == 0, "values must fit to 32 bits: {}", fmt::ptr(addr)); LOGMAN_THROW_A_FMT(Is64Bit || (length >> 32) == 0, "values must fit to 32 bits"); uint64_t Result {}; uint64_t Size = FEXCore::AlignUp(length, FEXCore::Utils::FEX_PAGE_SIZE); { // Frontend calls this with nullptr Thread during initialization. // This is why `GuardSignalDeferringSectionWithFallback` is used here. // To be more optimal the frontend should provide this code with a valid Thread object earlier. auto lk = FEXCore::GuardSignalDeferringSectionWithFallback(VMATracking.Mutex, Thread); if (reinterpret_cast(addr) < 0x1'0000'0000ULL) { Result = FEX::HLE::_SyscallHandler->Get32BitAllocator()->Munmap(addr, length); if (FEX::HLE::HasSyscallError(Result)) { return Result; } } else { Result = ::munmap(addr, length); if (Result == -1) { return -errno; } } FEX::HLE::_SyscallHandler->TrackMunmap(Thread, addr, length); } FEX::HLE::_SyscallHandler->InvalidateCodeRangeIfNecessary(Thread, reinterpret_cast(addr), Size); return Result; } uint64_t SyscallHandler::GuestMremap(bool Is64Bit, FEXCore::Core::InternalThreadState* Thread, void* old_address, size_t old_size, size_t new_size, int flags, void* new_address) { uint64_t Result {}; { auto lk = FEXCore::GuardSignalDeferringSection(FEX::HLE::_SyscallHandler->VMATracking.Mutex, Thread); if (Is64Bit) { Result = reinterpret_cast(::mremap(old_address, old_size, new_size, flags, new_address)); if (Result == -1) { return -errno; } } else { Result = reinterpret_cast(FEX::HLE::_SyscallHandler->Get32BitAllocator()->Mremap(old_address, old_size, new_size, flags, new_address)); if (FEX::HLE::HasSyscallError(Result)) { return Result; } } FEX::HLE::_SyscallHandler->TrackMremap(Thread, reinterpret_cast(old_address), old_size, new_size, flags, Result); } FEX::HLE::_SyscallHandler->InvalidateCodeRangeIfNecessaryOnRemap(Thread, reinterpret_cast(old_address), Result, old_size, new_size); return Result; } uint64_t SyscallHandler::GuestMprotect(FEXCore::Core::InternalThreadState* Thread, void* addr, size_t len, int prot) { uint64_t Result {}; { auto lk = FEXCore::GuardSignalDeferringSection(FEX::HLE::_SyscallHandler->VMATracking.Mutex, Thread); Result = ::mprotect(addr, len, prot); if (Result == -1) { return -errno; } FEX::HLE::_SyscallHandler->TrackMprotect(Thread, addr, len, prot); } FEX::HLE::_SyscallHandler->InvalidateCodeRangeIfNecessary(Thread, reinterpret_cast(addr), len); return Result; } uint64_t SyscallHandler::GuestShmat(bool Is64Bit, FEXCore::Core::InternalThreadState* Thread, int shmid, const void* shmaddr, int shmflg) { auto CTX = Thread->CTX; uint64_t Result {}; uint64_t Length {}; CTX->MarkMemoryShared(Thread); { auto lk = FEXCore::GuardSignalDeferringSection(FEX::HLE::_SyscallHandler->VMATracking.Mutex, Thread); if (Is64Bit) { Result = reinterpret_cast(::shmat(shmid, shmaddr, shmflg)); if (Result == -1) { return -errno; } } else { uint32_t Addr; Result = FEX::HLE::_SyscallHandler->Get32BitAllocator()->Shmat(shmid, shmaddr, shmflg, &Addr); if (FEX::HLE::HasSyscallError(Result)) { return Result; } Result = Addr; } shmid_ds stat; [[maybe_unused]] auto res = shmctl(shmid, IPC_STAT, &stat); LOGMAN_THROW_A_FMT(res != -1, "shmctl IPC_STAT failed"); Length = stat.shm_segsz; FEX::HLE::_SyscallHandler->TrackShmat(Thread, shmid, Result, shmflg, Length); } FEX::HLE::_SyscallHandler->InvalidateCodeRangeIfNecessary(Thread, Result, Length); return Result; } uint64_t SyscallHandler::GuestShmdt(bool Is64Bit, FEXCore::Core::InternalThreadState* Thread, const void* shmaddr) { uint64_t Result {}; uint64_t Length {}; { auto lk = FEXCore::GuardSignalDeferringSection(FEX::HLE::_SyscallHandler->VMATracking.Mutex, Thread); if (Is64Bit) { Result = ::shmdt(shmaddr); if (Result == -1) { return -errno; } } else { Result = FEX::HLE::_SyscallHandler->Get32BitAllocator()->Shmdt(shmaddr); if (FEX::HLE::HasSyscallError(Result)) { return Result; } } Length = FEX::HLE::_SyscallHandler->TrackShmdt(Thread, reinterpret_cast(shmaddr)); } FEX::HLE::_SyscallHandler->InvalidateCodeRangeIfNecessary(Thread, reinterpret_cast(shmaddr), Length); return Result; } // MMan Tracking void SyscallHandler::TrackMmap(FEXCore::Core::InternalThreadState* Thread, uint64_t addr, size_t length, int prot, int flags, int fd, off_t offset) { size_t Size = FEXCore::AlignUp(length, FEXCore::Utils::FEX_PAGE_SIZE); VMATracking::MappedResource* Resource = nullptr; if (!(flags & MAP_ANONYMOUS)) { struct stat64 buf; fstat64(fd, &buf); VMATracking::MRID mrid {buf.st_dev, buf.st_ino}; char Tmp[PATH_MAX]; auto PathLength = FEX::get_fdpath(fd, Tmp); if (PathLength != -1) { Tmp[PathLength] = '\0'; auto [Iter, Inserted] = VMATracking.EmplaceMappedResource(mrid, VMATracking::MappedResource {nullptr, nullptr, 0}); Resource = &Iter->second; if (Inserted) { Resource->AOTIRCacheEntry = CTX->LoadAOTIRCacheEntry(fextl::string(Tmp, PathLength)); Resource->Iterator = Iter; } } } else if (flags & MAP_SHARED) { VMATracking::MRID mrid {VMATracking::SpecialDev::Anon, AnonSharedId++}; auto [Iter, Inserted] = VMATracking.EmplaceMappedResource(mrid, VMATracking::MappedResource {nullptr, nullptr, 0}); LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error"); Resource = &Iter->second; Resource->Iterator = Iter; } else { Resource = nullptr; } VMATracking.TrackVMARange(CTX, Resource, addr, offset, Size, VMATracking::VMAFlags::fromFlags(flags), VMATracking::VMAProt::fromProt(prot)); } void SyscallHandler::TrackMunmap(FEXCore::Core::InternalThreadState* Thread, void* addr, size_t length) { uint64_t Size = FEXCore::AlignUp(length, FEXCore::Utils::FEX_PAGE_SIZE); VMATracking.DeleteVMARange(CTX, reinterpret_cast(addr), Size); } void SyscallHandler::TrackMprotect(FEXCore::Core::InternalThreadState* Thread, void* addr, size_t len, int prot) { uint64_t Size = FEXCore::AlignUp(len, FEXCore::Utils::FEX_PAGE_SIZE); VMATracking.ChangeProtectionFlags(reinterpret_cast(addr), Size, VMATracking::VMAProt::fromProt(prot)); } void SyscallHandler::TrackMremap(FEXCore::Core::InternalThreadState* Thread, uint64_t OldAddress, size_t OldSize, size_t NewSize, int flags, uint64_t NewAddress) { OldSize = FEXCore::AlignUp(OldSize, FEXCore::Utils::FEX_PAGE_SIZE); NewSize = FEXCore::AlignUp(NewSize, FEXCore::Utils::FEX_PAGE_SIZE); const auto OldVMA = VMATracking.FindVMAEntry(OldAddress); const auto OldResource = OldVMA->second.Resource; const auto OldOffset = OldVMA->second.Offset + OldAddress - OldVMA->first; const auto OldFlags = OldVMA->second.Flags; const auto OldProt = OldVMA->second.Prot; LOGMAN_THROW_A_FMT(OldVMA != VMATracking.VMAs.end(), "VMA Tracking corruption"); if (OldSize == 0) { // Mirror existing mapping // must be a shared mapping LOGMAN_THROW_A_FMT(OldResource != nullptr, "VMA Tracking error"); LOGMAN_THROW_A_FMT(OldFlags.Shared, "VMA Tracking error"); VMATracking.TrackVMARange(CTX, OldResource, NewAddress, OldOffset, NewSize, OldFlags, OldProt); } else { #ifndef MREMAP_DONTUNMAP // MREMAP_DONTUNMAP is kernel 5.7+ and might not exist #define MREMAP_DONTUNMAP 4 #endif if (!(flags & MREMAP_DONTUNMAP)) { VMATracking.DeleteVMARange(CTX, OldAddress, OldSize, OldResource); } // Make anonymous mapping VMATracking.TrackVMARange(CTX, OldResource, NewAddress, OldOffset, NewSize, OldFlags, OldProt); } } void SyscallHandler::TrackShmat(FEXCore::Core::InternalThreadState* Thread, int shmid, uint64_t shmaddr, int shmflg, uint64_t Length) { VMATracking::MRID mrid {VMATracking::SpecialDev::SHM, static_cast(shmid)}; auto [Iter, Inserted] = VMATracking.EmplaceMappedResource(mrid, VMATracking::MappedResource {nullptr, nullptr, Length}); auto Resource = &Iter->second; if (Inserted) { Resource->Iterator = Iter; } VMATracking.TrackVMARange(CTX, Resource, shmaddr, 0, Length, VMATracking::VMAFlags::fromFlags(MAP_SHARED), VMATracking::VMAProt::fromSHM(shmflg)); } uint64_t SyscallHandler::TrackShmdt(FEXCore::Core::InternalThreadState* Thread, uint64_t shmaddr) { return VMATracking.DeleteSHMRegion(CTX, reinterpret_cast(shmaddr)); } void SyscallHandler::TrackMadvise(FEXCore::Core::InternalThreadState* Thread, uintptr_t Base, uintptr_t Size, int advice) { Size = FEXCore::AlignUp(Size, FEXCore::Utils::FEX_PAGE_SIZE); { auto lk = FEXCore::GuardSignalDeferringSection(VMATracking.Mutex, Thread); // TODO } } } // namespace FEX::HLE