From a284adcd193286dbdc2b5d9e52c702abc7fdb3bd Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Misis Poiitidis Date: Mon, 2 May 2022 01:21:53 +0300 Subject: [PATCH] SMC: Add mprotect based tracking, --smc=mtrack, make default --- .../Source/Interface/Config/Config.json.in | 9 +- .../Source/Interface/Context/Context.cpp | 10 +- .../Source/Interface/Context/Context.h | 4 +- .../FEXCore/Source/Interface/Core/Core.cpp | 25 +- .../Source/Interface/Core/LookupCache.h | 8 +- .../FEXCore/Source/Interface/IR/AOTIR.cpp | 121 +++--- External/FEXCore/Source/Interface/IR/AOTIR.h | 29 +- .../FEXCore/include/FEXCore/Config/Config.h | 7 +- .../FEXCore/include/FEXCore/Core/Context.h | 9 +- .../include/FEXCore/HLE/SyscallHandler.h | 26 ++ Source/Tests/IRLoader.cpp | 21 + Source/Tests/LinuxSyscalls/CMakeLists.txt | 2 + Source/Tests/LinuxSyscalls/Syscalls.cpp | 63 +-- Source/Tests/LinuxSyscalls/Syscalls.h | 115 ++++++ .../Tests/LinuxSyscalls/Syscalls/Thread.cpp | 12 +- .../LinuxSyscalls/SyscallsSMCTracking.cpp | 354 +++++++++++++++++ .../LinuxSyscalls/SyscallsVMATracking.cpp | 366 ++++++++++++++++++ Source/Tools/FEXConfig/Main.cpp | 7 +- 18 files changed, 1018 insertions(+), 170 deletions(-) create mode 100644 Source/Tests/LinuxSyscalls/SyscallsSMCTracking.cpp create mode 100644 Source/Tests/LinuxSyscalls/SyscallsVMATracking.cpp diff --git a/External/FEXCore/Source/Interface/Config/Config.json.in b/External/FEXCore/Source/Interface/Config/Config.json.in index 24593ee0a..f31025c76 100644 --- a/External/FEXCore/Source/Interface/Config/Config.json.in +++ b/External/FEXCore/Source/Interface/Config/Config.json.in @@ -234,14 +234,15 @@ "Hacks": { "SMCChecks": { "Type": "uint8", - "Default": "FEXCore::Config::CONFIG_SMC_MMAN", - "TextDefault": "mman", + "Default": "FEXCore::Config::CONFIG_SMC_MTRACK", + "TextDefault": "mtrack", "ArgumentHandler": "SMCCheckHandler", "Desc": [ "Checks code for modification before execution.", "\tnone: No checks", - "\tmman: Invalidate on mmap, mprotect, munmap", - "\tfull: Validate code before every run (slow)" + "\tmtrack: Page tracking based invalidation", + "\tfull: Validate code before every run (slow)", + "\tmman: Invalidate on mmap, mprotect, munmap (deprecated, use mtrack)" ] }, "TSOEnabled": { diff --git a/External/FEXCore/Source/Interface/Context/Context.cpp b/External/FEXCore/Source/Interface/Context/Context.cpp index 49b172098..66c0e6826 100644 --- a/External/FEXCore/Source/Interface/Context/Context.cpp +++ b/External/FEXCore/Source/Interface/Context/Context.cpp @@ -149,7 +149,7 @@ namespace FEXCore::Context { void CleanupAfterFork(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) { CTX->CleanupAfterFork(Thread); } - + void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation) { CTX->SignalDelegation = SignalDelegation; } @@ -186,11 +186,11 @@ namespace FEXCore::Context { CTX->WriteFilesWithCode(Writer); } - void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name) { - return CTX->AddNamedRegion(Base, Length, Offset, Name); + IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(FEXCore::Context::Context *CTX, const std::string &Name) { + return CTX->LoadAOTIRCacheEntry(Name); } - void RemoveNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length) { - return CTX->RemoveNamedRegion(Base, Length); + void UnloadAOTIRCacheEntry(FEXCore::Context::Context *CTX, IR::AOTIRCacheEntry *Entry) { + return CTX->UnloadAOTIRCacheEntry(Entry); } namespace Debug { diff --git a/External/FEXCore/Source/Interface/Context/Context.h b/External/FEXCore/Source/Interface/Context/Context.h index 3a9adc029..75917d4d6 100644 --- a/External/FEXCore/Source/Interface/Context/Context.h +++ b/External/FEXCore/Source/Interface/Context/Context.h @@ -266,8 +266,8 @@ namespace FEXCore::Context { uint8_t GetGPRSize() const { return Config.Is64BitMode ? 8 : 4; } - void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename); - void RemoveNamedRegion(uintptr_t Base, uintptr_t Size); + IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(const std::string &filename); + void UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry); FEXCore::JITSymbols Symbols; diff --git a/External/FEXCore/Source/Interface/Core/Core.cpp b/External/FEXCore/Source/Interface/Core/Core.cpp index b7c9783fb..f691ac2f5 100644 --- a/External/FEXCore/Source/Interface/Core/Core.cpp +++ b/External/FEXCore/Source/Interface/Core/Core.cpp @@ -352,6 +352,9 @@ namespace FEXCore::Context { // Walk the threads and tell them to clear their caches // Useful when our block size is set to a large number and we need to step a single instruction for (auto &Thread : Threads) { + // Wait for thread to be fully constructed + // XXX: Look into thread partial construction issues + while(Thread->RunningEvents.WaitingToStart.load()) ; ClearCodeCache(Thread, true); } } @@ -632,7 +635,10 @@ namespace FEXCore::Context { } void Context::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr, uint64_t Start, uint64_t Length) { - Thread->LookupCache->AddBlockMapping(Address, Ptr, Start, Length); + // Only call MarkGuestExecutableRange if new pages are marked as containing code + if (Thread->LookupCache->AddBlockMapping(Address, Ptr, Start, Length)) { + Thread->CTX->SyscallHandler->MarkGuestExecutableRange(Start, Length); + } } void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache) { @@ -1177,26 +1183,19 @@ namespace FEXCore::Context { return Result; } - void Context::AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename) { - IRCaptureCache.AddNamedRegion(Base, Size, Offset, filename); + IR::AOTIRCacheEntry *Context::LoadAOTIRCacheEntry(const std::string &filename) { + auto rv = IRCaptureCache.LoadAOTIRCacheEntry(filename); if (DebugServer) { DebugServer->AlertLibrariesChanged(); } - - if (CodeObjectCacheService) { - CodeObjectCacheService->AsyncAddNamedRegionJob(Base, Size, Offset, filename); - } + return rv; } - void Context::RemoveNamedRegion(uintptr_t Base, uintptr_t Size) { - IRCaptureCache.RemoveNamedRegion(Base, Size); + void Context::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry) { + IRCaptureCache.UnloadAOTIRCacheEntry(Entry); if (DebugServer) { DebugServer->AlertLibrariesChanged(); } - - if (CodeObjectCacheService) { - CodeObjectCacheService->AsyncRemoveNamedRegionJob(Base, Size); - } } void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set *ExternalBranches, uint64_t SectionMaxAddress) { diff --git a/External/FEXCore/Source/Interface/Core/LookupCache.h b/External/FEXCore/Source/Interface/Core/LookupCache.h index 04ebaa3ee..56b04a5ba 100644 --- a/External/FEXCore/Source/Interface/Core/LookupCache.h +++ b/External/FEXCore/Source/Interface/Core/LookupCache.h @@ -72,7 +72,8 @@ public: std::map> CodePages; - void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) { + // Returns true if new pages are marked as containing code + bool AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) { std::lock_guard lk(WriteLock); #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED @@ -81,7 +82,10 @@ public: BlockList.emplace(Address, (uintptr_t)HostCode); LOGMAN_THROW_A_FMT(InsertPoint.second == true, "Dupplicate block mapping added"); + bool rv = false; + for (auto CurrentPage = Start >> 12, EndPage = (Start + Length) >> 12; CurrentPage <= EndPage; CurrentPage++) { + rv |= CodePages[CurrentPage].size() == 0; CodePages[CurrentPage].push_back(Address); } @@ -90,6 +94,8 @@ public: auto &L1Entry = reinterpret_cast(L1Pointer)[Address & L1_ENTRIES_MASK]; L1Entry.GuestCode = Address; L1Entry.HostCode = (uintptr_t)HostCode; + + return rv; } void Erase(uint64_t Address) { diff --git a/External/FEXCore/Source/Interface/IR/AOTIR.cpp b/External/FEXCore/Source/Interface/IR/AOTIR.cpp index b8a7b0d77..ebc0550c1 100644 --- a/External/FEXCore/Source/Interface/IR/AOTIR.cpp +++ b/External/FEXCore/Source/Interface/IR/AOTIR.cpp @@ -1,10 +1,11 @@ #include "Interface/Context/Context.h" #include "Interface/IR/AOTIR.h" -#include "Interface/Core/LookupCache.h" #include #include #include +#include +#include #include #include @@ -78,7 +79,7 @@ namespace FEXCore::IR { return true; } - bool LoadAOTIRCache(AOTCacheType *AOTIRCache, int streamfd) { + static bool LoadAOTIRCache(AOTIRCacheEntry *Entry, int streamfd) { uint64_t tag; if (!readAll(streamfd, (char*)&tag, sizeof(tag)) || tag != FEXCore::IR::AOTIR_COOKIE) @@ -100,6 +101,10 @@ namespace FEXCore::IR { if (!readAll(streamfd, (char*)&Module[0], Module.size())) return false; + if (Entry->FileId != Module) { + return false; + } + lseek(streamfd, -sizeof(ModSize) - ModSize - sizeof(IndexSize), SEEK_END); if (!readAll(streamfd, (char*)&IndexSize, sizeof(IndexSize))) @@ -120,19 +125,16 @@ namespace FEXCore::IR { auto Array = (AOTIRInlineIndex *)((char*)FilePtr + IndexOffset); - AOTIRCache->insert({Module, {Array, FilePtr, Size}}); + LOGMAN_THROW_A_FMT(Entry->Array == nullptr && Entry->FilePtr == nullptr, "Entry must not be initialized here"); + Entry->Array = Array; + Entry->FilePtr = FilePtr; + Entry->Size = Size; LogMan::Msg::DFmt("AOTIR: Module {} has {} functions", Module, Array->Count); return true; } - AOTIRCaptureCache::~AOTIRCaptureCache() { - for (auto &Mod: AOTIRCache) { - FEXCore::Allocator::munmap(Mod.second.mapping, Mod.second.size); - } - } - void AOTIRCaptureCache::FinalizeAOTIRCache() { AOTIRCaptureCacheWriteoutQueue_Flush(); @@ -231,39 +233,27 @@ namespace FEXCore::IR { void AOTIRCaptureCache::WriteFilesWithCode(std::function Writer) { std::shared_lock lk(AOTIRCacheLock); - for( const auto &File: FilesWithCode) { - Writer(File.first, File.second); + for( const auto &Entry: AOTIRCache) { + if (Entry.second.ContainsCode) { + Writer(Entry.second.FileId, Entry.second.Filename); + } } } AOTIRCaptureCache::PreGenerateIRFetchResult AOTIRCaptureCache::PreGenerateIRFetch(uint64_t GuestRIP, FEXCore::IR::IRListView *IRList) { - { - std::shared_lock lk(AOTIRCacheLock); - auto file = AddrToFile.lower_bound(GuestRIP); - if (file != AddrToFile.begin()) { - --file; - if (!file->second.ContainsCode) { - file->second.ContainsCode = true; - FilesWithCode[file->second.fileid] = file->second.filename; - } - } - } - + auto AOTIRCacheEntry = CTX->SyscallHandler->LookupAOTIRCacheEntry(GuestRIP); + PreGenerateIRFetchResult Result{}; - if (IRList == nullptr && CTX->Config.AOTIRLoad()) { - std::shared_lock lk(AOTIRCacheLock); - auto file = AddrToFile.lower_bound(GuestRIP); - if (file != AddrToFile.begin()) { - --file; - auto Mod = (FEXCore::IR::AOTIRInlineIndex*)file->second.CachedFileEntry; + + if (AOTIRCacheEntry.Entry) { + AOTIRCacheEntry.Entry->ContainsCode = true; - if (Mod == nullptr) { - file->second.CachedFileEntry = Mod = AOTIRCache[file->second.fileid].Array; - } + if (IRList == nullptr && CTX->Config.AOTIRLoad()) { + auto Mod = AOTIRCacheEntry.Entry->Array; if (Mod != nullptr) { - auto AOTEntry = Mod->Find(GuestRIP - file->second.Start + file->second.Offset); + auto AOTEntry = Mod->Find(GuestRIP - AOTIRCacheEntry.Offset); if (AOTEntry) { // verify hash @@ -303,14 +293,12 @@ namespace FEXCore::IR { bool GeneratedIR) { // Both generated ir and LibraryJITName need a named region lookup if (GeneratedIR || CTX->Config.LibraryJITNaming()) { - std::shared_lock lk(AOTIRCacheLock); - auto file = FindAddrForFile(StartAddr, Length); + auto AOTIRCacheEntry = CTX->SyscallHandler->LookupAOTIRCacheEntry(GuestRIP); - // Only go down this path if we actually found a library region - if (file != AddrToFile.end()) { + if (AOTIRCacheEntry.Entry) { if (DebugData && CTX->Config.LibraryJITNaming()) { - CTX->Symbols.RegisterNamedRegion(CodePtr, DebugData->HostCodeSize, file->second.filename); + CTX->Symbols.RegisterNamedRegion(CodePtr, DebugData->HostCodeSize, AOTIRCacheEntry.Entry->Filename); } // Add to AOT cache if aot generation is enabled @@ -319,16 +307,20 @@ namespace FEXCore::IR { auto hash = XXH3_64bits((void*)StartAddr, Length); - auto LocalRIP = GuestRIP - file->second.Start + file->second.Offset; - auto LocalStartAddr = StartAddr - file->second.Start + file->second.Offset; - auto fileid = file->second.fileid; + auto LocalRIP = GuestRIP - AOTIRCacheEntry.Offset; + auto LocalStartAddr = StartAddr - AOTIRCacheEntry.Offset; + auto FileId = AOTIRCacheEntry.Entry->FileId; auto RADataCopy = RAData->CreateCopy(); auto IRListCopy = IRList->CreateCopy(); - AOTIRCaptureCacheWriteoutQueue_Append([this, LocalRIP, LocalStartAddr, Length, hash, IRListCopy, RADataCopy, fileid]() { - auto *AotFile = &AOTIRCaptureCacheMap[fileid]; + AOTIRCaptureCacheWriteoutQueue_Append([this, LocalRIP, LocalStartAddr, Length, hash, IRListCopy, RADataCopy, FileId]() { + + // It is guaranteed via AOTIRCaptureCacheWriteoutLock and AOTIRCaptureCacheWriteoutFlusing that this will not run concurrently + // Memory coherency is guaranteed via AOTIRCaptureCacheWriteoutLock + + auto *AotFile = &AOTIRCaptureCacheMap[FileId]; if (!AotFile->Stream) { - AotFile->Stream = AOTIRWriter(fileid); + AotFile->Stream = AOTIRWriter(FileId); uint64_t tag = FEXCore::IR::AOTIR_COOKIE; AotFile->Stream->write((char*)&tag, sizeof(tag)); } @@ -366,21 +358,7 @@ namespace FEXCore::IR { return false; } - AOTIRCaptureCache::AddrToFileMapType::iterator AOTIRCaptureCache::FindAddrForFile(uint64_t Entry, uint64_t Length) { - // Thread safety here! We are returning an iterator to the map object - // This needs the AOTIRCacheLock locked prior to coming in to the function - auto file = AddrToFile.lower_bound(Entry); - if (file != AddrToFile.begin()) { - --file; - if (file->second.Start <= Entry && (file->second.Start + file->second.Len) >= (Entry + Length)) { - return file; - } - } - return AddrToFile.end(); - } - - void AOTIRCaptureCache::AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename) { - // TODO: Support overlapping maps and region splitting + AOTIRCacheEntry *AOTIRCaptureCache::LoadAOTIRCacheEntry(const std::string &filename) { auto base_filename = std::filesystem::path(filename).filename().string(); if (!base_filename.empty()) { @@ -396,21 +374,32 @@ namespace FEXCore::IR { std::unique_lock lk(AOTIRCacheLock); - AddrToFile.insert({ Base, { Base, Size, Offset, fileid, filename, nullptr, false} }); + auto Inserted = AOTIRCache.insert({fileid, AOTIRCacheEntry{0, 0, 0, fileid, filename, false}}); + auto Entry = &(Inserted.first->second); - if (CTX->Config.AOTIRLoad && !AOTIRCache.contains(fileid) && AOTIRLoader) { + LOGMAN_THROW_A_FMT(Entry->Array == nullptr, "Duplicate LoadAOTIRCacheEntry"); + + if (CTX->Config.AOTIRLoad && AOTIRLoader) { auto streamfd = AOTIRLoader(fileid); if (streamfd != -1) { - FEXCore::IR::LoadAOTIRCache(&AOTIRCache, streamfd); + FEXCore::IR::LoadAOTIRCache(Entry, streamfd); close(streamfd); } } + return Entry; } + + return nullptr; } - void AOTIRCaptureCache::RemoveNamedRegion(uintptr_t Base, uintptr_t Size) { - std::unique_lock lk(AOTIRCacheLock); - // TODO: Support partial removing - AddrToFile.erase(Base); + void AOTIRCaptureCache::UnloadAOTIRCacheEntry(AOTIRCacheEntry *Entry) { + LOGMAN_THROW_A_FMT(Entry != nullptr, "Removing not existing entry"); + + if (Entry->Array) { + FEXCore::Allocator::munmap(Entry->FilePtr, Entry->Size); + Entry->Array = nullptr; + Entry->FilePtr = nullptr; + Entry->Size = 0; + } } } diff --git a/External/FEXCore/Source/Interface/IR/AOTIR.h b/External/FEXCore/Source/Interface/IR/AOTIR.h index 5382f6938..da3893a8a 100644 --- a/External/FEXCore/Source/Interface/IR/AOTIR.h +++ b/External/FEXCore/Source/Interface/IR/AOTIR.h @@ -72,18 +72,19 @@ namespace FEXCore::IR { struct AOTIRCacheEntry { AOTIRInlineIndex *Array; - void *mapping; - size_t size; + void *FilePtr; + size_t Size; + std::string FileId; + std::string Filename; + bool ContainsCode; }; using AOTCacheType = std::unordered_map; - bool LoadAOTIRCache(AOTCacheType *AOTIRCache, int streamfd); class AOTIRCaptureCache final { public: AOTIRCaptureCache(FEXCore::Context::Context *ctx) : CTX {ctx} {} - ~AOTIRCaptureCache(); void FinalizeAOTIRCache(); void AOTIRCaptureCacheWriteoutQueue_Flush(); @@ -110,8 +111,8 @@ namespace FEXCore::IR { FEXCore::Core::DebugData *DebugData, bool GeneratedIR); - void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename); - void RemoveNamedRegion(uintptr_t Base, uintptr_t Size); + AOTIRCacheEntry *LoadAOTIRCacheEntry(const std::string &filename); + void UnloadAOTIRCacheEntry(AOTIRCacheEntry *Entry); // Callbacks void SetAOTIRLoader(std::function CacheReader) { @@ -135,27 +136,11 @@ namespace FEXCore::IR { std::queue> AOTIRCaptureCacheWriteoutQueue; - std::map FilesWithCode; - - struct AddrToFileEntry { - uint64_t Start; - uint64_t Len; - uint64_t Offset; - std::string fileid; - std::string filename; - void *CachedFileEntry; - bool ContainsCode; - }; - - using AddrToFileMapType = std::map; - AddrToFileMapType AddrToFile; FEXCore::IR::AOTCacheType AOTIRCache; std::function AOTIRLoader; std::function(const std::string&)> AOTIRWriter; std::function AOTIRRenamer; std::unordered_map AOTIRCaptureCacheMap; - - AddrToFileMapType::iterator FindAddrForFile(uint64_t Entry, uint64_t Length); }; } diff --git a/External/FEXCore/include/FEXCore/Config/Config.h b/External/FEXCore/include/FEXCore/Config/Config.h index 1b651e32a..33da62246 100644 --- a/External/FEXCore/include/FEXCore/Config/Config.h +++ b/External/FEXCore/include/FEXCore/Config/Config.h @@ -27,10 +27,12 @@ namespace Handler { static inline std::string_view SMCCheckHandler(std::string_view Value) { if (Value == "none") return "0"; - else if (Value == "mman") + else if (Value == "mtrack") return "1"; else if (Value == "full") return "2"; + else if (Value == "mman") + return "3"; return "0"; } static inline std::string_view CacheObjectCodeHandler(std::string_view Value) { @@ -57,8 +59,9 @@ namespace Handler { enum ConfigSMCChecks { CONFIG_SMC_NONE, - CONFIG_SMC_MMAN, + CONFIG_SMC_MTRACK, CONFIG_SMC_FULL, + CONFIG_SMC_MMAN, }; enum ConfigObjectCodeHandler { diff --git a/External/FEXCore/include/FEXCore/Core/Context.h b/External/FEXCore/include/FEXCore/Core/Context.h index 749494da4..8fdb63d9d 100644 --- a/External/FEXCore/include/FEXCore/Core/Context.h +++ b/External/FEXCore/include/FEXCore/Core/Context.h @@ -31,6 +31,10 @@ namespace FEXCore::HLE { class SyscallHandler; } +namespace FEXCore::IR { + struct AOTIRCacheEntry; +} + namespace FEXCore::Context { struct Context; enum ExitReason { @@ -236,8 +240,9 @@ namespace FEXCore::Context { FEX_DEFAULT_VISIBILITY FEXCore::CPUID::FunctionResults RunCPUIDFunction(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf); FEX_DEFAULT_VISIBILITY FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf, uint32_t CPU); - FEX_DEFAULT_VISIBILITY void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name); - FEX_DEFAULT_VISIBILITY void RemoveNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length); + FEX_DEFAULT_VISIBILITY FEXCore::IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(FEXCore::Context::Context *CTX, const std::string& Name); + FEX_DEFAULT_VISIBILITY void UnloadAOTIRCacheEntry(FEXCore::Context::Context *CTX, FEXCore::IR::AOTIRCacheEntry *Entry); + FEX_DEFAULT_VISIBILITY void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function CacheReader); FEX_DEFAULT_VISIBILITY void SetAOTIRWriter(FEXCore::Context::Context *CTX, std::function(const std::string&)> CacheWriter); FEX_DEFAULT_VISIBILITY void SetAOTIRRenamer(FEXCore::Context::Context *CTX, std::function CacheRenamer); diff --git a/External/FEXCore/include/FEXCore/HLE/SyscallHandler.h b/External/FEXCore/include/FEXCore/HLE/SyscallHandler.h index 30274b9f8..d85654413 100644 --- a/External/FEXCore/include/FEXCore/HLE/SyscallHandler.h +++ b/External/FEXCore/include/FEXCore/HLE/SyscallHandler.h @@ -1,13 +1,19 @@ #pragma once #include #include +#include #include +#include namespace FEXCore { class CodeLoader; } +namespace FEXCore::IR { + struct AOTIRCacheEntry; +} + namespace FEXCore::Context { struct Context; } @@ -43,6 +49,24 @@ namespace FEXCore::HLE { OS_HANGOVER, }; + class SyscallHandler; + struct AOTIRCacheEntryLookupResult { + AOTIRCacheEntryLookupResult(FEXCore::IR::AOTIRCacheEntry *Entry, uintptr_t Offset, FHU::ScopedSignalMaskWithSharedLock &&lk) + : Entry(Entry), Offset(Offset), lk(std::move(lk)) + { + + } + + AOTIRCacheEntryLookupResult(AOTIRCacheEntryLookupResult&&) = default; + + FEXCore::IR::AOTIRCacheEntry *Entry; + uintptr_t Offset; + + friend class SyscallHandler; + protected: + FHU::ScopedSignalMaskWithSharedLock lk; + }; + class SyscallHandler { public: virtual ~SyscallHandler() = default; @@ -53,6 +77,8 @@ namespace FEXCore::HLE { SyscallOSABI GetOSABI() const { return OSABI; } virtual FEXCore::CodeLoader *GetCodeLoader() const { return nullptr; } + virtual void MarkGuestExecutableRange(uint64_t Start, uint64_t Length) { } + virtual AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(uint64_t GuestAddr) = 0; protected: SyscallOSABI OSABI; diff --git a/Source/Tests/IRLoader.cpp b/Source/Tests/IRLoader.cpp index 93d4f65fe..bdb2ba5a6 100644 --- a/Source/Tests/IRLoader.cpp +++ b/Source/Tests/IRLoader.cpp @@ -15,6 +15,7 @@ $end_info$ #include #include #include +#include #include #include @@ -119,6 +120,25 @@ private: constexpr static uint64_t STACK_SIZE = 8 * 1024 * 1024; }; +class DummySyscallHandler: public FEXCore::HLE::SyscallHandler { + public: + + uint64_t HandleSyscall(FEXCore::Core::CpuStateFrame *Frame, FEXCore::HLE::SyscallArguments *Args) override { + LOGMAN_MSG_A_FMT("Syscalls not implemented"); + return 0; + } + + FEXCore::HLE::SyscallABI GetSyscallABI(uint64_t Syscall) override { + LOGMAN_MSG_A_FMT("Syscalls not implemented"); + return {0, false, 0 }; + } + + std::shared_mutex Mutex; + FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(uint64_t GuestAddr) override { + return {0, 0, FHU::ScopedSignalMaskWithSharedLock {Mutex}}; + } +}; + int main(int argc, char **argv, char **const envp) { LogMan::Throw::InstallHandler(AssertHandler); @@ -141,6 +161,7 @@ int main(int argc, char **argv, char **const envp) std::unique_ptr SignalDelegation = std::make_unique(); FEXCore::Context::SetSignalDelegator(CTX, SignalDelegation.get()); + FEXCore::Context::SetSyscallHandler(CTX, new DummySyscallHandler()); FEX::IRLoader::Loader Loader(Args[0], Args[1]); diff --git a/Source/Tests/LinuxSyscalls/CMakeLists.txt b/Source/Tests/LinuxSyscalls/CMakeLists.txt index 0ca4d9fc8..5c79b3bb8 100644 --- a/Source/Tests/LinuxSyscalls/CMakeLists.txt +++ b/Source/Tests/LinuxSyscalls/CMakeLists.txt @@ -6,6 +6,8 @@ add_library(LinuxEmulation STATIC LinuxAllocator.cpp SignalDelegator.cpp Syscalls.cpp + SyscallsSMCTracking.cpp + SyscallsVMATracking.cpp x32/Syscalls.cpp x32/EPoll.cpp x32/FD.cpp diff --git a/Source/Tests/LinuxSyscalls/Syscalls.cpp b/Source/Tests/LinuxSyscalls/Syscalls.cpp index 3fba4c8cf..1fef28c43 100644 --- a/Source/Tests/LinuxSyscalls/Syscalls.cpp +++ b/Source/Tests/LinuxSyscalls/Syscalls.cpp @@ -27,6 +27,9 @@ $end_info$ #include #include #include +#include +#include +#include #include #include @@ -586,6 +589,10 @@ SyscallHandler::SyscallHandler(FEXCore::Context::Context *_CTX, FEX::HLE::Signal HostKernelVersion = CalculateHostKernelVersion(); GuestKernelVersion = CalculateGuestKernelVersion(); Alloc32Handler = FEX::HLE::Create32BitAllocator(); + + if (SMCChecks == FEXCore::Config::CONFIG_SMC_MTRACK) { + SignalDelegation->RegisterHostSignalHandler(SIGSEGV, HandleSegfault, true); + } } SyscallHandler::~SyscallHandler() { @@ -669,54 +676,22 @@ uint64_t UnimplementedSyscallSafe(FEXCore::Core::CpuStateFrame *Frame, uint64_t return -ENOSYS; } +void SyscallHandler::LockBeforeFork() { + FM.GetFDLock()->lock(); -//// VMA Tracking //// -static std::string get_fdpath(int fd) -{ - std::error_code ec; - return std::filesystem::canonical(std::filesystem::path("/proc/self/fd") / std::to_string(fd), ec).string(); + // XXX shared_mutex has issues with locking and forks + // VMATracking.Mutex.lock(); + + // Add other mutexes here } -void SyscallHandler::TrackMmap(uintptr_t Base, uintptr_t Size, int Prot, int Flags, int fd, off_t Offset) { - if (!(Flags & MAP_ANONYMOUS)) { - auto filename = get_fdpath(fd); +void SyscallHandler::UnlockAfterFork() { + // Add other mutexes here - FEXCore::Context::AddNamedRegion(CTX, Base, Size, Offset, filename); - } - if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN) { - FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size); - } -} - -void SyscallHandler::TrackMunmap(uintptr_t Base, uintptr_t Size) { - if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN) { - FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size); - } -} - -void SyscallHandler::TrackMprotect(uintptr_t Base, uintptr_t Size, int Prot) { - if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN && Prot & PROT_EXEC) { - FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size); - } -} - -void SyscallHandler::TrackMremap(uintptr_t OldAddress, size_t OldSize, size_t NewSize, int flags, uintptr_t NewAddress) { - if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN) { - FEXCore::Context::InvalidateGuestCodeRange(CTX, OldAddress, OldSize); - FEXCore::Context::InvalidateGuestCodeRange(CTX, NewAddress, NewSize); - } -} - -void SyscallHandler::TrackShmat(int shmid, uintptr_t Base, int shmflg) { - // TODO -} - -void SyscallHandler::TrackShmdt(uintptr_t Base) { - // TODO -} - -void SyscallHandler::TrackMadvise(uintptr_t Base, uintptr_t Size, int advice) { - // TODO + // XXX shared_mutex has issues with locking and forks + // VMATracking.Mutex.unlock(); + + FM.GetFDLock()->unlock(); } } diff --git a/Source/Tests/LinuxSyscalls/Syscalls.h b/Source/Tests/LinuxSyscalls/Syscalls.h index 869c6def0..d5adc4564 100644 --- a/Source/Tests/LinuxSyscalls/Syscalls.h +++ b/Source/Tests/LinuxSyscalls/Syscalls.h @@ -16,11 +16,14 @@ $end_info$ #include #include +#include #include #include #include #include +#include +#include #ifdef _M_X86_64 #define SYSCALL_ARCH_NAME x64 #elif _M_ARM_64 @@ -170,6 +173,7 @@ public: // does a guest munmap as if done via a guest syscall virtual int GuestMunmap(void *addr, uint64_t length) = 0; + ///// Memory Manager tracking ///// void TrackMmap(uintptr_t Base, uintptr_t Size, int Prot, int Flags, int fd, off_t Offset); void TrackMunmap(uintptr_t Base, uintptr_t Size); void TrackMprotect(uintptr_t Base, uintptr_t Size, int Prot); @@ -178,6 +182,14 @@ public: void TrackShmdt(uintptr_t Base); void TrackMadvise(uintptr_t Base, uintptr_t Size, int advice); + ///// VMA (Virtual Memory Area) tracking ///// + static bool HandleSegfault(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext); + virtual void MarkGuestExecutableRange(uint64_t Start, uint64_t Length) override; + + ///// FORK tracking ///// + void LockBeforeFork(); + void UnlockAfterFork(); + protected: SyscallHandler(FEXCore::Context::Context *_CTX, FEX::HLE::SignalDelegator *_SignalDelegation); @@ -209,6 +221,109 @@ private: #endif std::unique_ptr Alloc32Handler{}; + + ///// VMA (Virtual Memory Area) tracking ///// + + // AOTIRCacheEntryLookupResult also includes a shared lock guard, so the pointed AOTIRCacheEntry return can be safely used + FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(uint64_t GuestAddr) final override; + + + struct SpecialDev { + static constexpr uint64_t Anon = 0x1'0000'0000; // Anonymous shared mapping, id is incrementing allocation number + static constexpr uint64_t SHM = 0x2'0000'0000; // sys-v shm, id is shmid + }; + + // Memory Resource ID + // An id that can be used to identify when shared mappings actually have the same backing storage + // when dev != SpecialDev::Anon, this is unique system wide + struct MRID { + uint64_t dev; // kernel dev_t is actually 32-bits, we use the extra bits to track SpecialDevs + uint64_t id; + + bool operator<(const MRID& other) const { + return std::tie(dev, id) < std::tie(other.dev, other.id); + } + }; + + struct VMAEntry; + + // Used to all MAP_SHARED VMAs of a system resource. + struct MappedResource { + using ContainerType = std::map; + + FEXCore::IR::AOTIRCacheEntry *AOTIRCacheEntry; + VMAEntry *FirstVMA; + uint64_t Length; // 0 if not fixed size + ContainerType::iterator Iterator; + }; + + union VMAProt { + struct { + bool Readable: 1; + bool Writable: 1; + bool Executable: 1; + }; + uint8_t All: 3; + + + static VMAProt fromProt(int Prot); + static VMAProt fromSHM(int SHMFlg); + }; + + struct VMAFlags { + bool Shared: 1; + + static VMAFlags fromFlags(int Flags); + }; + + struct VMAEntry { + MappedResource *Resource; + + // these are for Intrusive linked list tracking, starting from Resource->FirstVMA + VMAEntry *ResourcePrevVMA; + VMAEntry *ResourceNextVMA; + + uint64_t Base; + uint64_t Offset; + uint64_t Length; + + VMAFlags Flags; + VMAProt Prot; + }; + + struct VMATracking { + using VMAEntry = SyscallHandler::VMAEntry; + std::shared_mutex Mutex; + + // Memory ranges indexed by page aligned starting address + std::map VMAs; + + using VMACIterator = decltype(VMAs)::const_iterator; + + MappedResource::ContainerType MappedResources; + + // Mutex must be at least shared_locked before calling + VMACIterator LookupVMAUnsafe(uint64_t GuestAddr) const; + + // Mutex must be unique_locked before calling + void SetUnsafe(FEXCore::Context::Context *Ctx, MappedResource *MappedResource, uintptr_t Base, uintptr_t Offset, uintptr_t Length, VMAFlags Flags, VMAProt Prot); + + // Mutex must be unique_locked before calling + void ClearUnsafe(FEXCore::Context::Context *Ctx, uintptr_t Base, uintptr_t Length, MappedResource *PreservedMappedResource = nullptr); + + // Mutex must be unique_locked before calling + void ChangeUnsafe(uintptr_t Base, uintptr_t Length, VMAProt Prot); + + // Mutex must be unique_locked before calling + // Returns the Size fo the Shm or 0 if not found + uintptr_t ClearShmUnsafe(FEXCore::Context::Context *Ctx, uintptr_t Base); + private: + bool ListRemove(VMAEntry *Mapping); + void ListReplace(VMAEntry *Mapping, VMAEntry *NewMapping); + void ListInsertAfter(VMAEntry *Mapping, VMAEntry *NewMapping); + void ListPrepend(MappedResource *Resource, VMAEntry *NewVMA); + static void ListCheckVMALinks(VMAEntry *VMA); + } VMATracking; }; uint64_t HandleSyscall(SyscallHandler *Handler, FEXCore::Core::CpuStateFrame *Frame, FEXCore::HLE::SyscallArguments *Args); diff --git a/Source/Tests/LinuxSyscalls/Syscalls/Thread.cpp b/Source/Tests/LinuxSyscalls/Syscalls/Thread.cpp index 426d02217..172c2ee43 100644 --- a/Source/Tests/LinuxSyscalls/Syscalls/Thread.cpp +++ b/Source/Tests/LinuxSyscalls/Syscalls/Thread.cpp @@ -171,11 +171,9 @@ namespace FEX::HLE { } uint64_t ForkGuest(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::CpuStateFrame *Frame, uint32_t flags, void *stack, pid_t *parent_tid, pid_t *child_tid, void *tls) { - // Just before we fork, pull the Filemanagement mutex so it is in a safe state - // Locking the mutex will mean that both processes will end up with a locked mutex - auto Mutex = FEX::HLE::_SyscallHandler->FM.GetFDLock(); - Mutex->lock(); - + // Just before we fork, we lock all syscall mutexes so that both processes will end up with a locked mutex + FEX::HLE::_SyscallHandler->LockBeforeFork(); + pid_t Result{}; if (flags & CLONE_VFORK) { // XXX: We don't currently support a vfork as it causes problems. @@ -186,8 +184,8 @@ namespace FEX::HLE { Result = fork(); } - // Unlock the mutex on both sides of the fork - Mutex->unlock(); + // Unlock the mutexes on both sides of the fork + FEX::HLE::_SyscallHandler->UnlockAfterFork(); if (Result == 0) { // Child diff --git a/Source/Tests/LinuxSyscalls/SyscallsSMCTracking.cpp b/Source/Tests/LinuxSyscalls/SyscallsSMCTracking.cpp new file mode 100644 index 000000000..11a2dfeb4 --- /dev/null +++ b/Source/Tests/LinuxSyscalls/SyscallsSMCTracking.cpp @@ -0,0 +1,354 @@ +/* +$info$ +category: LinuxSyscalls ~ Linux syscall emulation, marshaling and passthrough +tags: LinuxSyscalls|common +desc: SMC/MMan Tracking +$end_info$ +*/ + +#include +#include +#include + +#include "Tests/LinuxSyscalls/Syscalls.h" + +#include +#include +#include +#include + +namespace FEX::HLE { + +/// Helpers /// +auto SyscallHandler::VMAProt::fromProt (int Prot) -> VMAProt { + return VMAProt { + .Readable = (Prot & PROT_READ) != 0, + .Writable = (Prot & PROT_WRITE) != 0, + .Executable = (Prot & PROT_EXEC) != 0, + }; +} + +auto SyscallHandler::VMAProt::fromSHM (int SHMFlg) -> VMAProt { + return VMAProt { + .Readable = true, + .Writable = SHMFlg & SHM_RDONLY ? false : true, + .Executable = false, + }; +} + +auto SyscallHandler::VMAFlags::fromFlags(int Flags) -> VMAFlags { + return VMAFlags { + .Shared = (Flags & MAP_SHARED) != 0, // also includes MAP_SHARED_VALIDATE + }; +} + +// SMC interactions +bool SyscallHandler::HandleSegfault(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) { + auto CTX = Thread->CTX; + + const auto FaultAddress = (uintptr_t)((siginfo_t *)info)->si_addr; + + { + FHU::ScopedSignalMaskWithSharedLock lk(_SyscallHandler->VMATracking.Mutex); + + auto VMATracking = &_SyscallHandler->VMATracking; + + // If the write spans two pages, they will be flushed one at a time (generating two faults) + auto Entry = VMATracking->LookupVMAUnsafe(FaultAddress); + + // If an untracked address, or the mapping wasn't writable, it can't be handled here + if (Entry == VMATracking->VMAs.end() || !Entry->second.Prot.Writable) { + return false; + } + + auto FaultBase = FEXCore::AlignDown(FaultAddress, FHU::FEX_PAGE_SIZE); + + if (Entry->second.Flags.Shared) { + LOGMAN_THROW_A_FMT(Entry->second.Resource, "VMA tracking error"); + + auto Offset = FaultBase - Entry->first + Entry->second.Offset; + + auto VMA = Entry->second.Resource->FirstVMA; + LOGMAN_THROW_A_FMT(VMA, "VMA tracking error"); + + // Flush all mirrors, remap the page writable as needed + do { + if (VMA->Offset <= Offset && (VMA->Offset + VMA->Length) > Offset) { + auto FaultBaseMirrored = Offset - VMA->Offset + VMA->Base; + if (VMA->Prot.Writable) { + auto rv = mprotect((void *)FaultBaseMirrored, FHU::FEX_PAGE_SIZE, PROT_READ | PROT_WRITE); + LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", FaultBaseMirrored, FHU::FEX_PAGE_SIZE); + } + FEXCore::Context::InvalidateGuestCodeRange(CTX, FaultBaseMirrored, FHU::FEX_PAGE_SIZE); + } + } while ((VMA = VMA->ResourceNextVMA)); + } else { + // Mark as read write before flush, so that if code is compiled after the Flush but before returning, the segfault will be + // re-raised + auto rv = mprotect((void *)FaultBase, FHU::FEX_PAGE_SIZE, PROT_READ | PROT_WRITE); + LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", FaultBase, FHU::FEX_PAGE_SIZE); + FEXCore::Context::InvalidateGuestCodeRange(CTX, FaultBase, FHU::FEX_PAGE_SIZE); + } + + return true; + } +} + +void SyscallHandler::MarkGuestExecutableRange(uint64_t Start, uint64_t Length) { + const auto Base = Start & FHU::FEX_PAGE_MASK; + const auto Top = FEXCore::AlignUp(Start + Length, FHU::FEX_PAGE_SIZE); + + { + if (SMCChecks != FEXCore::Config::CONFIG_SMC_MTRACK) { + return; + } + + FHU::ScopedSignalMaskWithSharedLock lk(VMATracking.Mutex); + + // Find the first mapping at or after the range ends, or ::end(). + // Top points to the address after the end of the range + auto Mapping = VMATracking.VMAs.lower_bound(Top); + + while (Mapping != VMATracking.VMAs.begin()) { + Mapping--; + + const auto MapBase = Mapping->first; + const auto MapTop = MapBase + Mapping->second.Length; + + if (MapTop <= Base) { + // Mapping ends before the Range start, exit + break; + } else { + const auto ProtectBase = std::max(MapBase, Base); + const auto ProtectSize = std::min(MapTop, Top) - ProtectBase; + + if (Mapping->second.Flags.Shared) { + LOGMAN_THROW_A_FMT(Mapping->second.Resource, "VMA tracking error"); + + const auto OffsetBase = ProtectBase - Mapping->first + Mapping->second.Offset; + const auto OffsetTop = OffsetBase + ProtectSize; + + auto VMA = Mapping->second.Resource->FirstVMA; + LOGMAN_THROW_A_FMT(VMA, "VMA tracking error"); + + do { + auto VMAOffsetBase = VMA->Offset; + auto VMAOffsetTop = VMA->Offset + VMA->Length; + auto VMABase = VMA->Base; + + if (VMA->Prot.Writable && VMAOffsetBase < OffsetTop && VMAOffsetTop > OffsetBase) { + + const auto MirroredBase = std::max(VMAOffsetBase, OffsetBase); + const auto MirroredSize = std::min(OffsetTop, VMAOffsetTop) - MirroredBase; + + auto rv = mprotect((void *)(MirroredBase - VMAOffsetBase + VMABase), MirroredSize, PROT_READ); + LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", MirroredBase, MirroredSize); + } + } while ((VMA = VMA->ResourceNextVMA)); + + } else if (Mapping->second.Prot.Writable) { + int rv = mprotect((void *)ProtectBase, ProtectSize, PROT_READ); + + LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", ProtectBase, ProtectSize); + } + } + } + } +} + +// Used for AOT +FEXCore::HLE::AOTIRCacheEntryLookupResult SyscallHandler::LookupAOTIRCacheEntry(uint64_t GuestAddr) { + FHU::ScopedSignalMaskWithSharedLock lk(_SyscallHandler->VMATracking.Mutex); + auto rv = FEXCore::HLE::AOTIRCacheEntryLookupResult(nullptr, 0, std::move(lk)); + + // Get the first mapping after GuestAddr, or end + // GuestAddr is inclusive + // If the write spans two pages, they will be flushed one at a time (generating two faults) + auto Entry = _SyscallHandler->VMATracking.LookupVMAUnsafe(GuestAddr); + + if (Entry != _SyscallHandler->VMATracking.VMAs.end()) { + rv.Entry = Entry->second.Resource ? Entry->second.Resource->AOTIRCacheEntry : nullptr; + rv.Offset = Entry->second.Base - Entry->second.Offset; + } + + return rv; +} + +// MMan Tracking +static std::string get_fdpath(int fd) { + std::error_code ec; + return std::filesystem::canonical(std::filesystem::path("/proc/self/fd") / std::to_string(fd), ec).string(); +} + +void SyscallHandler::TrackMmap(uintptr_t Base, uintptr_t Size, int Prot, int Flags, int fd, off_t Offset) { + Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE); + + { + FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex); + + static uint64_t AnonSharedId = 1; + + MappedResource *Resource = nullptr; + + if (!(Flags & MAP_ANONYMOUS)) { + struct stat64 buf; + fstat64(fd, &buf); + MRID mrid {buf.st_dev, buf.st_ino}; + + auto [Iter, Inserted] = VMATracking.MappedResources.emplace(mrid, MappedResource {nullptr, nullptr, 0}); + Resource = &Iter->second; + + if (Inserted) { + auto filename = get_fdpath(fd); + Resource->AOTIRCacheEntry = FEXCore::Context::LoadAOTIRCacheEntry(CTX, filename); + Resource->Iterator = Iter; + } + + } else if (Flags & MAP_SHARED) { + MRID mrid{SpecialDev::Anon, AnonSharedId++}; + + auto [Iter, Inserted] = VMATracking.MappedResources.emplace(mrid, MappedResource{nullptr, nullptr, 0}); + LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error"); + Resource = &Iter->second; + Resource->Iterator = Iter; + } else { + Resource = nullptr; + } + + VMATracking.SetUnsafe(CTX, Resource, Base, Offset, Size, VMAFlags::fromFlags(Flags), VMAProt::fromProt(Prot)); + } + + if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) { + FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size); + } +} + +void SyscallHandler::TrackMunmap(uintptr_t Base, uintptr_t Size) { + Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE); + + { + FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex); + + VMATracking.ClearUnsafe(CTX, Base, Size); + } + + if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) { + FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size); + } +} + +void SyscallHandler::TrackMprotect(uintptr_t Base, uintptr_t Size, int Prot) { + Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE); + + { + FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex); + + VMATracking.ChangeUnsafe(Base, Size, VMAProt::fromProt(Prot)); + } + + if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) { + FEXCore::Context::InvalidateGuestCodeRange(CTX, Base, Size); + } +} + +void SyscallHandler::TrackMremap(uintptr_t OldAddress, size_t OldSize, size_t NewSize, int flags, uintptr_t NewAddress) { + OldSize = FEXCore::AlignUp(OldSize, FHU::FEX_PAGE_SIZE); + NewSize = FEXCore::AlignUp(NewSize, FHU::FEX_PAGE_SIZE); + + { + + FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex); + + const auto OldVMA = VMATracking.LookupVMAUnsafe(OldAddress); + + const auto OldResource = OldVMA->second.Resource; + const auto OldOffset = OldVMA->second.Offset + OldAddress - OldVMA->first; + const auto OldFlags = OldVMA->second.Flags; + const auto OldProt = OldVMA->second.Prot; + + LOGMAN_THROW_A_FMT(OldVMA != VMATracking.VMAs.end(), "VMA Tracking corruption"); + + if (OldSize == 0) { + // Mirror existing mapping + // must be a shared mapping + LOGMAN_THROW_A_FMT(OldResource != nullptr, "VMA Tracking error"); + LOGMAN_THROW_A_FMT(OldFlags.Shared, "VMA Tracking error"); + VMATracking.SetUnsafe(CTX, OldResource, NewAddress, OldOffset, NewSize, OldFlags, OldProt); + } else { + +// MREMAP_DONTUNMAP is kernel 5.7+ +#ifdef MREMAP_DONTUNMAP + if (!(flags & MREMAP_DONTUNMAP)) +#endif + { + VMATracking.ClearUnsafe(CTX, OldAddress, OldSize, OldResource); + } + + // Make anonymous mapping + VMATracking.SetUnsafe(CTX, OldResource, NewAddress, OldOffset, NewSize, OldFlags, OldProt); + } + } + + if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) { + if (OldAddress != NewAddress) { + if (OldSize != 0) { + // This also handles the MREMAP_DONTUNMAP case + FEXCore::Context::InvalidateGuestCodeRange(CTX, OldAddress, OldSize); + } + } else { + // If mapping shrunk, flush the unmapped region + if (OldSize > NewSize) { + FEXCore::Context::InvalidateGuestCodeRange(CTX, OldAddress + NewSize, OldSize - NewSize); + } + } + } +} + +void SyscallHandler::TrackShmat(int shmid, uintptr_t Base, int shmflg) { + shmid_ds stat; + + auto res = shmctl(shmid, IPC_STAT, &stat); + LOGMAN_THROW_A_FMT(res != -1, "shmctl IPC_STAT failed"); + + uint64_t Length = stat.shm_segsz; + + { + FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex); + + // TODO + MRID mrid{SpecialDev::SHM, static_cast(shmid)}; + + auto ResourceInserted = VMATracking.MappedResources.insert({mrid, {nullptr, nullptr, Length}}); + auto Resource = &ResourceInserted.first->second; + if (ResourceInserted.second) { + Resource->Iterator = ResourceInserted.first; + } + VMATracking.SetUnsafe(CTX, Resource, Base, 0, Length, VMAFlags::fromFlags(MAP_SHARED), + VMAProt::fromProt((shmflg & SHM_RDONLY) ? PROT_READ : (PROT_READ | PROT_WRITE)) + ); + } + if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) { + FEXCore::Context::InvalidateGuestCodeRange(CTX, Base, Length); + } +} + +void SyscallHandler::TrackShmdt(uintptr_t Base) { + FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex); + + auto Length = VMATracking.ClearShmUnsafe(CTX, Base); + + if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) { + // This might over flush if the shm has holes in it + FEXCore::Context::InvalidateGuestCodeRange(CTX, Base, Length); + } +} + +void SyscallHandler::TrackMadvise(uintptr_t Base, uintptr_t Size, int advice) { + Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE); + { + FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex); + // TODO + } +} + +} \ No newline at end of file diff --git a/Source/Tests/LinuxSyscalls/SyscallsVMATracking.cpp b/Source/Tests/LinuxSyscalls/SyscallsVMATracking.cpp new file mode 100644 index 000000000..b66db2657 --- /dev/null +++ b/Source/Tests/LinuxSyscalls/SyscallsVMATracking.cpp @@ -0,0 +1,366 @@ +/* +$info$ +category: LinuxSyscalls ~ Linux syscall emulation, marshaling and passthrough +tags: LinuxSyscalls|common +desc: VMA Tracking +$end_info$ +*/ + +#include "Tests/LinuxSyscalls/Syscalls.h" + +namespace FEX::HLE { +/// List Operations /// + +inline void SyscallHandler::VMATracking::ListCheckVMALinks(VMAEntry *VMA) { + if (VMA) { + LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA != VMA, "VMA tracking error"); + LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA != VMA, "VMA tracking error"); + } +} + +// Removes a VMA from corresponding MappedResource list +// Returns true if list is empty +bool SyscallHandler::VMATracking::ListRemove(VMAEntry *VMA) { + LOGMAN_THROW_A_FMT(VMA->Resource != nullptr, "VMA tracking error"); + + // if it has prev, make prev to next + if (VMA->ResourcePrevVMA) { + LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA->ResourceNextVMA == VMA, "VMA tracking error"); + VMA->ResourcePrevVMA->ResourceNextVMA = VMA->ResourceNextVMA; + } else { + LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA == VMA, "VMA tracking error"); + } + + // if it has next, make next to prev + if (VMA->ResourceNextVMA) { + LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA->ResourcePrevVMA == VMA, "VMA tracking error"); + VMA->ResourceNextVMA->ResourcePrevVMA = VMA->ResourcePrevVMA; + } + + // If it is the first in the list, make Next the first in the list + if (VMA->Resource && VMA->Resource->FirstVMA == VMA) { + LOGMAN_THROW_A_FMT(!VMA->ResourceNextVMA || VMA->ResourceNextVMA->ResourcePrevVMA == nullptr, "VMA tracking error"); + + VMA->Resource->FirstVMA = VMA->ResourceNextVMA; + } + + ListCheckVMALinks(VMA); + ListCheckVMALinks(VMA->ResourceNextVMA); + ListCheckVMALinks(VMA->ResourcePrevVMA); + + // Return true if list is empty + return VMA->Resource->FirstVMA == nullptr; +} + +// Replaces a VMA in corresponding MappedResource list +// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup +void SyscallHandler::VMATracking::ListReplace(VMAEntry *VMA, VMAEntry *NewVMA) { + LOGMAN_THROW_A_FMT(VMA->Resource != nullptr, "VMA tracking error"); + + LOGMAN_THROW_A_FMT(VMA->Resource == NewVMA->Resource, "VMA tracking error"); + LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == VMA->ResourcePrevVMA, "VMA tracking error"); + LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == VMA->ResourceNextVMA, "VMA tracking error"); + + if (VMA->ResourcePrevVMA) { + LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA != VMA, "VMA tracking error"); + LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA->ResourceNextVMA == VMA, "VMA tracking error"); + VMA->ResourcePrevVMA->ResourceNextVMA = NewVMA; + } else { + LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA == VMA, "VMA tracking error"); + VMA->Resource->FirstVMA = NewVMA; + } + + if (VMA->ResourceNextVMA) { + LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA->ResourcePrevVMA == VMA, "VMA tracking error"); + VMA->ResourceNextVMA->ResourcePrevVMA = NewVMA; + } + + ListCheckVMALinks(VMA); + ListCheckVMALinks(NewVMA); + ListCheckVMALinks(VMA->ResourceNextVMA); + ListCheckVMALinks(VMA->ResourcePrevVMA); +} + +// Inserts a VMA in corresponding MappedResource list +// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup +void SyscallHandler::VMATracking::ListInsertAfter(VMAEntry *AfterVMA, VMAEntry *NewVMA) { + LOGMAN_THROW_A_FMT(NewVMA->Resource != nullptr, "VMA tracking error"); + + LOGMAN_THROW_A_FMT(AfterVMA->Resource == NewVMA->Resource, "VMA tracking error"); + LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == AfterVMA, "VMA tracking error"); + LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == AfterVMA->ResourceNextVMA, "VMA tracking error"); + + if (AfterVMA->ResourceNextVMA) { + LOGMAN_THROW_A_FMT(AfterVMA->ResourceNextVMA->ResourcePrevVMA == AfterVMA, "VMA tracking error"); + AfterVMA->ResourceNextVMA->ResourcePrevVMA = NewVMA; + } + AfterVMA->ResourceNextVMA = NewVMA; + + ListCheckVMALinks(AfterVMA); + ListCheckVMALinks(NewVMA); + ListCheckVMALinks(AfterVMA->ResourceNextVMA); + ListCheckVMALinks(AfterVMA->ResourcePrevVMA); +} + +// Prepends a VMA +// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup +void SyscallHandler::VMATracking::ListPrepend(MappedResource *Resource, VMAEntry *NewVMA) { + LOGMAN_THROW_A_FMT(Resource != nullptr, "VMA tracking error"); + + LOGMAN_THROW_A_FMT(NewVMA->Resource == Resource, "VMA tracking error"); + LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == nullptr, "VMA tracking error"); + LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == Resource->FirstVMA, "VMA tracking error"); + + if (Resource->FirstVMA) { + LOGMAN_THROW_A_FMT(Resource->FirstVMA->ResourcePrevVMA == nullptr, "VMA tracking error"); + Resource->FirstVMA->ResourcePrevVMA = NewVMA; + } + + Resource->FirstVMA = NewVMA; + + ListCheckVMALinks(NewVMA); + ListCheckVMALinks(NewVMA->ResourceNextVMA); + ListCheckVMALinks(NewVMA->ResourcePrevVMA); +} + +/// VMA tracking /// + +// Lookup a VMA by address +SyscallHandler::VMATracking::VMACIterator SyscallHandler::VMATracking::LookupVMAUnsafe(uint64_t GuestAddr) const { + auto Entry = VMAs.upper_bound(GuestAddr); + + if (Entry != VMAs.begin()) { + --Entry; + + if (Entry->first <= GuestAddr && (Entry->first + Entry->second.Length) > GuestAddr) { + return Entry; + } + } + + return VMAs.end(); +} + +// Set or Replace mappings in a range with a new mapping +void SyscallHandler::VMATracking::SetUnsafe(FEXCore::Context::Context *CTX, MappedResource *MappedResource, uintptr_t Base, + uintptr_t Offset, uintptr_t Length, VMAFlags Flags, VMAProt Prot) { + ClearUnsafe(CTX, Base, Length, MappedResource); + + auto [Iter, Inserted] = VMAs.emplace( + Base, VMAEntry{MappedResource, nullptr, MappedResource ? MappedResource->FirstVMA : nullptr, Base, Offset, Length, Flags, Prot}); + + LOGMAN_THROW_A_FMT(Inserted == true, "VMA Tracking corruption"); + + if (MappedResource) { + // Insert to the front of the linked list + ListPrepend(MappedResource, &Iter->second); + } +} + +// Remove mappings in a range, possibly splitting them if needed and +// freeing their associated MappedResource unless it is equal to PreservedMappedResource +void SyscallHandler::VMATracking::ClearUnsafe(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, + MappedResource *PreservedMappedResource) { + const auto Top = Base + Length; + + // find the first Mapping at or after the Range ends, or ::end() + // Top is the address after the end + auto CurrentIter = VMAs.lower_bound(Top); + + // Iterate backwards all mappings + while (CurrentIter != VMAs.begin()) { + CurrentIter--; + + const auto Current = &CurrentIter->second; + const auto MapBase = Current->Base; + const auto MapTop = MapBase + Current->Length; + const auto OffsetDiff = Current->Offset - MapBase; + + if (MapTop <= Base) { + // Mapping ends before the Range start, exit + break; + } else { + const bool HasFirstPart = MapBase < Base; + const bool HasTrailingPart = MapTop > Top; + + // (1) HasFirstPart, !HasTrailingPart -> trim + // (2) HasFirstPart, HasTrailingPart -> trim, insert trailing, list add after first part + // (3) !HasFirstPart, !HasTrailing part -> list remove, erase + // (4) !HasFirstPart, HasTrailing part -> insert trailing, list replace first part, erase + + if (HasFirstPart) { + // Handle trim for (1) & (2) + Current->Length = Base - MapBase; + } else if (!HasTrailingPart) { + // Handle all of (3) + // Mapping is included or equal to Range, delete + + // If linked to a Mapped Resource, remove from linked list and possibly delete the Mapped Resource + if (Current->Resource) { + if (ListRemove(Current) && Current->Resource != PreservedMappedResource) { + if (Current->Resource->AOTIRCacheEntry) { + FEXCore::Context::UnloadAOTIRCacheEntry(CTX, Current->Resource->AOTIRCacheEntry); + } + MappedResources.erase(Current->Resource->Iterator); + } + } + + // returns next element, so -- is safe at loop + CurrentIter = VMAs.erase(CurrentIter); + continue; // we're done + } + + const bool ReplaceAndErase = !HasFirstPart; + + if (HasTrailingPart) { + // Handle insert of (2), (4) + + // insert trailing part, link it after Mapping + auto NewOffset = OffsetDiff + Top; + auto NewLength = MapTop - Top; + + auto [Iter, Inserted] = VMAs.emplace( + Top, VMAEntry{Current->Resource, ReplaceAndErase ? Current->ResourcePrevVMA : Current, Current->ResourceNextVMA, Top, NewOffset, NewLength, Current->Flags, Current->Prot}); + LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error"); + auto TrailingPart = &Iter->second; + if (Current->Resource) { + if (ReplaceAndErase) { + // Handle list replace of (4) + ListReplace(Current, TrailingPart); + } else { + // Handle list insert (2) + ListInsertAfter(Current, TrailingPart); + } + } + } + + if (ReplaceAndErase) { + // Handle erase of (4) + // returns next element, so -- is safe at loop + CurrentIter = VMAs.erase(CurrentIter); + } + } + } +} + +// Change flags of mappings in a range and split the mappings if needed +void SyscallHandler::VMATracking::ChangeUnsafe(uintptr_t Base, uintptr_t Length, VMAProt NewProt) { + const auto Top = Base + Length; + + // find the first Mapping at or after the Range ends, or ::end() + // Top is the address after the end + auto MappingIter = VMAs.lower_bound(Top); + + // Iterate backwards all mappings + while (MappingIter != VMAs.begin()) { + MappingIter--; + + auto Current = &MappingIter->second; + const auto MapBase = Current->Base; + const auto MapTop = MapBase + Current->Length; + const auto MapFlags = Current->Flags; + const auto MapProt = Current->Prot; + + const auto OffsetDiff = Current->Offset - MapBase; + + if (MapTop <= Base) { + // Mapping ends before the Range start, exit + break; + } else if (MapProt.All == NewProt.All) { + // Mapping already has the needed prots + continue; + } else { + const bool HasFirstPart = MapBase < Base; + const bool HasTrailingPart = MapTop > Top; + + if (HasFirstPart) { + // Mapping starts before range, split first part + + // Trim end of original mapping + Current->Length = Base - MapBase; + + // Make new VMA with new flags, insert for length of range + auto NewOffset = OffsetDiff + Base; + auto NewLength = Top - Base; + + auto [Iter, Inserted] = + VMAs.emplace(Base, VMAEntry{Current->Resource, Current, Current->ResourceNextVMA, Base, NewOffset, NewLength, MapFlags, NewProt}); + LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error"); + auto RestOfMapping = &Iter->second; + + if (Current->Resource) { + ListInsertAfter(Current, RestOfMapping); + } + + Current = RestOfMapping; + } else { + // Mapping starts in range, just change Prot + Current->Prot = NewProt; + } + + if (HasTrailingPart) { + // ends after Range, split last part and insert with original flags + + // Trim the mapping (possibly already trimmed) + Current->Length = Top - Current->Base; + + // prot has already been changed + + // Make new VMA with original flags, insert for remaining length + auto NewOffset = OffsetDiff + Top; + auto NewLength = MapTop - Top; + + auto [Iter, Inserted] = + VMAs.emplace(Top, VMAEntry{Current->Resource, Current, Current->ResourceNextVMA, Top, NewOffset, NewLength, MapFlags, MapProt}); + LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error"); + auto TrailingMapping = &Iter->second; + + if (Current->Resource) { + ListInsertAfter(Current, TrailingMapping); + } + } + } + } +} + +// This matches the peculiarities algorithm used in linux ksys_shmdt (linux kernel 5.16, ipc/shm.c) +uintptr_t SyscallHandler::VMATracking::ClearShmUnsafe(FEXCore::Context::Context *CTX, uintptr_t Base) { + + // Find first VMA at or after Base + // Iterate until first SHM VMA, with matching offset, get length + // Then, erase any later occurrences of this SHM + + // returns first element that is greater or equal or ::end + auto Entry = VMAs.lower_bound(Base); + + for (; Entry != VMAs.end(); ++Entry) { + LOGMAN_THROW_A_FMT(Entry->second.Base >= Base, "VMA tracking corruption"); + if (Entry->second.Base - Base == Entry->second.Offset && Entry->second.Resource && + Entry->second.Resource->Iterator->first.dev == SpecialDev::SHM) { + break; + } + } + + if (Entry == VMAs.end()) { + return 0; + } + + const auto ShmLength = Entry->second.Resource->Iterator->second.Length; + const auto Resource = Entry->second.Resource; + + do { + if (Entry->second.Resource == Resource) { + if (ListRemove(&Entry->second)) { + if (Entry->second.Resource->AOTIRCacheEntry) { + FEXCore::Context::UnloadAOTIRCacheEntry(CTX, Entry->second.Resource->AOTIRCacheEntry); + } + MappedResources.erase(Entry->second.Resource->Iterator); + } + Entry = VMAs.erase(Entry); + } else { + Entry++; + } + } while (Entry != VMAs.end() && (Entry->second.Base + Entry->second.Length - Base) <= ShmLength); + + return ShmLength; +} +} \ No newline at end of file diff --git a/Source/Tools/FEXConfig/Main.cpp b/Source/Tools/FEXConfig/Main.cpp index a804cddc4..3a0bd28f0 100644 --- a/Source/Tools/FEXConfig/Main.cpp +++ b/Source/Tools/FEXConfig/Main.cpp @@ -568,16 +568,19 @@ namespace { if (**Value == "0") { SMCChecks = FEXCore::Config::CONFIG_SMC_NONE; } else if (**Value == "1") { - SMCChecks = FEXCore::Config::CONFIG_SMC_MMAN; + SMCChecks = FEXCore::Config::CONFIG_SMC_MTRACK; } else if (**Value == "2") { SMCChecks = FEXCore::Config::CONFIG_SMC_FULL; + } else if (**Value == "3") { + SMCChecks = FEXCore::Config::CONFIG_SMC_MMAN; } } bool SMCChanged = false; SMCChanged |= ImGui::RadioButton("None", &SMCChecks, FEXCore::Config::CONFIG_SMC_NONE); ImGui::SameLine(); - SMCChanged |= ImGui::RadioButton("MMan", &SMCChecks, FEXCore::Config::CONFIG_SMC_MMAN); ImGui::SameLine(); + SMCChanged |= ImGui::RadioButton("MTrack (Default)", &SMCChecks, FEXCore::Config::CONFIG_SMC_MTRACK); ImGui::SameLine(); SMCChanged |= ImGui::RadioButton("Full", &SMCChecks, FEXCore::Config::CONFIG_SMC_FULL); + SMCChanged |= ImGui::RadioButton("MMan (Deprecated)", &SMCChecks, FEXCore::Config::CONFIG_SMC_MMAN); ImGui::SameLine(); if (SMCChanged) { LoadedConfig->EraseSet(FEXCore::Config::ConfigOption::CONFIG_SMCCHECKS, std::to_string(SMCChecks));