LinuxSyscalls: Implement code map writing for future code caching

FEXCore/Source/Interface/Config/Config.json.in

@@ -16,6 +16,13 @@
           "Maximum number of instruction to store in a block"
         ]
       },
+      "EnableCodeCachingWIP": {
+        "Type": "bool",
+        "Default": "false",
+        "Desc": [
+          "Enable the code caching subsystem"
+        ]
+      },
       "HostFeatures": {
         "Type": "strenum",
         "Default": "FEXCore::Config::HostFeatures::OFF",

FEXCore/Source/Interface/Context/Context.h

@@ -156,6 +156,16 @@ public:
     return CodeCache;
   }
 
+  void SetCodeMapWriter(fextl::unique_ptr<CodeMapWriter> Writer) override {
+    CodeMapWriter = std::move(Writer);
+  }
+
+  void FlushAndCloseCodeMap() override {
+    if (CodeMapWriter) {
+      CodeMapWriter.reset();
+    }
+  }
+
   void OnCodeBufferAllocated(const std::shared_ptr<CPU::CodeBuffer>&) override;
   void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override;
   void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) override;
@@ -226,6 +236,7 @@ public:
   FEXCore::ThunkHandler* ThunkHandler {};
   fextl::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
   CodeCache CodeCache;
+  fextl::unique_ptr<CodeMapWriter> CodeMapWriter;
 
   SignalDelegator* SignalDelegation {};
 

FEXCore/Source/Interface/Core/Core.cpp

@@ -437,6 +437,10 @@ void ContextImpl::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread
 
   Profiler::PostForkAction(Child);
   if (Child) {
+    if (CodeMapWriter) {
+      CodeMapWriter->ResetAfterFork();
+    }
+
     CodeInvalidationMutex.StealAndDropActiveLocks();
     if (Config.StrictInProcessSplitLocks) {
       StrictSplitLockMutex = 0;
@@ -720,7 +724,8 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
   if (SourcecodeResolver && Config.GDBSymbols()) {
     auto MappedSection = SyscallHandler->LookupExecutableFileSection(*Thread, GuestRIP);
     if (MappedSection) {
-      MappedSection->FileInfo.SourcecodeMap = SourcecodeResolver->GenerateMap(MappedSection->FileInfo.Filename, MappedSection->FileInfo.FileId);
+      MappedSection->FileInfo.SourcecodeMap =
+        SourcecodeResolver->GenerateMap(MappedSection->FileInfo.Filename, CodeMap::GetBaseFilename(MappedSection->FileInfo, false));
     }
   }
 
@@ -859,6 +864,13 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr);
   }
 
+  if (CodeMapWriter) {
+    auto Region = SyscallHandler->LookupExecutableFileSection(*Thread, GuestRIP);
+    if (Region && Region->FileStartVA != 0) {
+      CodeMapWriter->AppendBlock(*Region, GuestRIP);
+    }
+  }
+
   return (uintptr_t)CodePtr;
 }
 

FEXCore/include/FEXCore/Core/Context.h

@@ -136,6 +136,8 @@ public:
   FEX_DEFAULT_VISIBILITY virtual FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) = 0;
 
   virtual AbstractCodeCache& GetCodeCache() = 0;
+  virtual void SetCodeMapWriter(fextl::unique_ptr<CodeMapWriter>) = 0;
+  virtual void FlushAndCloseCodeMap() = 0;
 
   FEX_DEFAULT_VISIBILITY virtual void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) = 0;
   FEX_DEFAULT_VISIBILITY virtual void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) = 0;

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -475,6 +475,9 @@ int main(int argc, char** argv, char** const envp) {
   auto SyscallHandler = Loader.Is64BitMode() ?
                           FEX::HLE::x64::CreateHandler(CTX.get(), SignalDelegation.get(), ThunkHandler.get()) :
                           FEX::HLE::x32::CreateHandler(CTX.get(), SignalDelegation.get(), ThunkHandler.get(), std::move(Allocator));
+  if (FEXCore::Config::Get_ENABLECODECACHINGWIP()) {
+    CTX->SetCodeMapWriter(fextl::make_unique<FEXCore::CodeMapWriter>(*SyscallHandler));
+  }
 
   // Load VDSO in to memory prior to mapping our ELFs.
   auto VDSOMapping = FEX::VDSO::LoadVDSOThunks(Loader.Is64BitMode(), SyscallHandler.get());

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -712,6 +712,7 @@ void SignalDelegator::HandleGuestSignal(FEX::HLE::ThreadStateObject* ThreadObjec
   } else if (Handler.OldAction.handler == SIG_IGN || (Handler.OldAction.handler == SIG_DFL && Handler.DefaultBehaviour == DEFAULT_IGNORE)) {
     // Do nothing
   } else if (Handler.OldAction.handler == SIG_DFL && (Handler.DefaultBehaviour == DEFAULT_COREDUMP || Handler.DefaultBehaviour == DEFAULT_TERM)) {
+    CTX->FlushAndCloseCodeMap();
 
 #ifndef FEX_DISABLE_TELEMETRY
     // In the case of signals that cause coredump or terminate, save telemetry early.

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp

@@ -197,6 +197,8 @@ static fextl::string GetShebangInterpFilename(const fextl::string& Filename) {
 
 uint64_t ExecveHandler(FEXCore::Core::CpuStateFrame* Frame, const char* pathname, char* const* argv, char* const* envp, ExecveAtArgs Args) {
   auto SyscallHandler = FEX::HLE::_SyscallHandler;
+  Frame->Thread->CTX->FlushAndCloseCodeMap();
+
   fextl::string Filename {};
 
   fextl::string RootFS = SyscallHandler->RootFSPath();

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h

@@ -101,7 +101,10 @@ struct ExecveAtArgs {
 
 uint64_t ExecveHandler(FEXCore::Core::CpuStateFrame* Frame, const char* pathname, char* const* argv, char* const* envp, ExecveAtArgs Args);
 
-class SyscallHandler : public FEXCore::HLE::SyscallHandler, FEXCore::HLE::SourcecodeResolver, public FEXCore::Allocator::FEXAllocOperators {
+class SyscallHandler : public FEXCore::HLE::SyscallHandler,
+                       FEXCore::HLE::SourcecodeResolver,
+                       public FEXCore::CodeMapOpener,
+                       public FEXCore::Allocator::FEXAllocOperators {
 public:
   ThreadManager TM;
   FEX::HLE::SeccompEmulator SeccompEmulator;
@@ -277,6 +280,8 @@ public:
   std::optional<FEXCore::ExecutableFileSectionInfo>
   LookupExecutableFileSection(FEXCore::Core::InternalThreadState& Thread, uint64_t GuestAddr) final override;
 
+  int OpenCodeMapFile() override;
+
   FEXCore::HLE::ExecutableRangeInfo QueryGuestExecutableRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Address) override;
 
   ///// FORK tracking /////

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp

@@ -566,6 +566,8 @@ void RegisterThread(FEX::HLE::SyscallHandler* Handler) {
   });
 
   REGISTER_SYSCALL_IMPL(exit_group, [](FEXCore::Core::CpuStateFrame* Frame, int status) -> uint64_t {
+    Frame->Thread->CTX->FlushAndCloseCodeMap();
+
     // Save telemetry if we're exiting.
     FEX::HLE::_SyscallHandler->GetSignalDelegator()->SaveTelemetry();
     FEX::HLE::_SyscallHandler->TM.CleanupForExit();

Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp

@@ -8,6 +8,7 @@ $end_info$
 */
 
 #include "Common/FDUtils.h"
+#include "Common/FEXServerClient.h"
 #include "Common/FileMappingBaseAddress.h"
 
 #include <filesystem>
@@ -312,6 +313,34 @@ uint64_t SyscallHandler::GuestMremap(bool Is64Bit, FEXCore::Core::InternalThread
   return Result;
 }
 
+int SyscallHandler::OpenCodeMapFile() {
+  // Query from FEXServer whether this is the first instance of this executable; if it is, also enable code dumping!
+  FEX_CONFIG_OPT(RootFSPath, ROOTFS);
+  FEX_CONFIG_OPT(Multiblock, MULTIBLOCK);
+  auto ProgramName = FEXCore::Config::Get(FEXCore::Config::CONFIG_APP_FILENAME);
+  LOGMAN_THROW_A_FMT(ProgramName && ProgramName.value()->c_str()[0] == '/', "");
+
+  // Check RootFS first, then the plain path
+  auto ProgramFD = open((RootFSPath() + ProgramName.value()->c_str()).c_str(), O_RDONLY);
+  if (ProgramFD == -1) {
+    ProgramFD = open(ProgramName.value()->c_str(), O_RDONLY);
+  }
+  if (ProgramFD == -1) {
+    return -1;
+  }
+
+  int CodeMapFD = FEXServerClient::RequestCodeMapFD(FEXServerClient::GetServerFD(), ProgramFD, Multiblock);
+  close(ProgramFD);
+  if (CodeMapFD == -1) {
+    return -1;
+  }
+
+  // Ensure the file descriptor is closed on exec
+  auto flags = fcntl(CodeMapFD, F_GETFD);
+  fcntl(CodeMapFD, F_SETFD, flags | FD_CLOEXEC);
+  return CodeMapFD;
+}
+
 uint64_t SyscallHandler::GuestMprotect(FEXCore::Core::InternalThreadState* Thread, void* addr, size_t len, int prot) {
   uint64_t Result {};
 

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp

@@ -245,6 +245,9 @@ void ThreadManager::DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool Need
     auto It = std::find(Threads.begin(), Threads.end(), Thread);
     LOGMAN_THROW_A_FMT(It != Threads.end(), "Thread wasn't in Threads");
     Threads.erase(It);
+    if (Threads.empty()) {
+      Thread->Thread->CTX->FlushAndCloseCodeMap();
+    }
   }
 
   Stat.DeallocateSlot(Thread->Thread->ThreadStats);