Merge pull request #6003 from yassinsolim/portable-exec-fd

LinuxSyscalls: Preserve argv and path across portable reexec

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -393,6 +393,8 @@ int main(int argc, char** argv, char** const envp) {
 
   int FEXFD {StealFEXFDFromEnv("FEX_EXECVEFD")};
   int FEXSeccompFD {StealFEXFDFromEnv("FEX_SECCOMPFD")};
+  const bool FEXFDPathBacked = getenv("FEX_EXECVEFD_PATH") != nullptr;
+  unsetenv("FEX_EXECVEFD_PATH");
 
   // Early init trivial handlers.
   LogMan::Throw::InstallHandler(FEX::Logging::AssertHandler);
@@ -511,9 +513,14 @@ int main(int argc, char** argv, char** const envp) {
     FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_FILENAME, Program.ProgramPath);
     FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_CONFIG_NAME, Program.ProgramName);
   } else if (FEXFD != -1) {
-    // Anonymous program.
-    FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_FILENAME, "<Anonymous>");
-    FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_CONFIG_NAME, "<Anonymous>");
+    if (FEXFDPathBacked) {
+      FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_FILENAME, Program.ProgramPath);
+      FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_CONFIG_NAME, Program.ProgramName);
+    } else {
+      // Anonymous program.
+      FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_FILENAME, "<Anonymous>");
+      FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_CONFIG_NAME, "<Anonymous>");
+    }
   } else {
     {
       char ExistsTempPath[PATH_MAX];

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp

@@ -316,6 +316,10 @@ uint64_t ExecveHandler(FEXCore::Core::CpuStateFrame* Frame, const char* pathname
   // This will stay inside of our emulated environment since binfmt_misc will capture it
   const bool IsBinfmtCompatible = SyscallHandler->IsInterpreterInstalled() && !NeedsFDCopy &&
                                   (Type == ELFLoader::ELFContainer::ELFType::TYPE_X86_32 || Type == ELFLoader::ELFContainer::ELFType::TYPE_X86_64);
+  // Without a visible binfmt interpreter, the next FEX process resolves guest paths through RootFS.
+  // Keep the already resolved host ELF open so a binary outside that RootFS can still be executed.
+  const bool NeedsLoaderFD = !IsFDExec && !SyscallHandler->IsInterpreterInstalled() &&
+                             (Type == ELFLoader::ELFContainer::ELFType::TYPE_X86_32 || Type == ELFLoader::ELFContainer::ELFType::TYPE_X86_64);
 
   // We are trying to execute an ELF of a different architecture
   // We can't know if we can support this without architecture specific checks and binfmt_misc parsing
@@ -328,7 +332,7 @@ uint64_t ExecveHandler(FEXCore::Core::CpuStateFrame* Frame, const char* pathname
   // - seccomp inheritance
   // - FEXServer FD inheritance (unshare(CLONE_NEWNET))
   // - FD_CLOEXEC set on FD on anonymous file FD.
-  const bool NeedsEnvpCopy = (IsFDExec && !(IsBinfmtCompatible || IsOtherELF)) || HasSeccomp || NeedsFDCopy;
+  const bool NeedsEnvpCopy = (IsFDExec && !(IsBinfmtCompatible || IsOtherELF)) || HasSeccomp || NeedsFDCopy || NeedsLoaderFD;
 
   // We are trying to execute a shebang handled by a different architecture interpreter (e.g. /usr/bin/python from the host FS).
   // In this case we just defer to the kernel.
@@ -350,6 +354,13 @@ uint64_t ExecveHandler(FEXCore::Core::CpuStateFrame* Frame, const char* pathname
         // so duplicate the FD if FD_CLOEXEC is set, which removes the FD_CLOEXEC flag.
         Args.dirfd = dup(Args.dirfd);
         FDExecCopy = true;
+      } else if (NeedsLoaderFD) {
+        Args.dirfd = open(Filename.c_str(), O_RDONLY);
+        if (Args.dirfd == -1) {
+          CloseSeccompFD();
+          return -errno;
+        }
+        FDExecCopy = true;
       }
 
       // Remove AT_EMPTY_PATH flag now.
@@ -363,6 +374,10 @@ uint64_t ExecveHandler(FEXCore::Core::CpuStateFrame* Frame, const char* pathname
 
       // Insert the FD for FEX to track.
       EnvpArgs.emplace_back(FDExecEnv.data());
+      if (NeedsLoaderFD) {
+        // Distinguish this path-backed FD from an anonymous memfd exec.
+        EnvpArgs.emplace_back("FEX_EXECVEFD_PATH=1");
+      }
     }
 
     if (HasSeccomp) {
@@ -424,8 +439,11 @@ uint64_t ExecveHandler(FEXCore::Core::CpuStateFrame* Frame, const char* pathname
 
     // It is valid to provide nullptr first argument.
     if (*OldArgv) {
-      // Skip filename argument
-      ++OldArgv;
+      // The direct fallback uses the executable path as argv[0]. An FD-backed load uses the
+      // binfmt preserve-argv0 layout, so the caller-supplied argv[0] has to stay in the vector.
+      if (!NeedsLoaderFD) {
+        ++OldArgv;
+      }
       while (*OldArgv) {
         // Append the arguments together
         ExecveArgs.emplace_back(*OldArgv);

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Thread.cpp

@@ -253,7 +253,7 @@ void RegisterThread(FEX::HLE::SyscallHandler* Handler) {
     });
 
   // launch a new process under fex
-  // currently does not propagate argv[0] correctly
+  // the ELF self-reexec fallback preserves the caller-supplied argv[0]
   REGISTER_SYSCALL_IMPL_X32(execve, [](FEXCore::Core::CpuStateFrame* Frame, const char* pathname, uint32_t* argv, uint32_t* envp) -> uint64_t {
     fextl::vector<const char*> Args;
     fextl::vector<const char*> Envp;

Source/Tools/LinuxEmulation/LinuxSyscalls/x64/Thread.cpp

@@ -285,7 +285,7 @@ void RegisterThread(FEX::HLE::SyscallHandler* Handler) {
   });
 
   // launch a new process under fex
-  // currently does not propagate argv[0] correctly
+  // the ELF self-reexec fallback preserves the caller-supplied argv[0]
   REGISTER_SYSCALL_IMPL_X64(execve, [](FEXCore::Core::CpuStateFrame* Frame, const char* pathname, char* const argv[], char* const envp[]) -> uint64_t {
     fextl::vector<const char*> Args;
     fextl::vector<const char*> Envp;

unittests/FEXLinuxTests/CMakeLists.txt

@@ -124,3 +124,12 @@ add_custom_target(fex_linux_tests_all
   USES_TERMINAL
   COMMAND "ctest" "--output-on-failure" "--timeout" "30" ${TEST_JOB_FLAG} "-R" "\.*\.flt$$"
   DEPENDS FEXLinuxTests FEXLinuxTests_32 FEX)
+
+foreach(EXEC_FD_BITNESS 64 32)
+  set(EXEC_FD_CHAIN_BIN "${CMAKE_CURRENT_BINARY_DIR}/FEXLinuxTests_${EXEC_FD_BITNESS}/exec_fd_chain.${EXEC_FD_BITNESS}")
+  add_test(NAME "exec_fd_chain.${EXEC_FD_BITNESS}.jit.flt"
+    COMMAND "$<TARGET_FILE:FEX>" "${EXEC_FD_CHAIN_BIN}")
+  # Force the self-reexec fallback. FEX_ROOTFS still comes from the caller environment.
+  set_property(TEST "exec_fd_chain.${EXEC_FD_BITNESS}.jit.flt" APPEND PROPERTY ENVIRONMENT "FEX_PORTABLE=1")
+  set_property(TEST "exec_fd_chain.${EXEC_FD_BITNESS}.jit.flt" APPEND PROPERTY ENVIRONMENT "FEX_OUTPUTLOG=stderr")
+endforeach()

unittests/FEXLinuxTests/tests/CMakeLists.txt

@@ -56,3 +56,6 @@ target_link_options(smc-exec-stack.${BITNESS} PRIVATE -Wl,-z,execstack)
 
 # Must use lld because it has the nognustack option
 target_link_options(smc-missing-gnustack.${BITNESS} PRIVATE -fuse-ld=lld -Wl,-z,nognustack)
+
+# Plain executable, outside the Catch2 guest harness. It re-execs itself twice.
+add_executable(exec_fd_chain.${BITNESS} exec_fd_chain.c)

unittests/FEXLinuxTests/tests/exec_fd_chain.c

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: MIT
+// Three-stage x86 executable chain for the no-binfmt self-reexec path.
+// The binary is deliberately outside the FEX RootFS. Each stage uses a
+// caller-supplied argv[0] that differs from the executable path.
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+extern char** environ;
+
+static void fail(const char* message) {
+  fprintf(stderr, "exec_fd_chain: %s\n", message);
+  exit(1);
+}
+
+static int private_exec_env_leaked(void) {
+  for (char** env = environ; env && *env; ++env) {
+    if (strncmp(*env, "FEX_EXECVEFD=", 13) == 0 || strncmp(*env, "FEX_EXECVEFD_PATH=", 18) == 0) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+static void read_exe(char* path, size_t size) {
+  ssize_t bytes = readlink("/proc/self/exe", path, size - 1);
+  if (bytes < 0) {
+    fail("readlink /proc/self/exe");
+  }
+  path[bytes] = '\0';
+}
+
+static void check_exe(const char* expected) {
+  char path[4096];
+  read_exe(path, sizeof(path));
+  if (strcmp(path, expected) != 0) {
+    fprintf(stderr, "exec_fd_chain: exe '%s' expected '%s'\n", path, expected);
+    exit(1);
+  }
+}
+
+int main(int argc, char** argv) {
+  char self[4096];
+  read_exe(self, sizeof(self));
+
+  if (argc == 1) {
+    char* child_argv[] = {"runtime-helper", "stage1", self, NULL};
+    execve(self, child_argv, environ);
+    perror("exec_fd_chain: execve stage1");
+    return 1;
+  }
+
+  if (private_exec_env_leaked()) {
+    fail("private exec fd environment leaked");
+  }
+
+  if (argc == 3 && strcmp(argv[1], "stage1") == 0) {
+    if (strcmp(argv[0], "runtime-helper") != 0) {
+      fprintf(stderr, "exec_fd_chain: stage1 argv0 '%s'\n", argv[0]);
+      return 1;
+    }
+    check_exe(argv[2]);
+    char* child_argv[] = {"worker", "stage2", argv[2], "sentinel", NULL};
+    execve(self, child_argv, environ);
+    perror("exec_fd_chain: execve stage2");
+    return 1;
+  }
+
+  if (argc == 4 && strcmp(argv[1], "stage2") == 0) {
+    if (strcmp(argv[0], "worker") != 0) {
+      fprintf(stderr, "exec_fd_chain: stage2 argv0 '%s'\n", argv[0]);
+      return 1;
+    }
+    if (strcmp(argv[3], "sentinel") != 0) {
+      fail("missing sentinel");
+    }
+    check_exe(argv[2]);
+    printf("exec_fd_chain=ok exe=%s argv0=%s\n", argv[2], argv[0]);
+    return 0;
+  }
+
+  fprintf(stderr, "exec_fd_chain: unexpected argc %d\n", argc);
+  return 1;
+}