LinuxSyscalls: Update signal mask at deferring time

In regular x86 programs, when a signal occurs, the signal will not be handled within the signal handler. However, under FEX's defer signal mechanism, the signal is not immediately masked when it is deferred. When returning to the location that receives the signal and continues processing, the signal might be received again, causing inconsistency between the emulation and the actual program.

Here is an unit test for this patch from ltp:
https://github.com/linux-test-project/ltp/blob/master/testcases/kernel/syscalls/timer_settime/timer_settime03.c

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -536,6 +536,27 @@ static bool IsAsyncSignal(const siginfo_t* Info, int Signal) {
   return true;
 }
 
+uint64_t SignalDelegator::GetNewSigMask(int Signal) const {
+  const SignalHandler& Handler = HostHandlers[Signal];
+  // Set up a new mask based on this signals signal mask
+  uint64_t NewMask = Handler.GuestAction.sa_mask.Val;
+
+  // If NODEFER then the new signal mask includes this signal
+  if (!(Handler.GuestAction.sa_flags & SA_NODEFER)) {
+    NewMask |= (1ULL << (Signal - 1));
+  }
+
+  // Walk our required signals and stop masking them if requested
+  for (size_t i = 0; i < MAX_SIGNALS; ++i) {
+    if (HostHandlers[i + 1].Required.load(std::memory_order_relaxed)) {
+      // Never mask our required signals
+      NewMask &= ~(1ULL << i);
+    }
+  }
+
+  return NewMask;
+}
+
 void SignalDelegator::HandleGuestSignal(FEX::HLE::ThreadStateObject* ThreadObject, int Signal, void* Info, void* UContext) {
   auto Thread = ThreadObject->Thread;
   ucontext_t* _context = (ucontext_t*)UContext;
@@ -561,6 +582,8 @@ void SignalDelegator::HandleGuestSignal(FEX::HLE::ThreadStateObject* ThreadObjec
         const auto& Top = ThreadObject->SignalInfo.DeferredSignalFrames.back();
         Signal = Top.Signal;
         SigInfo = Top.Info;
+        // sig mask has been updated at the defer time, recover the original mask
+        memcpy(&_context->uc_sigmask, &Top.SigMask, sizeof(uint64_t));
         ThreadObject->SignalInfo.DeferredSignalFrames.pop_back();
 
         // Until we re-protect the page to PROT_NONE, FEX will now *permanently* defer signals and /not/ check them.
@@ -594,11 +617,19 @@ void SignalDelegator::HandleGuestSignal(FEX::HLE::ThreadStateObject* ThreadObjec
                            "Deferred signals vector hit "
                            "capacity size. This will "
                            "likely crash! Asserting now!");
+
         ThreadObject->SignalInfo.DeferredSignalFrames.emplace_back(ThreadStateObject::DeferredSignalState {
           .Info = SigInfo,
           .Signal = Signal,
+          .SigMask = _context->uc_sigmask.__val[0],
         });
 
+        uint64_t NewMask = GetNewSigMask(Signal);
+
+        // Update our host signal mask so we don't hit race conditions with signals
+        // This allows us to maintain the expected signal mask through the guest signal handling and then all the way back again
+        memcpy(&_context->uc_sigmask, &NewMask, sizeof(uint64_t));
+
         // Now update the faulting page permissions so it will fault on write.
         mprotect(reinterpret_cast<void*>(&Thread->InterruptFaultPage), sizeof(Thread->InterruptFaultPage), PROT_NONE);
 
@@ -633,21 +664,7 @@ void SignalDelegator::HandleGuestSignal(FEX::HLE::ThreadStateObject* ThreadObjec
   } else {
     if (Handler.GuestHandler &&
         Handler.GuestHandler(Thread, Signal, &SigInfo, UContext, &Handler.GuestAction, &ThreadObject->SignalInfo.GuestAltStack)) {
-      // Set up a new mask based on this signals signal mask
-      uint64_t NewMask = Handler.GuestAction.sa_mask.Val;
-
-      // If NODEFER then the new signal mask includes this signal
-      if (!(Handler.GuestAction.sa_flags & SA_NODEFER)) {
-        NewMask |= (1ULL << (Signal - 1));
-      }
-
-      // Walk our required signals and stop masking them if requested
-      for (size_t i = 0; i < MAX_SIGNALS; ++i) {
-        if (HostHandlers[i + 1].Required.load(std::memory_order_relaxed)) {
-          // Never mask our required signals
-          NewMask &= ~(1ULL << i);
-        }
-      }
+      uint64_t NewMask = GetNewSigMask(Signal);
 
       // Update our host signal mask so we don't hit race conditions with signals
       // This allows us to maintain the expected signal mask through the guest signal handling and then all the way back again

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.h

@@ -277,6 +277,8 @@ private:
   bool HandleSignalPause(FEXCore::Core::InternalThreadState* Thread, int Signal, void* info, void* ucontext);
   bool HandleSIGILL(FEXCore::Core::InternalThreadState* Thread, int Signal, void* info, void* ucontext);
 
+  uint64_t GetNewSigMask(int Signal) const;
+
   std::mutex HostDelegatorMutex;
   std::mutex GuestDelegatorMutex;
   bool SupportsAVX;

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h

@@ -18,6 +18,7 @@ $end_info$
 #include <FEXCore/Utils/Profiler.h>
 #include <FEXCore/Utils/SignalScopeGuards.h>
 
+#include <bits/types/sigset_t.h>
 #include <cstdint>
 #include <linux/seccomp.h>
 
@@ -37,6 +38,7 @@ struct ThreadStateObject : public FEXCore::Allocator::FEXAllocOperators {
   struct DeferredSignalState {
     siginfo_t Info;
     int Signal;
+    uint64_t SigMask;
   };
 
   FEXCore::Core::InternalThreadState* Thread;