WOW64: Support disabling DEP

Required for older 32-bit games that assumes the execute bit is implicit
from read.

Source/Windows/Common/InvalidationTracker.cpp

@@ -30,6 +30,19 @@ InvalidationTracker::InvalidationTracker(FEXCore::Context::Context& CTX, const s
   }
 }
 
+static bool ProtHasExec(ULONG Prot) {
+  return (Prot & (PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY)) != 0;
+}
+
+static bool ProtIsReadable(ULONG Prot) {
+  return (Prot & (PAGE_READONLY | PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE |
+                  PAGE_EXECUTE_WRITECOPY)) != 0;
+}
+
+static bool ProtIsWritable(ULONG Prot) {
+  return (Prot & (PAGE_READWRITE | PAGE_WRITECOPY | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY)) != 0;
+}
+
 void InvalidationTracker::HandleMemoryProtectionNotification(uint64_t Address, uint64_t Size, ULONG Prot) {
   const auto AlignedBase = Address & FEXCore::Utils::FEX_PAGE_MASK;
   const auto AlignedSize = (Address - AlignedBase + Size + FEXCore::Utils::FEX_PAGE_SIZE - 1) & FEXCore::Utils::FEX_PAGE_MASK;
@@ -38,16 +51,27 @@ void InvalidationTracker::HandleMemoryProtectionNotification(uint64_t Address, u
     std::unique_lock Lock(IntervalsLock);
 
     FEXCore::IntervalList<uint64_t>::Interval ProtInterval {AlignedBase, AlignedBase + AlignedSize};
-    if (Prot & (PAGE_EXECUTE | PAGE_EXECUTE_READ | PAGE_EXECUTE_READWRITE | PAGE_EXECUTE_WRITECOPY)) {
+
+    const bool HasExec = ProtHasExec(Prot);
+    const bool EffectiveExec = HasExec || (DEPDisabled && ProtIsReadable(Prot));
+    const bool EffectiveRWX = EffectiveExec && ProtIsWritable(Prot);
+
+    if (EffectiveExec) {
       XIntervals.Insert(ProtInterval);
-      if (Prot & (PAGE_EXECUTE_WRITECOPY | PAGE_EXECUTE_READWRITE)) {
+      if (EffectiveRWX) {
         LogMan::Msg::DFmt("Add SMC interval: {:X} - {:X}", AlignedBase, AlignedBase + AlignedSize);
         RWXIntervals.Insert(ProtInterval);
       }
+      if (DEPDisabled && !HasExec) {
+        DEPPromotedIntervals.Insert(ProtInterval);
+      }
       return true;
     } else if (XIntervals.Intersect(ProtInterval)) {
       XIntervals.Remove(ProtInterval);
       RWXIntervals.Remove(ProtInterval);
+      if (DEPDisabled) {
+        DEPPromotedIntervals.Remove(ProtInterval);
+      }
       return true;
     }
 
@@ -60,6 +84,53 @@ void InvalidationTracker::HandleMemoryProtectionNotification(uint64_t Address, u
   }
 }
 
+void InvalidationTracker::HandleProcessExecuteFlagsChange(ULONG Flags) {
+  const bool DisableDEP = (Flags & MEM_EXECUTE_OPTION_ENABLE) != 0;
+
+  std::scoped_lock CodeLock(CTX.GetCodeInvalidationMutex());
+  std::unique_lock Lock(IntervalsLock);
+
+  if (DisableDEP == DEPDisabled) {
+    return;
+  }
+
+  DEPDisabled = DisableDEP;
+
+  if (DisableDEP) {
+    DEPPromotedIntervals.Clear();
+
+    MEMORY_BASIC_INFORMATION Info;
+    uint64_t Address = 0;
+
+    while (VirtualQuery(reinterpret_cast<LPCVOID>(Address), &Info, sizeof(Info))) {
+      uint64_t BaseAddress = reinterpret_cast<uint64_t>(Info.BaseAddress);
+      if (Info.State == MEM_COMMIT && ProtIsReadable(Info.Protect) && !ProtHasExec(Info.Protect)) {
+        const auto AlignedBase = BaseAddress & FEXCore::Utils::FEX_PAGE_MASK;
+        const auto AlignedSize = (BaseAddress - AlignedBase + Info.RegionSize + FEXCore::Utils::FEX_PAGE_SIZE - 1) & FEXCore::Utils::FEX_PAGE_MASK;
+        FEXCore::IntervalList<uint64_t>::Interval ProtInterval {AlignedBase, AlignedBase + AlignedSize};
+
+        XIntervals.Insert(ProtInterval);
+        if (ProtIsWritable(Info.Protect)) {
+          RWXIntervals.Insert(ProtInterval);
+        }
+        DEPPromotedIntervals.Insert(ProtInterval);
+      }
+
+      Address = BaseAddress + Info.RegionSize;
+    }
+  } else {
+    for (const auto& Interval : DEPPromotedIntervals) {
+      XIntervals.Remove(Interval);
+      RWXIntervals.Remove(Interval);
+    }
+    DEPPromotedIntervals.Clear();
+  }
+
+  // Invalidate all cached code: previously-compiled blocks may contain NoExec stubs for addresses
+  // that are now executable (or reference regions whose executability just changed).
+  InvalidateIntervalInternalLocked(0, std::numeric_limits<uint64_t>::max());
+}
+
 void InvalidationTracker::HandleImageMap(std::string_view Name, uint64_t Address) {
   auto* Nt = RtlImageNtHeader(reinterpret_cast<HMODULE>(Address));
   auto* SectionsBegin = IMAGE_FIRST_SECTION(Nt);
@@ -146,9 +217,12 @@ void InvalidationTracker::ReprotectRWXIntervals(uint64_t Address, uint64_t Size)
 }
 
 bool InvalidationTracker::HandleRWXAccessViolation(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPc, uint64_t FaultAddress) {
-  const bool NeedsInvalidate = [&](uint64_t Address) {
+  const auto [NeedsInvalidate, UntrapProt] = [&](uint64_t Address) -> std::pair<bool, ULONG> {
     std::shared_lock Lock(IntervalsLock);
-    return RWXIntervals.Query(Address).Enclosed;
+    if (!RWXIntervals.Query(Address).Enclosed) {
+      return {false, 0};
+    }
+    return {true, GetUntrapProt(Address)};
   }(FaultAddress);
 
   if (NeedsInvalidate) {
@@ -162,7 +236,7 @@ bool InvalidationTracker::HandleRWXAccessViolation(FEXCore::Core::InternalThread
       ULONG TmpProt;
       void* TmpAddress = reinterpret_cast<void*>(FaultAddress);
       SIZE_T TmpSize = 1;
-      NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_EXECUTE_READWRITE, &TmpProt);
+      NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, UntrapProt, &TmpProt);
     }
     DetectMonoBackpatcherBlock(Thread, HostPc);
     return true;
@@ -223,7 +297,7 @@ void InvalidationTracker::DisableSMCDetection() {
   SMCDetectionDisabled = true;
   uint64_t Address = 0;
 
-  // Reprotect all RWX intervals as RWX
+  // Reprotect all RWX intervals as writable
   FEXCore::IntervalList<uint64_t>::QueryResult Query;
   do {
     Query = RWXIntervals.Query(Address);
@@ -231,12 +305,26 @@ void InvalidationTracker::DisableSMCDetection() {
       void* TmpAddress = reinterpret_cast<void*>(Address);
       SIZE_T TmpSize = static_cast<SIZE_T>(Query.Size);
       ULONG TmpProt;
-      NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_EXECUTE_READWRITE, &TmpProt);
+      NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, GetUntrapProt(Address), &TmpProt);
     }
     Address += Query.Size;
   } while (Query.Size);
 }
 
+ULONG InvalidationTracker::GetTrapProt(uint64_t Address) const {
+  if (DEPDisabled && DEPPromotedIntervals.Query(Address).Enclosed) {
+    return PAGE_READONLY;
+  }
+  return PAGE_EXECUTE_READ;
+}
+
+ULONG InvalidationTracker::GetUntrapProt(uint64_t Address) const {
+  if (DEPDisabled && DEPPromotedIntervals.Query(Address).Enclosed) {
+    return PAGE_READWRITE;
+  }
+  return PAGE_EXECUTE_READWRITE;
+}
+
 void InvalidationTracker::InvalidateIntervalInternal(uint64_t Address, uint64_t Size) {
   std::scoped_lock CodeLock(CTX.GetCodeInvalidationMutex());
   InvalidateIntervalInternalLocked(Address, Size);
@@ -275,7 +363,7 @@ bool InvalidationTracker::ProtectRWXIntervalsInternal(uint64_t Address, uint64_t
       void* TmpAddress = reinterpret_cast<void*>(Address);
       SIZE_T TmpSize = static_cast<SIZE_T>(std::min(End, Address + Query.Size) - Address);
       ULONG TmpProt;
-      NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, ForWriteLocked ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ, &TmpProt);
+      NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, ForWriteLocked ? GetUntrapProt(Address) : GetTrapProt(Address), &TmpProt);
     } else if (!Query.Size) {
       // No more regions past `Address` in the interval list
       break;

Source/Windows/Common/InvalidationTracker.h

@@ -24,6 +24,7 @@ class InvalidationTracker {
 public:
   InvalidationTracker(FEXCore::Context::Context& CTX, const std::unordered_map<DWORD, FEXCore::Core::InternalThreadState*>& Threads);
   void HandleMemoryProtectionNotification(uint64_t Address, uint64_t Size, ULONG Prot);
+  void HandleProcessExecuteFlagsChange(ULONG Flags);
   void HandleImageMap(std::string_view Name, uint64_t Address);
   struct InvalidateContainingSectionResult {
     uint64_t SectionStart;
@@ -51,12 +52,20 @@ private:
   // and any code in the range will be invalidated before protection as RWX, otherwise protects as RX if false.
   bool ProtectRWXIntervalsInternal(uint64_t Address, uint64_t Size, bool ForWriteLocked);
 
+  // Returns the correct protection for trapping (removing write) or untrapping (restoring write) an RWX interval.
+  // For DEP-promoted regions (originally non-exec), uses PAGE_READONLY/PAGE_READWRITE instead of PAGE_EXECUTE_READ/PAGE_EXECUTE_READWRITE.
+  // NOTE: Must be called with IntervalsLock held.
+  ULONG GetTrapProt(uint64_t Address) const;
+  ULONG GetUntrapProt(uint64_t Address) const;
+
   FEXCore::IntervalList<uint64_t> XIntervals;
   FEXCore::IntervalList<uint64_t> RWXIntervals;
   std::shared_mutex IntervalsLock;
   FEXCore::Context::Context& CTX;
   const std::unordered_map<DWORD, FEXCore::Core::InternalThreadState*>& Threads;
   bool SMCDetectionDisabled {false}; // Protected by IntervalsLock
+  bool DEPDisabled {false};          // Protected by IntervalsLock
+  FEXCore::IntervalList<uint64_t> DEPPromotedIntervals; // Protected by IntervalsLock
 
   bool MonoBackpatcherDetectionPending {false};
   uint64_t MonoBase {0};

Source/Windows/WOW64/BTInterface.h

@@ -26,6 +26,7 @@ void STDMETHODCALLTYPE BTCpuNotifyMemoryFree(void* Address, SIZE_T Size, ULONG F
 NTSTATUS STDMETHODCALLTYPE BTCpuNotifyMapViewOfSection(void* Unk1, void* Address, void* Unk2, SIZE_T Size, ULONG AllocType, ULONG Prot);
 void STDMETHODCALLTYPE BTCpuNotifyUnmapViewOfSection(void* Address, BOOL After, ULONG Status);
 void STDMETHODCALLTYPE BTCpuNotifyReadFile(HANDLE Handle, void* Address, SIZE_T Size, BOOL After, NTSTATUS Status);
+void STDMETHODCALLTYPE BTCpuNotifyProcessExecuteFlagsChange(ULONG Flags);
 BOOLEAN STDMETHODCALLTYPE BTCpuIsProcessorFeaturePresent(UINT Feature);
 void STDMETHODCALLTYPE BTCpuUpdateProcessorInformation(SYSTEM_CPU_INFORMATION* Info);
 }

Source/Windows/WOW64/Module.cpp

@@ -1039,6 +1039,11 @@ void BTCpuNotifyReadFile(HANDLE Handle, void* Address, SIZE_T Size, BOOL After,
   }
 }
 
+void BTCpuNotifyProcessExecuteFlagsChange(ULONG Flags) {
+  std::scoped_lock Lock(ThreadCreationMutex);
+  InvalidationTracker->HandleProcessExecuteFlagsChange(Flags);
+}
+
 BOOLEAN WINAPI BTCpuIsProcessorFeaturePresent(UINT Feature) {
   return CPUFeatures->IsFeaturePresent(Feature) ? TRUE : FALSE;
 }

Source/Windows/WOW64/libwow64fex.def

@@ -12,6 +12,7 @@ EXPORTS
   BTCpuNotifyMemoryFree
   BTCpuNotifyMapViewOfSection
   BTCpuNotifyUnmapViewOfSection
+  BTCpuNotifyProcessExecuteFlagsChange
   BTCpuNotifyReadFile
   BTCpuProcessInit
   BTCpuResetToConsistentState

Source/Windows/include/winternl.h

@@ -466,6 +466,11 @@ typedef enum _MEMORY_INFORMATION_CLASS {
 
 #define SystemEmulationBasicInformation (SYSTEM_INFORMATION_CLASS)62
 
+#define MEM_EXECUTE_OPTION_DISABLE 0x01
+#define MEM_EXECUTE_OPTION_ENABLE 0x02
+#define MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION 0x04
+#define MEM_EXECUTE_OPTION_PERMANENT 0x08
+
 #define ProcessFexHardwareTso (PROCESSINFOCLASS)2000
 #define ProcessFexUnalignAtomic (PROCESSINFOCLASS)2001