unittests: add tests for stack mapping

unittests/FEXLinuxTests/tests/CMakeLists.txt

@@ -46,3 +46,9 @@ target_link_libraries(smc-shared-2.${BITNESS} PRIVATE rt pthread)
 target_link_libraries(thunk_testlib.${BITNESS} PRIVATE ${CMAKE_DL_LIBS})
 
 target_link_libraries(timer-sigev-thread.${BITNESS} PRIVATE rt pthread)
+
+target_link_libraries(smc-unexec-stack.${BITNESS} PRIVATE -Wl,-z,noexecstack)
+
+target_link_options(smc-exec-stack.${BITNESS} PRIVATE -Wl,-z,execstack)
+
+target_link_options(smc-missing-gnustack.${BITNESS} PRIVATE -fuse-ld=lld -Wl,-z,nognustack)

unittests/FEXLinuxTests/tests/smc/smc-exec-stack.cpp

@@ -0,0 +1,38 @@
+#include <catch2/catch_test_macros.hpp>
+#include <signal.h>
+#include <ucontext.h>
+#include <sys/mman.h>
+
+bool got_signal = false;
+
+static void sigsegv_handler(int signal, siginfo_t *siginfo, void* context) {
+  REQUIRE(siginfo->si_code == SEGV_ACCERR);
+  got_signal = true;
+  size_t page_size = sysconf(_SC_PAGESIZE);
+  void *fault_addr = (void *)((uintptr_t)(siginfo->si_addr) & ~(page_size - 1));
+  REQUIRE(mprotect(fault_addr, page_size, PROT_READ | PROT_WRITE | PROT_EXEC) == 0);
+}
+
+TEST_CASE("smc-exec-stack: PT_GNU_STACK == RWX") {
+  // Register signal handler
+  struct sigaction act {};
+  act.sa_sigaction = sigsegv_handler;
+  act.sa_flags = SA_SIGINFO;
+  sigaction(SIGSEGV, &act, nullptr);
+  
+  // Try executing from stack
+  uint8_t stack_code = 0xC3; // ret
+  ((void (*)())(&stack_code))();
+  REQUIRE(got_signal == false);
+
+  // Executing from other memory should fail
+  size_t page_size = sysconf(_SC_PAGESIZE);
+  uint8_t *mem_code = static_cast<uint8_t *>(mmap(NULL, page_size, PROT_READ | PROT_WRITE,
+      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+  REQUIRE(mem_code != nullptr);
+  *mem_code = 0xC3; // ret
+  ((void (*)())(mem_code))();
+  REQUIRE(got_signal == true);
+
+  munmap(mem_code, page_size);
+}

unittests/FEXLinuxTests/tests/smc/smc-missing-gnustack.cpp

@@ -0,0 +1,49 @@
+#include <catch2/catch_test_macros.hpp>
+#include <signal.h>
+#include <ucontext.h>
+#include <sys/mman.h>
+
+bool got_signal = false;
+
+static void sigsegv_handler(int signal, siginfo_t *siginfo, void* context) {
+  REQUIRE(siginfo->si_code == SEGV_ACCERR);
+  got_signal = true;
+  size_t page_size = sysconf(_SC_PAGESIZE);
+  void *fault_addr = (void *)((uintptr_t)(siginfo->si_addr) & ~(page_size - 1));
+  REQUIRE(mprotect(fault_addr, page_size, PROT_READ | PROT_WRITE | PROT_EXEC) == 0);
+}
+
+TEST_CASE("smc-missing-gnustack: PT_GNU_STACK missing") {
+  // Register signal handler
+  struct sigaction act {};
+  act.sa_sigaction = sigsegv_handler;
+  act.sa_flags = SA_SIGINFO;
+  sigaction(SIGSEGV, &act, nullptr);
+
+  // Try executing from stack
+  uint8_t stack_code = 0xC3; // ret
+  ((void (*)())(&stack_code))();
+
+#ifdef __i386__
+  REQUIRE(got_signal == false);
+#else
+  REQUIRE(got_signal == true);
+  got_signal = false;
+#endif
+
+  // Executing from other memory should fail on 64 bit but work on 32 bit
+  size_t page_size = sysconf(_SC_PAGESIZE);
+  uint8_t *mem_code = static_cast<uint8_t *>(mmap(NULL, page_size, PROT_READ | PROT_WRITE,
+      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+  REQUIRE(mem_code != nullptr);
+  *mem_code = 0xC3; // ret
+  ((void (*)())(mem_code))();
+
+#ifdef __i386__
+  REQUIRE(got_signal == false);
+#else
+  REQUIRE(got_signal == true);
+#endif
+
+  munmap(mem_code, page_size);
+}

unittests/FEXLinuxTests/tests/smc/smc-unexec-stack.cpp

@@ -0,0 +1,30 @@
+#include <catch2/catch_test_macros.hpp>
+#include <signal.h>
+#include <ucontext.h>
+#include <sys/mman.h>
+
+bool got_signal = false;
+void *tramp = nullptr;
+
+static void sigsegv_handler(int signal, siginfo_t *siginfo, void* context) {
+  REQUIRE(siginfo->si_code == SEGV_ACCERR);
+  got_signal = true;
+  size_t page_size = sysconf(_SC_PAGESIZE);
+  void *fault_addr = (void *)((uintptr_t)(siginfo->si_addr) & ~(page_size - 1));
+  REQUIRE(mprotect(fault_addr, page_size, PROT_READ | PROT_WRITE | PROT_EXEC) == 0);
+}
+
+TEST_CASE("smc-unexec-stack: PT_GNU_STACK == RW") {
+  // Register signal handler
+  struct sigaction act {};
+  act.sa_sigaction = sigsegv_handler;
+  act.sa_flags = SA_SIGINFO;
+  sigaction(SIGSEGV, &act, nullptr);
+
+  // Try executing from stack
+  uint8_t stack_code = 0xC3; // ret instruction
+  tramp = (void *)((uintptr_t)&stack_code & ~(sysconf(_SC_PAGESIZE) - 1));
+  ((void (*)())(&stack_code))();
+
+  REQUIRE(got_signal == true);
+}