Fix vulkan proc addr bug in XeSS

ThunkLibs/libvulkan/Guest.cpp

@@ -64,6 +64,10 @@ static PFN_vkVoidFunction MakeGuestCallable(const char* origin, PFN_vkVoidFuncti
 }
 
 PFN_vkVoidFunction vkGetDeviceProcAddr(VkDevice a_0, const char* a_1) {
+  // The spec requires a self-lookup to succeed, which XeSS relies on.
+  if (a_1 == std::string_view {"vkGetDeviceProcAddr"}) {
+    return (PFN_vkVoidFunction)vkGetDeviceProcAddr;
+  }
   auto Ret = fexfn_pack_vkGetDeviceProcAddr(a_0, a_1);
   if (!Ret) {
     return nullptr;
@@ -74,6 +78,8 @@ PFN_vkVoidFunction vkGetDeviceProcAddr(VkDevice a_0, const char* a_1) {
 PFN_vkVoidFunction vkGetInstanceProcAddr(VkInstance a_0, const char* a_1) {
   if (a_1 == std::string_view {"vkGetDeviceProcAddr"}) {
     return (PFN_vkVoidFunction)vkGetDeviceProcAddr;
+  } else if (a_1 == std::string_view {"vkGetInstanceProcAddr"}) {
+    return (PFN_vkVoidFunction)vkGetInstanceProcAddr;
   } else {
     auto Ret = fexfn_pack_vkGetInstanceProcAddr(a_0, a_1);
     if (!Ret) {

unittests/FEXLinuxTests/CMakeLists.txt

@@ -38,7 +38,7 @@ function(AddTests Tests BinDirectory Bitness)
     set(BIN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${BinDirectory}/${TEST_NAME}.${Bitness}")
     set(TEST_CASE "${TEST_NAME}.${Bitness}")
 
-    if(TEST_NAME STREQUAL "thunk_testlib")
+    if(TEST_NAME STREQUAL "thunk_testlib" OR TEST_NAME STREQUAL "vulkan_procaddr")
       # Test thunking only if thunks are enabled and supported
       if(NOT BUILD_THUNKS OR ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)
         continue()
@@ -63,7 +63,12 @@ function(AddTests Tests BinDirectory Bitness)
       set_property(TEST "${TEST_CASE}.jit.flt" APPEND PROPERTY ENVIRONMENT "FEX_THUNKCONFIG=${CMAKE_SOURCE_DIR}/Data/CI/FEXLinuxTestsThunks.json")
     endif()
 
-    if (ARCHITECTURE_x86_64 AND NOT TEST_NAME STREQUAL "thunk_testlib")
+    if(TEST_NAME STREQUAL "vulkan_procaddr")
+      set_property(TEST "${TEST_CASE}.jit.flt" APPEND PROPERTY ENVIRONMENT
+        "FEX_THUNKCONFIG=${CMAKE_SOURCE_DIR}/Data/CI/VulkanThunks.json;FEX_THUNKHOSTLIBS=${HOSTLIBS_DATA_DIRECTORY}/HostThunks;FEX_THUNKGUESTLIBS=${CMAKE_INSTALL_PREFIX}/share/fex-emu/GuestThunks")
+    endif()
+
+    if (ARCHITECTURE_x86_64 AND NOT TEST_NAME STREQUAL "thunk_testlib" AND NOT TEST_NAME STREQUAL "vulkan_procaddr")
       # Add host test case
       add_test(NAME "${TEST_CASE}.host.flt"
         COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"

unittests/FEXLinuxTests/tests/CMakeLists.txt

@@ -45,6 +45,9 @@ target_link_libraries(smc-shared-2.${BITNESS} PRIVATE rt pthread)
 
 target_link_libraries(thunk_testlib.${BITNESS} PRIVATE ${CMAKE_DL_LIBS})
 
+target_link_libraries(vulkan_procaddr.${BITNESS} PRIVATE ${CMAKE_DL_LIBS})
+target_include_directories(vulkan_procaddr.${BITNESS} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../../External/Vulkan-Headers/include")
+
 target_link_libraries(timer-sigev-thread.${BITNESS} PRIVATE rt pthread)
 
 target_link_libraries(smc-unexec-stack.${BITNESS} PRIVATE -Wl,-z,noexecstack)

unittests/FEXLinuxTests/tests/thunks/vulkan_procaddr.cpp

@@ -0,0 +1,60 @@
+// Regression test for self-lookups in vkGetInstanceProcAddr and vkGetDeviceProcAddr
+// XeSS relies on this behavior, and not emulating it correctly caused a crash in Doom: The Dark Ages
+
+#include <dlfcn.h>
+
+#include <catch2/catch_test_macros.hpp>
+
+#define VK_NO_PROTOTYPES
+#include <vulkan/vulkan.h>
+
+TEST_CASE("Vulkan ProcAddr self-lookups") {
+  void* lib = dlopen("libvulkan.so.1", RTLD_NOW);
+  if (!lib) {
+    WARN("libvulkan.so.1 not available, nothing to test");
+    return;
+  }
+  auto gipa = (PFN_vkGetInstanceProcAddr)dlsym(lib, "vkGetInstanceProcAddr");
+  auto gdpa = (PFN_vkGetDeviceProcAddr)dlsym(lib, "vkGetDeviceProcAddr");
+  REQUIRE(gipa);
+  REQUIRE(gdpa);
+
+  CHECK(gipa(nullptr, "vkGetInstanceProcAddr") != nullptr);
+
+  VkApplicationInfo app {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, .apiVersion = VK_API_VERSION_1_2};
+  VkInstanceCreateInfo instance_info {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, .pApplicationInfo = &app};
+  VkInstance instance {};
+  if (((PFN_vkCreateInstance)gipa(nullptr, "vkCreateInstance"))(&instance_info, nullptr, &instance) != VK_SUCCESS) {
+    WARN("vkCreateInstance failed. No Vulkan driver installed?");
+    return;
+  }
+
+  CHECK(gipa(instance, "vkGetInstanceProcAddr") != nullptr);
+  CHECK(gipa(instance, "vkGetDeviceProcAddr") != nullptr);
+
+  uint32_t count = 1;
+  VkPhysicalDevice physical {};
+  ((PFN_vkEnumeratePhysicalDevices)gipa(instance, "vkEnumeratePhysicalDevices"))(instance, &count, &physical);
+  if (count == 0) {
+    WARN("no Vulkan physical device. No Vulkan driver installed?");
+    ((PFN_vkDestroyInstance)gipa(instance, "vkDestroyInstance"))(instance, nullptr);
+    return;
+  }
+
+  const float priority = 1.0f;
+  VkDeviceQueueCreateInfo queue {.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, .queueCount = 1, .pQueuePriorities = &priority};
+  VkDeviceCreateInfo device_info {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, .queueCreateInfoCount = 1, .pQueueCreateInfos = &queue};
+  VkDevice device {};
+  REQUIRE(((PFN_vkCreateDevice)gipa(instance, "vkCreateDevice"))(physical, &device_info, nullptr, &device) == VK_SUCCESS);
+
+  // This checks that XeSS will get the answer it expects.
+  auto self = (PFN_vkGetDeviceProcAddr)gdpa(device, "vkGetDeviceProcAddr");
+  REQUIRE(self != nullptr);
+  CHECK(self(device, "vkDestroyDevice") != nullptr);
+  
+  // XeSS doesn't seem to rely on this function, but let's make sure it works correctly anyways.
+  CHECK(gdpa(device, "vkGetInstanceProcAddr") == nullptr);
+
+  ((PFN_vkDestroyDevice)gdpa(device, "vkDestroyDevice"))(device, nullptr);
+  ((PFN_vkDestroyInstance)gipa(instance, "vkDestroyInstance"))(instance, nullptr);
+}