Merge pull request #1984 from Sonicadvance1/functional_thunk_ci

Thunks: Adds functional thunk testing to CI

.github/workflows/ccpp.yml

@@ -64,7 +64,7 @@ jobs:
       # Note the current convention is to use the -S and -B options here to specify source
       # and build directories, but this is only available with CMake 3.13 and higher.
       # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DENABLE_INTERPRETER=True -DBUILD_FEX_LINUX_TESTS=True -DBUILD_THUNKS=True
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DENABLE_INTERPRETER=True -DBUILD_FEX_LINUX_TESTS=True -DBUILD_THUNKS=True -DCMAKE_INSTALL_PREFIX=${{runner.workspace}}/build/install
 
     - name: Build
       working-directory: ${{runner.workspace}}/build
@@ -188,6 +188,40 @@ jobs:
       working-directory: ${{runner.workspace}}/build
       run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ThunkgenTests.log || true
 
+    - name: Install
+      if: matrix.arch[1] == 'x64'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE --target install
+
+    - name: Test GL No-Thunks
+      if: matrix.arch[1] == 'x64'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      env:
+        DISPLAY: ":0"
+      run: cmake --build . --config $BUILD_TYPE --target thunk_functional_tests_nothunks
+
+    - name: No thunks Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_NoThunkResults.log || true
+
+    - name: Test GL Thunks
+      if: matrix.arch[1] == 'x64'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      env:
+        DISPLAY: ":0"
+      run: cmake --build . --config $BUILD_TYPE --target thunk_functional_tests_thunks
+
+    - name: Thunks Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ThunkResults.log || true
+
     - name: Truncate test results
       if: ${{ always() }}
       shell: bash
@@ -207,4 +241,3 @@ jobs:
         name: Results-${{ env.runner_name }}
         path: ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log
         retention-days: 3
-

CI/GLThunks.json

@@ -0,0 +1,5 @@
+{
+  "ThunksDB": {
+    "GL": 1
+  }
+}

CI/VulkanThunks.json

@@ -0,0 +1,5 @@
+{
+  "ThunksDB": {
+    "Vulkan": 1
+  }
+}

Scripts/CI_FetchRootFS.py

@@ -83,6 +83,12 @@ def HashFile(file):
 
     return int.from_bytes(x.digest(), "big")
 
+def RemoveRootFSFolder(RootFSPath):
+    print("Removing previous rootfs extraction before copying")
+    shutil.rmtree(RootFSPath, ignore_errors = True)
+    # Recreate the folder
+    os.makedirs(RootFSPath)
+
 def CheckFilesystemForFS(RootFSMountPath, RootFSPath, DistroFit):
     # Check if rootfs mount path exists
     if (not os.path.exists(RootFSMountPath) or
@@ -105,6 +111,7 @@ def CheckFilesystemForFS(RootFSMountPath, RootFSPath, DistroFit):
     MountRootFSImagePath = RootFSMountPath + DistroFit[3]
     RootFSImagePath = RootFSPath + "/" + os.path.basename(DistroFit[3])
     NeedsExtraction = False
+    PreviouslyExistingRootFS = False
 
     if not os.path.exists(MountRootFSImagePath):
         print("Image {} doesn't exist".format(MountRootFSImagePath))
@@ -113,29 +120,39 @@ def CheckFilesystemForFS(RootFSMountPath, RootFSPath, DistroFit):
     if not os.path.exists(RootFSImagePath):
         # Copy over
         print("RootFS image doesn't exist. Copying")
-        shutil.copyfile(MountRootFSImagePath, RootFSImagePath)
-        NeedsExtraction = True
-
-    # Now hash the image
-    RootFSHash = HashFile(RootFSImagePath)
-    if RootFSHash != DistroFit[4]:
-        print("Hash {} did not match {}, copying new image".format(hex(RootFSHash), hex(DistroFit[4])))
+        RemoveRootFSFolder(RootFSPath)
         shutil.copyfile(MountRootFSImagePath, RootFSImagePath)
         NeedsExtraction = True
 
     # Check if the image needs to be extracted
     if not os.path.exists(RootFSPath + "/usr"):
         NeedsExtraction = True
+    else:
+        PreviouslyExistingRootFS = True
+
+    # Now hash the image
+    RootFSHash = HashFile(RootFSImagePath)
+    if RootFSHash != DistroFit[4]:
+        print("Hash {} did not match {}, copying new image".format(hex(RootFSHash), hex(DistroFit[4])))
+
+        if PreviouslyExistingRootFS:
+            RemoveRootFSFolder(RootFSPath)
+
+        shutil.copyfile(MountRootFSImagePath, RootFSImagePath)
+        NeedsExtraction = True
 
     if NeedsExtraction:
         print("Extracting rootfs")
+
         CmdResult = subprocess.call(["unsquashfs", "-f", "-d", RootFSPath, RootFSImagePath])
         if CmdResult != 0:
-            print("Couldn't extract squashfs")
+            print("Couldn't extract squashfs. Removing image file to be safe")
+            os.remove(RootFSImagePath)
             return False
 
     if not os.path.exists(RootFSPath + "/usr"):
-        print("Couldn't extract squashfs")
+        print("Couldn't extract squashfs. Removing image file to be safe")
+        os.remove(RootFSImagePath)
         return False
 
     print("RootFS successfully checked and extracted")

ThunkLibs/libvulkan/libvulkan_interface.cpp

@@ -1,7 +1,9 @@
 #include <common/GeneratorInterface.h>
 
 template<auto>
-struct fex_gen_config;
+struct fex_gen_config {
+    unsigned version = 1;
+};
 
 #define VK_USE_PLATFORM_XLIB_XRANDR_EXT
 #define VK_USE_PLATFORM_XLIB_KHR

unittests/CMakeLists.txt

@@ -6,8 +6,10 @@ add_subdirectory(POSIX/)
 add_subdirectory(gvisor-tests/)
 add_subdirectory(gcc-target-tests-32/)
 add_subdirectory(gcc-target-tests-64/)
+
 if (BUILD_THUNKS)
   add_subdirectory(ThunkLibs)
+  add_subdirectory(ThunkFunctionalTests)
 endif()
 
 if (BUILD_FEX_LINUX_TESTS)

unittests/ThunkFunctionalTests/CMakeLists.txt

@@ -0,0 +1,57 @@
+set(FUNCTIONAL_DEPENDS "")
+
+function(AddThunksTest Bin ThunksFile)
+  set (ARGS
+    "-t" "${CMAKE_INSTALL_PREFIX}/lib/fex-emu/HostThunks"
+    "-j" "${CMAKE_INSTALL_PREFIX}/share/fex-emu/GuestThunks"
+    "-o" "stderr" "--no-silent" "-c" "irjit" "-n" "500"
+    )
+  if (NOT ThunksFile)
+    set (TEST_NAME ThunkFunctionalTest-NoThunks-${Bin})
+  else()
+    set (TEST_NAME ThunkFunctionalTest-Thunks-${Bin})
+    list (APPEND ARGS
+      "-k" "${CMAKE_SOURCE_DIR}/CI/${ThunksFile}")
+  endif()
+
+  add_test(NAME ${TEST_NAME}
+    COMMAND "$<TARGET_FILE:FEXLoader>"
+    ${ARGS}
+    "--"
+    ${Bin})
+    set_property(TEST ${TEST_NAME} APPEND PROPERTY DEPENDS "${Bin}")
+
+  list(APPEND FUNCTIONAL_DEPENDS "${TEST_NAME}")
+endfunction()
+
+function(AddTest Bin ThunksFile)
+  AddThunksTest("${Bin}" "")
+  AddThunksTest("${Bin}" "${ThunksFile}")
+endfunction()
+
+AddTest("/usr/bin/glxinfo" "GLThunks.json")
+AddTest("/usr/bin/vulkaninfo" "VulkanThunks.json")
+
+execute_process(COMMAND "nproc" OUTPUT_VARIABLE CORES)
+string(STRIP ${CORES} CORES)
+
+add_custom_target(
+  thunk_functional_tests_nothunks
+  WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+  USES_TERMINAL
+  COMMAND "ctest" "--timeout" "302" "-j${CORES}" "-R" "ThunkFunctionalTest-NoThunks-\.*"
+  DEPENDS "${FUNCTIONAL_DEPENDS}")
+
+add_custom_target(
+  thunk_functional_tests_thunks
+  WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+  USES_TERMINAL
+  COMMAND "ctest" "--timeout" "302" "-j${CORES}" "-R" "ThunkFunctionalTest-Thunks-\.*"
+  DEPENDS "${FUNCTIONAL_DEPENDS}")
+
+add_custom_target(
+  thunk_functional_tests
+  WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+  USES_TERMINAL
+  COMMAND "ctest" "--timeout" "302" "-j${CORES}" "-R" "ThunkFunctionalTest\.*"
+  DEPENDS "${FUNCTIONAL_DEPENDS}")