CI: Adds support for flakes

If a test is marked as a flake then it will be tried five times before
giving up.

Works around the problem of needing to babysit CI once a PR is pushed.
As long as we have all the flake tests marked.

Scripts/guest_test_runner.py

@@ -3,44 +3,55 @@ import os
 import sys
 import subprocess
 
-# Args: <Known Failures file> <ExpectedOutputsFile> <DisabledTestsFile> <TestName> <FexExecutable> <FexArgs>...
+def LoadTestsFile(File):
+    Dict = {}
+    if not os.path.exists(File):
+        return Dict
+
+    with open(File) as dtf:
+        for line in dtf:
+            test = line.split("#")[0].strip() # remove comments and empty spaces
+            if len(test) > 0:
+                Dict[test] = 1
+
+    return Dict
+
+def LoadTestsFileResults(File):
+    Dict = {}
+    if not os.path.exists(File):
+        return Dict
+
+    with open(File) as dtf:
+        for line in dtf:
+            test = line.split("#")[0].strip() # remove comments and empty spaces
+            if len(test) > 0:
+                parts = line.split(" ")
+                Dict[parts[0]] = int(parts[1])
+
+    return Dict
+
+
+# Args: <Known Failures file> <ExpectedOutputsFile> <DisabledTestsFile> <FlakeTestsFile> <TestName> <Mode> <FexExecutable> <FexArgs>...
 
 # fexargs should also include the test executable
 
-if (len(sys.argv) < 6):
+if (len(sys.argv) < 7):
     sys.exit()
 
 known_failures_file = sys.argv[1]
 expected_output_file = sys.argv[2]
 disabled_tests_file = sys.argv[3]
-test_name = sys.argv[4]
-mode = sys.argv[5]
-fexecutable = sys.argv[6]
+flake_tests_file = sys.argv[4]
+test_name = sys.argv[5]
+mode = sys.argv[6]
+fexecutable = sys.argv[7]
+StartingFEXArgsOffset = 8
 
-known_failures = { }
-expected_output = { }
-disabled_tests = { }
-
-# Open the known failures file and add it to a dictionary
-with open(known_failures_file) as kff:
-    for line in kff:
-        test = line.split("#")[0].strip() # remove comments and empty spaces
-        if len(test) > 0:
-            known_failures[test] = 1
-
-# Open expected outputs and add it to dictionary
-with open(expected_output_file) as eof:
-    for line in eof:
-        line = test = line.split("#")[0].strip() # remove comments and empty spaces
-        if len(line) > 0:
-            parts = line.split(" ")
-            expected_output[parts[0]] = int(parts[1])
-
-with open(disabled_tests_file) as dtf:
-    for line in dtf:
-        test = line.split("#")[0].strip() # remove comments and empty spaces
-        if len(test) > 0:
-            disabled_tests[test] = 1
+# Open test expected information files and load in to dictionaries.
+known_failures = LoadTestsFile(known_failures_file)
+expected_output = LoadTestsFileResults(expected_output_file)
+disabled_tests = LoadTestsFile(disabled_tests_file)
+flake_tests = LoadTestsFile(flake_tests_file)
 
 # run with timeout to avoid locking up
 RunnerArgs = []
@@ -54,25 +65,37 @@ if (mode == "guest"):
         RunnerArgs.append(ROOTFS_ENV)
 
 # Add the rest of the arguments
-for i in range(len(sys.argv) - 7):
-    RunnerArgs.append(sys.argv[7 + i])
+for i in range(len(sys.argv) - StartingFEXArgsOffset):
+    RunnerArgs.append(sys.argv[StartingFEXArgsOffset + i])
 
 #print(RunnerArgs)
 
 ResultCode = 0
 
+# Handle flakes
+TryCount = 1
+if (flake_tests.get(test_name)):
+    TryCount = 5
+
 if (disabled_tests.get(test_name)):
     ResultCode = -73
-else:
-    # Run the test and wait for it to end to get the result
-    Process = subprocess.Popen(RunnerArgs)
-    Process.wait()
-    ResultCode = Process.returncode
 
 # expect zero by default
 if (not test_name in expected_output):
     expected_output[test_name] = 0
 
+if ResultCode == 0:
+    for Try in range(TryCount):
+        # Run the test and wait for it to end to get the result
+        print(RunnerArgs)
+        Process = subprocess.Popen(RunnerArgs)
+        Process.wait()
+        ResultCode = Process.returncode
+
+        # Break if the expected output is the result code
+        if (expected_output[test_name] == ResultCode):
+            break
+
 if (expected_output[test_name] != ResultCode):
     if (test_name in expected_output):
         print("test failed, expected is", expected_output[test_name], "but got", ResultCode)

unittests/FEXLinuxTests/CMakeLists.txt

@@ -62,6 +62,7 @@ function(AddTests Tests BinDirectory BitnessList)
           "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures"
           "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
           "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests"
+          "${CMAKE_CURRENT_SOURCE_DIR}/Flake_Tests"
           "${TEST_CASE}"
           "guest"
           "$<TARGET_FILE:FEXLoader>"
@@ -75,6 +76,7 @@ function(AddTests Tests BinDirectory BitnessList)
             "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures_Host"
             "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
             "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests_Host"
+            "${CMAKE_CURRENT_SOURCE_DIR}/Flake_Tests"
             "${TEST_CASE}"
             "host"
             "${BIN_PATH}"

unittests/FEXLinuxTests/Flake_Tests

@@ -0,0 +1,5 @@
+smc-mt-1.32
+smc-mt-2.32
+smc-mt-1.64
+smc-mt-2.64
+pthread_cancel.64

unittests/POSIX/CMakeLists.txt

@@ -12,9 +12,10 @@ foreach(POSIX_TEST ${POSIX_TESTS})
   if (ENABLE_INTERPRETER)
     add_test(NAME "${TEST_NAME}.int.posix"
       COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
-      "${CMAKE_SOURCE_DIR}/unittests/POSIX/Known_Failures"
-      "${CMAKE_SOURCE_DIR}/unittests/POSIX/Expected_Output"
-      "${CMAKE_SOURCE_DIR}/unittests/POSIX/Disabled_Tests"
+      "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures"
+      "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
+      "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests"
+      "${CMAKE_CURRENT_SOURCE_DIR}/Flake_Tests"
       "${TEST_NAME}"
       "guest"
       "${CMAKE_BINARY_DIR}/Bin/FEXLoader"
@@ -24,9 +25,10 @@ foreach(POSIX_TEST ${POSIX_TESTS})
 
   add_test(NAME "${TEST_NAME}.jit.posix"
     COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
-    "${CMAKE_SOURCE_DIR}/unittests/POSIX/Known_Failures"
-    "${CMAKE_SOURCE_DIR}/unittests/POSIX/Expected_Output"
-    "${CMAKE_SOURCE_DIR}/unittests/POSIX/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Flake_Tests"
     "${TEST_NAME}"
     "guest"
     "${CMAKE_BINARY_DIR}/Bin/FEXLoader"

unittests/POSIX/Flake_Tests

@@ -0,0 +1,3 @@
+# CPU scheduling can cause these tests to take > 10ms for their error margin.
+conformance-interfaces-sigtimedwait-1-1.test
+conformance-interfaces-sigtimedwait-2-1.test

unittests/gcc-target-tests-32/CMakeLists.txt

@@ -13,9 +13,10 @@ foreach(TEST ${TESTS})
 
   add_test(NAME "${TEST_NAME}.jit.gcc-target-32"
     COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
-    "${CMAKE_SOURCE_DIR}/unittests/gcc-target-tests-32/Known_Failures"
-    "${CMAKE_SOURCE_DIR}/unittests/gcc-target-tests-32/Expected_Output"
-    "${CMAKE_SOURCE_DIR}/unittests/gcc-target-tests-32/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Flake_Tests"
     "${TEST_NAME}"
     "guest"
     "${CMAKE_BINARY_DIR}/Bin/FEXLoader"

unittests/gcc-target-tests-64/CMakeLists.txt

@@ -13,9 +13,10 @@ foreach(TEST ${TESTS})
 
   add_test(NAME "${TEST_NAME}.jit.gcc-target-64"
     COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
-    "${CMAKE_SOURCE_DIR}/unittests/gcc-target-tests-64/Known_Failures"
-    "${CMAKE_SOURCE_DIR}/unittests/gcc-target-tests-64/Expected_Output"
-    "${CMAKE_SOURCE_DIR}/unittests/gcc-target-tests-64/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Flake_Tests"
     "${TEST_NAME}"
     "guest"
     "${CMAKE_BINARY_DIR}/Bin/FEXLoader"

unittests/gvisor-tests/CMakeLists.txt

@@ -13,9 +13,10 @@ foreach(TEST ${TESTS})
 
   add_test(NAME "${TEST_NAME}.jit.gvisor"
     COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
-    "${CMAKE_SOURCE_DIR}/unittests/gvisor-tests/Known_Failures"
-    "${CMAKE_SOURCE_DIR}/unittests/gvisor-tests/Expected_Output"
-    "${CMAKE_SOURCE_DIR}/unittests/gvisor-tests/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Flake_Tests"
     "${TEST_NAME}"
     "guest"
     "${CMAKE_BINARY_DIR}/Bin/FEXLoader"

unittests/gvisor-tests/Flake_Tests

@@ -0,0 +1 @@
+pselect_test