CMake: Check for binfmt_misc conflicts before install

Check for qemu and box binfmt_misc file conflicts before the
`binfmt_misc` install command.

This ensures if you're building from source that you won't inadvertently
install conflicting binfmt_misc files, breaking program execution.

Scripts/CheckBinfmtNotInstall.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+for binfmt in "$@"
+do
+  result=0
+  if command -v update-binfmts>/dev/null; then
+    update-binfmts --find "$binfmt" 1>&- 2>&-
+    if [ $? -eq 0 ]
+    then
+      # If we found the binfmt_misc file passed in then error
+      result=1
+    fi
+  fi
+
+  if [ $result -eq 0 ]
+  then
+    if [ -f "$binfmt" ]; then
+      # If the binfmt_misc file exists then error
+      result=1
+    fi
+  fi
+
+  if [ $result -eq 1 ]
+  then
+    echo "==============================================================="
+    echo "$binfmt binfmt file is installed!"
+    echo "This conflicts with FEX-Emu's binfmt_misc!"
+    echo "This will cause issues when running FEX-Emu through binfmt_misc"
+    echo "Not installing until you uninstall this binfmt_misc file!"
+    echo "==============================================================="
+    exit 1
+  fi
+done
+
+exit 0

Source/Tests/CMakeLists.txt

@@ -67,10 +67,19 @@ endif()
 install(PROGRAMS "${PROJECT_SOURCE_DIR}/Scripts/FEXUpdateAOTIRCache.sh" DESTINATION bin RENAME FEXUpdateAOTIRCache)
 
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+  # Check for conflicting binfmt before installing
+  set (CONFLICTING_BINFMTS_32
+    ${CMAKE_INSTALL_PREFIX}/share/binfmts/qemu-i386
+    ${CMAKE_INSTALL_PREFIX}/share/binfmts/box86)
+  set (CONFLICTING_BINFMTS_64
+    ${CMAKE_INSTALL_PREFIX}/share/binfmts/qemu-x86_64
+    ${CMAKE_INSTALL_PREFIX}/share/binfmts/box64)
+
   find_program(UPDATE_BINFMTS_PROGRAM update-binfmts)
   if (UPDATE_BINFMTS_PROGRAM)
     add_custom_target(binfmt_misc_32
       echo "Attempting to install FEX-x86 misc now."
+      COMMAND "${CMAKE_SOURCE_DIR}/Scripts/CheckBinfmtNotInstall.sh" ${CONFLICTING_BINFMTS_32}
       COMMAND "update-binfmts" "--importdir=${CMAKE_INSTALL_PREFIX}/share/binfmts/" "--import" "FEX-x86"
       COMMAND ${CMAKE_COMMAND} -E
       echo "binfmt_misc FEX-x86 installed"
@@ -79,6 +88,7 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
     add_custom_target(binfmt_misc_64
       COMMAND ${CMAKE_COMMAND} -E
       echo "Attempting to install FEX-x86_64 misc now."
+      COMMAND "${CMAKE_SOURCE_DIR}/Scripts/CheckBinfmtNotInstall.sh" ${CONFLICTING_BINFMTS_64}
       COMMAND "update-binfmts" "--importdir=${CMAKE_INSTALL_PREFIX}/share/binfmts/" "--import" "FEX-x86_64"
       COMMAND ${CMAKE_COMMAND} -E
       echo "binfmt_misc FEX-x86_64 installed"