diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index 7f46aec65..fc87251ec 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -113,13 +113,24 @@ jobs: working-directory: ${{runner.workspace}}/build run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_GCC64.log || true + - name: Struct verifier tests + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target struct_verifier + + - name: Struct verifier Test Results move + if: ${{ always() }} + shell: bash + working-directory: ${{runner.workspace}}/build + run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_StructVerifier.log || true + - name: Truncate test results if: ${{ always() }} shell: bash working-directory: ${{runner.workspace}}/build # Cap out the log files at 20M in case something crash spins and dumps fault text # ASM tests get quite close to 10MB - run: truncate --size=20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true + run: truncate --size=<20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true - name: Set runner name if: ${{ always() }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 0149a0435..897c5f952 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,14 +234,16 @@ endif() add_compile_options(-Wall) +if (BUILD_TESTS) + include(CTest) + enable_testing() + message(STATUS "Unit tests are enabled") +endif() add_subdirectory(External/FEXCore) add_subdirectory(Source/) if (BUILD_TESTS) - include(CTest) - enable_testing() - message(STATUS "Unit tests are enabled") add_subdirectory(unittests/) endif() diff --git a/Scripts/StructPackVerifier.py b/Scripts/StructPackVerifier.py new file mode 100755 index 000000000..d49dfd5d5 --- /dev/null +++ b/Scripts/StructPackVerifier.py @@ -0,0 +1,661 @@ +#!/usr/bin/python3 +import clang.cindex +from clang.cindex import CursorKind +from clang.cindex import TypeKind +from clang.cindex import TranslationUnit +import sys +from dataclasses import dataclass, field +import subprocess +import logging +logger = logging.getLogger() +logger.setLevel(logging.WARNING) + +@dataclass +class TypeDefinition: + TYPE_UNKNOWN = 0 + TYPE_STRUCT = 1 + TYPE_UNION = 2 + TYPE_FIELD = 3 + + name: str + type: int + def __init__(self, Name, Type): + self.name = Name + self.type = Type + + @property + def Name(self): + return self.name + @property + def Type(self): + return self.type + +@dataclass +class AliasType: + ALIAS_X86_32 = 0 + ALIAS_X86_64 = 1 + ALIAS_AARCH64 = 2 + ALIAS_WIN32 = 3 + ALIAS_WIN64 = 4 + Name: str + AliasType: int + def __init__(self, Name, Type): + self.Name = Name + self.AliasType = Type + +@dataclass +class StructDefinition(TypeDefinition): + Size: int + Aliases: list + Members: list + ExpectFEXMatch: bool + + def __init__(self, Name, Size): + super(StructDefinition, self).__init__(Name, TypeDefinition.TYPE_STRUCT) + self.Size = Size + self.Aliases = [] + self.Members = [] + self.ExpectFEXMatch = False + +@dataclass +class UnionDefinition(TypeDefinition): + Size: int + Aliases: list + Members: list + ExpectFEXMatch: bool + + def __init__(self, Name, Size): + super(UnionDefinition, self).__init__(Name, TypeDefinition.TYPE_UNION) + self.Size = Size + self.Aliases = [] + self.Members = [] + self.ExpectFEXMatch = False + +@dataclass +class FieldDefinition(TypeDefinition): + Size: int + OffsetOf: int + Alignment: int + def __init__(self, Name, Size, OffsetOf, Alignment): + super(FieldDefinition, self).__init__(Name, TypeDefinition.TYPE_FIELD) + self.Size = Size + self.OffsetOf = OffsetOf + self.Alignment = Alignment + +@dataclass +class ArchDB: + Parsed: bool + ArchName: str + NamespaceScope: list + CurrentNamespace: str + TU: TranslationUnit + Structs: dict + Unions: dict + FieldDecls: list + def __init__(self, ArchName): + self.Parsed = True + self.ArchName = ArchName + self.NamespaceScope = [] + self.CurrentNamespace = "" + self.TU = None + self.Structs = {} + self.Unions = {} + self.FieldDecls = [] + +class DBList: + DBs: list + def __init__(self, DB32, DB64, DBAArch64, DBWin32, DBWin64): + self.DBs = [DB32, DB64, DBAArch64, DBWin32, DBWin64] + +def FindClangArguments(OriginalArguments): + AddedArguments = ["clang"] + AddedArguments.extend(OriginalArguments) + AddedArguments.extend(["-v", "-x", "c++", "-S", "-"]) + Proc = subprocess.Popen(AddedArguments, stderr = subprocess.PIPE, stdin = subprocess.DEVNULL) + NewIncludes = [] + BeginSearch = False + while True: + Line = Proc.stderr.readline().strip() + + if not Line: + Proc.terminate() + break + + if (Line == b"End of search list."): + BeginSearch = False + Proc.terminate() + break + + if (BeginSearch == True): + NewIncludes.append("-I" + Line.decode('ascii')) + + if (Line == b"#include <...> search starts here:"): + BeginSearch = True + + # Add back original arguments + NewIncludes.extend(OriginalArguments) + return NewIncludes + +def SetNamespace(Arch): + Arch.CurrentNamespace = "" + for Namespace in Arch.NamespaceScope: + Arch.CurrentNamespace = Arch.CurrentNamespace + Namespace + "::" + +def HandleStructDeclCursor(Arch, Cursor, NameOverride = ""): + # Append namespace + CursorName = "" + StructType = Cursor.type + if (len(StructType.spelling) == 0): + CursorName = NameOverride + else: + CursorName = StructType.spelling + + if (len(CursorName) != 0): + Arch.NamespaceScope.append(CursorName) + SetNamespace(Arch) + + Struct = StructDefinition( + Name = CursorName, + Size = StructType.get_size()) + + # Handle children + Arch.Structs[Struct.Name] = HandleStructElements(Arch, Struct, Cursor) + + # Pop namespace off + if (len(CursorName) != 0): + Arch.NamespaceScope.pop() + SetNamespace(Arch) + + return Arch + +def HandleUnionDeclCursor(Arch, Cursor, NameOverride = ""): + # Append namespace + CursorName = "" + + if (len(Cursor.spelling) == 0): + CursorName = NameOverride + else: + CursorName = Cursor.spelling + + if (len(CursorName) != 0): + Arch.NamespaceScope.append(CursorName) + SetNamespace(Arch) + + UnionType = Cursor.type + Union = UnionDefinition( + Name = CursorName, + Size = UnionType.get_size()) + Arch.Unions[Union.Name] = Union + + # Handle children + Arch.Unions[Union.Name] = HandleStructElements(Arch, Union, Cursor) + + # Pop namespace off + if (len(CursorName) != 0): + Arch.NamespaceScope.pop() + SetNamespace(Arch) + + return Arch + +def HandleTypeDefDeclCursor(Arch, Cursor): + TypeDefType = Cursor.underlying_typedef_type + CanonicalType = TypeDefType.get_canonical() + + if (TypeDefType.kind == TypeKind.ELABORATED and CanonicalType.kind == TypeKind.RECORD): + TypeDefName = Cursor.type.get_typedef_name() + if (len(TypeDefName) != 0): + logging.info ("Found Typedef Decl'{0}'".format(TypeDefName)) + logging.info ("\tSize of type: {0}".format(CanonicalType.get_size())); + HandleTypeDefDecl(Arch, Cursor, TypeDefName) + + # Append namespace + Arch.NamespaceScope.append(TypeDefName) + SetNamespace(Arch) + + Arch = HandleCursor(Arch, Cursor) + #StructType = Cursor.type + #Struct = StructDefinition( + # Name = TypeDefName, + # Size = CanonicalType.get_size()) + #Arch.Structs[TypeDefName] = Struct + + ## Handle children + #Arch.Structs[TypeDefName] = HandleStructElements(Arch, Struct, Cursor) + + # Pop namespace off + Arch.NamespaceScope.pop() + SetNamespace(Arch) + return Arch + +def HandleStructElements(Arch, Struct, Cursor): + for Child in Cursor.get_children(): + logging.info ("\t\tStruct/Union Children: Cursor \"{0}{1}\" of kind {2}".format(Arch.CurrentNamespace, Child.spelling, Child.kind)) + if (Child.kind == CursorKind.ANNOTATE_ATTR): + if (Child.spelling.startswith("alias-")): + Sections = Child.spelling.split("-") + if (Sections[1] == "x86_32"): + Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_X86_32)) + elif (Sections[1] == "x86_64"): + Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_X86_64)) + elif (Sections[1] == "aarch64"): + Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_AARCH64)) + elif (Sections[1] == "win32"): + Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_WIN32)) + elif (Sections[1] == "win64"): + Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_WIN64)) + else: + logging.critical ("Can't handle alias type '{0}'".format(Child.spelling)) + elif (Child.spelling == "fex-match"): + Struct.ExpectedFEXMatch = True + else: + # Unknown annotation + pass + elif (Child.kind == CursorKind.FIELD_DECL): + ParentType = Cursor.type + FieldType = Child.type + Field = FieldDefinition( + Name = Child.spelling, + Size = FieldType.get_size(), + OffsetOf = ParentType.get_offset(Child.spelling), + Alignment = FieldType.get_align()) + + logging.info ("\t{0}".format(Child.spelling)) + logging.info ("\t\tSize of type: {0}".format(FieldType.get_size())); + logging.info ("\t\tAlignment of type: {0}".format(FieldType.get_align())); + logging.info ("\t\tOffsetof of type: {0}".format(ParentType.get_offset(Child.spelling))); + Struct.Members.append(Field) + Arch.FieldDecls.append(Field) + elif (Child.kind == CursorKind.STRUCT_DECL): + ParentType = Cursor.type + FieldType = Child.type + Field = FieldDefinition( + Name = Child.spelling, + Size = FieldType.get_size(), + OffsetOf = ParentType.get_offset(Child.spelling), + Alignment = FieldType.get_align()) + + logging.info ("\t{0}".format(Child.spelling)) + logging.info ("\t\tSize of type: {0}".format(FieldType.get_size())); + logging.info ("\t\tAlignment of type: {0}".format(FieldType.get_align())); + logging.info ("\t\tOffsetof of type: {0}".format(ParentType.get_offset(Child.spelling))); + Struct.Members.append(Field) + Arch.FieldDecls.append(Field) + Arch = HandleStructDeclCursor(Arch, Child) + elif (Child.kind == CursorKind.UNION_DECL): + Struct = HandleStructElements(Arch, Struct, Child) + #ParentType = Cursor.type + #FieldType = Child.type + #Field = FieldDefinition( + # Name = Child.spelling, + # Size = FieldType.get_size(), + # OffsetOf = ParentType.get_offset(Child.spelling), + # Alignment = FieldType.get_align()) + + #logging.info ("\t{0}".format(Child.spelling)) + #logging.info ("\t\tSize of type: {0}".format(FieldType.get_size())); + #logging.info ("\t\tAlignment of type: {0}".format(FieldType.get_align())); + #logging.info ("\t\tOffsetof of type: {0}".format(ParentType.get_offset(Child.spelling))); + #Struct.Members.append(Field) + #Arch.FieldDecls.append(Field) + #Arch = HandleUnionDeclCursor(Arch, Child) + elif (Child.kind == CursorKind.TYPEDEF_DECL): + Arch = HandleTypeDefDeclCursor(Arch, Child) + else: + Arch = HandleCursor(Arch, Child) + + return Struct + +def HandleTypeDefDecl(Arch, Cursor, Name): + for Child in Cursor.get_children(): + if (Child.kind == CursorKind.UNION_DECL): + pass + elif (Child.kind == CursorKind.STRUCT_DECL): + Arch = HandleStructDeclCursor(Arch, Child, Name) + elif (Child.kind == CursorKind.UNION_DECL): + Arch = HandleUnionDeclCursor(Arch, Child, Name) + elif (Child.kind == CursorKind.TYPEDEF_DECL): + Arch = HandleTypeDefDeclCursor(Arch, Child) + elif (Child.kind == CursorKind.TYPE_REF or + Child.kind == CursorKind.NAMESPACE_REF or + Child.kind == CursorKind.TEMPLATE_REF): + # Safe to pass on + pass + else: + logging.critical ("Unhandled TypedefDecl {0}-{1}-{2}".format(Child.kind, Child.type.spelling, Child.spelling)) + +def HandleCursor(Arch, Cursor): + if (Cursor.kind.is_invalid()): + Diags = TU.diagnostics + for Diag in Diags: + logging.warning (Diag.format()) + + Arch.Parsed = False + return + + for Child in Cursor.get_children(): + logging.info ("\tCursor \"{0}\" of kind {1}".format(Child.spelling, Child.kind)) + if (Child.kind == CursorKind.TRANSLATION_UNIT): + Arch = HandleCursor(Arch, Child) + elif (Child.kind == CursorKind.FIELD_DECL): + pass + elif (Child.kind == CursorKind.UNION_DECL): + Arch = HandleUnionDeclCursor(Arch, Child) + elif (Child.kind == CursorKind.STRUCT_DECL): + Arch = HandleStructDeclCursor(Arch, Child) + elif (Child.kind == CursorKind.TYPEDEF_DECL): + Arch = HandleTypeDefDeclCursor(Arch, Child) + elif (Child.kind == CursorKind.NAMESPACE): + # Append namespace + Arch.NamespaceScope.append(Child.spelling) + SetNamespace(Arch) + + # Handle children + Arch = HandleCursor(Arch, Child) + + # Pop namespace off + Arch.NamespaceScope.pop() + SetNamespace(Arch) + elif (Child.kind == CursorKind.TYPE_REF): + # Safe to pass on + pass + else: + Arch = HandleCursor(Arch, Child) + + return Arch + +def GetDB(Arch, filename, args): + Index = clang.cindex.Index.create() + try: + TU = Index.parse(filename, args=args, options=TranslationUnit.PARSE_INCOMPLETE) + except TranslationUnitLoadError: + Arch.Parsed = False + Diags = TU.diagnostics + for Diag in Diags: + logging.warning (Diag.format()) + + return + + Arch.TU = TU + HandleCursor(Arch, TU.cursor) + + # Get diagnostics + Diags = TU.diagnostics + if (len(Diags) != 0): + logging.warning ("Diagnostics from Arch: {0}".format(Arch.ArchName)) + + for Diag in Diags: + logging.warning (Diag.format()) + + return Arch + +def GetCompar(ComparisonName, DBs): + if (ComparisonName.lower() == "x86_32"): + return DBs.DBs[AliasType.ALIAS_X86_32] + elif (ComparisonName.lower() == "x86_64"): + return DBs.DBs[AliasType.ALIAS_X86_64] + elif (ComparisonName.lower() == "win32"): + return DBs.DBs[AliasType.ALIAS_WIN32] + elif (ComparisonName.lower() == "win64"): + return DBs.DBs[AliasType.ALIAS_WIN64] + elif (ComparisonName.lower() == "aarch64"): + return DBs.DBs[AliasType.ALIAS_AARCH64] + +def PrintMissingMembers(Struct1, Struct2): + for Member1 in Struct1.Members: + WasMissing = True + for Member2 in Struct2.Members: + if (Member1.Name == Member2.Name): + WasMissing = False + break + if (WasMissing): + logging.error ("\t'{0}' member '{1}' Doesn't exist in '{2}'".format(Struct1.Name, Member1.Name, Struct2.Name)); + +def CompareStructs(Struct1, Struct2): + HadWarning = False + HadError = False + + # Check if the struct size is a mismatch + if (Struct1.Size != Struct2.Size): + logging.warning ("\t#### Warning: Struct size mismatch. {0} != {1}".format(Struct1.Size, Struct2.Size)) + logging.warning ("\t\tMight not be a problem if struct isn't in used inside another struct, or end of object") + HadWarning = True + + # Check if the number of members differ + if (len(Struct1.Members) != len(Struct2.Members)): + logging.error ("@@@@ ERROR: Struct fields mismatch! Number of fields don't match! {0} != {1}".format(len(Struct1.Members), len(Struct2.Members))); + PrintMissingMembers(Struct1, Struct2) + PrintMissingMembers(Struct2, Struct1) + HadError = True + else: + # Compare the members themselves + for StructMemberIndex in range(0, len(Struct1.Members)): + Member1 = Struct1.Members[StructMemberIndex] + Member2 = Struct2.Members[StructMemberIndex] + if (Member1.Type == TypeDefinition.TYPE_FIELD): + if (Member1.Size != Member2.Size): + logging.error ("\t@@@@ ERROR: Member '{0}' mismatch Size! {1} != {2}".format(Member1.Name, Member1.Size, Member2.Size)); + HadError = True + if (Member1.OffsetOf != Member2.OffsetOf): + logging.error ("\t@@@@ ERROR: Member '{0}' mismatch OffsetOf! {1} != {2}".format(Member1.Name, Member1.OffsetOf, Member2.OffsetOf)); + HadError = True + if (Member1.Alignment != Member2.Alignment): + logging.error ("\t@@@@ ERROR: Member '{0}' mismatch Alignment! {1} != {2}".format(Member1.Name, Member1.Alignment, Member2.Alignment)); + logging.error ("\t\tProbably not a problem if offset and size matches"); + HadWarning = True + else: + logging.critical ("Oops, didn't handle member type {0}".format(Member1.Type)) + pass + + return not (HadWarning or HadError) + +def CompareAliases(DB, DBs): + Passed = True + for StructKey, StructDef in DB.Structs.items(): + if (len(StructKey) == 0): + # XXX: Oops, shouldn't have anonymous structs + continue + + logging.info ("Comparing Aliases {0}".format(StructDef.Name)) + + for Alias in StructDef.Aliases: + OtherDB = DBs.DBs[Alias.AliasType] + OtherStruct = OtherDB.Structs.get(Alias.Name) + if (OtherStruct == None): + logging.critical ("Couldn't find alias {0} in {1} DB".format(Alias.Name, OtherDB.ArchName)) + Passed = False + continue + + ThisAlias = CompareStructs(StructDef, OtherStruct) + if not (ThisAlias): + logging.error ("Couldn't Alias to Arch {0} successfully".format(OtherDB.ArchName)) + Passed &= ThisAlias + return Passed + +def CompareCrossArch(DB1, DB2): + Passed = True + for StructKey, StructDef in DB1.Structs.items(): + if (len(StructKey) == 0): + # XXX: Oops, shouldn't have anonymous structs + continue + + logging.info ("Comparing crossArch {0}".format(StructDef.Name)) + if (StructDef.ExpectFEXMatch): + Struct2 = DB2.Structs.get(StructDef.Name) + if (Struct2 == None): + logging.critical ("Couldn't find Struct {0} in {1} DB".format(StructDef.Name, DB2.ArchName)) + Passed = False + continue + + Passed &= CompareStructs(StructDef, Struct2) + + return Passed + +def main(): + if sys.version_info[0] < 3: + logging.critical ("Python 3 or a more recent version is required.") + + if (len(sys.argv) < 2): + print ("usage: %s " % (sys.argv[0])) + print ("\t-c1 : Base Comparison Type"); + print ("\t-c2 : Second Comparison Type"); + print ("\t-win: Parse Windows"); + sys.exit ("\t-no-linux: Do not parse Linux"); + + ParseLinux = True + ParseWindows = False + + Header = "" + Comparison1 = "" + Comparison2 = "" + BaseArgs = [] + + StartOfArgs = 0 + + # Parse our arguments + ArgIndex = 1 + while ArgIndex < len(sys.argv): + Arg = sys.argv[ArgIndex] + if (Arg == "--"): + StartOfArgs = ArgIndex + 1 + break; + + if (Arg == "-c1"): + ArgIndex += 1 + Comparison1 = sys.argv[ArgIndex] + elif (Arg == "-c2"): + ArgIndex += 1 + Comparison2 = sys.argv[ArgIndex] + elif (Arg == "-win"): + ParseWindows = True + elif (Arg == "-no-linux"): + ParseLinux = False + else: + Header = Arg + StartOfArgs = ArgIndex + 1 + break + + # Increment + ArgIndex += 1 + + # Add arguments for clang + for ArgIndex in range(StartOfArgs, len(sys.argv)): + BaseArgs.append(sys.argv[ArgIndex]) + + args_x86_32 = [ + "-I/usr/i686-linux-gnu/include/c++/10/i686-linux-gnu/", + "-I/usr/i686-linux-gnu/include/", + "-O2", + "-m32", + "--target=i686-linux-unknown", + ] + + args_x86_64 = [ + "-I/usr/include/x86_64-linux-gnu", + "-I/usr/x86_64-linux-gnu/include/c++/10/x86_64-linux-gnu/", + "-I/usr/x86_64-linux-gnu/include/", + "-O2", + "--target=x86_64-linux-unknown", + ] + + args_aarch64 = [ + "-I/usr/aarch64-linux-gnu/include/c++/10/aarch64-linux-gnu/", + "-I/usr/aarch64-linux-gnu/include/", + "-O2", + "--target=aarch64-linux-unknown", + ] + + args_x86_win32 = [ + "-I/usr/lib/gcc/i686-w64-mingw32/10-win32/include/c++/", + "-I/usr/lib/gcc/i686-w64-mingw32/10-win32/include/c++/i686-w64-mingw32/", + "-O2", + "-m32", + "--target=i686-pc-win32", + ] + + args_x86_win64 = [ + "-I/usr/lib/gcc/x86_64-w64-mingw32/10-win32/include/c++/", + "-I/usr/lib/gcc/x86_64-w64-mingw32/10-win32/include/c++/x86_64-w64-mingw32/", + "-O2", + "--target=x86_64-pc-win32", + ] + + # Add all the arguments to the different lists + args_x86_32.extend(BaseArgs) + args_x86_64.extend(BaseArgs) + args_aarch64.extend(BaseArgs) + args_x86_win32.extend(BaseArgs) + args_x86_win64.extend(BaseArgs) + + # We need to find the default arguments through clang invocations + args_x86_32 = FindClangArguments(args_x86_32) + args_x86_64 = FindClangArguments(args_x86_64) + args_aarch64 = FindClangArguments(args_aarch64) + args_x86_win32 = FindClangArguments(args_x86_win32) + args_x86_win64 = FindClangArguments(args_x86_win64) + + Arch_x86_32 = ArchDB("x86_32") + Arch_x86_64 = ArchDB("x86_64") + Arch_aarch64 = ArchDB("aarch64") + Arch_x86_win32 = ArchDB("win32") + Arch_x86_win64 = ArchDB("win64") + + if (ParseLinux): + Arch_x86_32 = GetDB(Arch_x86_32, Header, args_x86_32) + Arch_x86_64 = GetDB(Arch_x86_64, Header, args_x86_64) + Arch_aarch64 = GetDB(Arch_aarch64, Header, args_aarch64) + + if not (Arch_x86_32.Parsed): + logging.critical ("Couldn't parse:{0}".format(Arch_x86_32.ArchName)) + + if not (Arch_x86_64.Parsed): + logging.critical ("Couldn't parse:{0}".format(Arch_x86_64.ArchName)) + + if not (Arch_aarch64.Parsed): + logging.critical ("Couldn't parse:{0}".format(Arch_aarch64.ArchName)) + + if (ParseWindows): + Arch_x86_win32 = GetDB(Arch_x86_win32, Header, args_x86_win32) + Arch_x86_win64 = GetDB(Arch_x86_win64, Header, args_x86_win64) + + if not (Arch_x86_win32.Parsed): + logging.critical ("Couldn't parse:{0}".format(Arch_x86_win32.ArchName)) + + if not (Arch_x86_win64.Parsed): + logging.critical ("Couldn't parse:{0}".format(Arch_x86_win64.ArchName)) + + DBs = DBList(Arch_x86_32, + Arch_x86_64, + Arch_aarch64, + Arch_x86_win32, + Arch_x86_win64) + + Result = 0 + if (len(Comparison1) != 0 and len(Comparison2) != 0): + CompDB1 = GetCompar(Comparison1, DBs) + CompDB2 = GetCompar(Comparison2, DBs) + + # Now compare across the two compared architectures + Result = 0 if CompareCrossArch(CompDB1, CompDB2) else 1 + elif (len(Comparison1) != 0): + CompDB1 = GetCompar(Comparison1, DBs) + + # First compare the aliases to make sure we are matching + Result = 0 if CompareAliases(CompDB1, DBs) else 1 + + if (Result == 1): + logging.error("Execution environment") + Args = "[ " + for Arg in sys.argv: + Args += Arg + ", " + Args += " ]" + logging.error(Args) + Args = "" + for Arg in sys.argv: + Args += "\"" + Arg + "\" " + + logging.error(Args) + return Result + +if __name__ == "__main__": +# execute only if run as a script + sys.exit(main()) diff --git a/Source/Tests/LinuxSyscalls/CMakeLists.txt b/Source/Tests/LinuxSyscalls/CMakeLists.txt index b1a4f41ef..0c8389b33 100644 --- a/Source/Tests/LinuxSyscalls/CMakeLists.txt +++ b/Source/Tests/LinuxSyscalls/CMakeLists.txt @@ -55,3 +55,68 @@ add_library(LinuxEmulation STATIC target_link_libraries(LinuxEmulation FEXCore pthread numa) target_include_directories(LinuxEmulation PRIVATE ${CMAKE_BINARY_DIR}/generated) + +set(HEADERS_TO_VERIFY + x32/Types.h x86_32 # This needs to match structs to 32bit structs + ) + +list(LENGTH HEADERS_TO_VERIFY ARG_COUNT) +math(EXPR ARG_COUNT "${ARG_COUNT}-1") + +set (ARGS + "-x" "c++" + "-std=c++20") +# Global include directories +get_directory_property (INC_DIRS INCLUDE_DIRECTORIES) +list(TRANSFORM INC_DIRS PREPEND "-I") +list(APPEND ARGS ${INC_DIRS}) + +# FEXCore directories +get_target_property(INC_DIRS FEXCore INTERFACE_INCLUDE_DIRECTORIES) +list(TRANSFORM INC_DIRS PREPEND "-I") +list(APPEND ARGS ${INC_DIRS}) + +foreach(Index RANGE 0 ${ARG_COUNT} 2) + math(EXPR TEST_TYPE_INDEX "${Index}+1") + + list(GET HEADERS_TO_VERIFY ${Index} HEADER) + list(GET HEADERS_TO_VERIFY ${TEST_TYPE_INDEX} TEST_TYPE) + + file(RELATIVE_PATH REL_HEADER ${CMAKE_BINARY_DIR} "${CMAKE_CURRENT_SOURCE_DIR}/${HEADER}") + set(TEST_NAME "${TEST_DESC}/Test_verify_${HEADER}") + set(TEST_NAME_ARCH "${TEST_DESC}/Test_verify_arch_${HEADER}") + + add_test( + NAME ${TEST_NAME}_x86_64 + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "x86_64" "${REL_HEADER}" ${ARGS}) + + add_test( + NAME ${TEST_NAME}_aarch64 + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "aarch64" "${REL_HEADER}" ${ARGS}) + + add_test( + NAME ${TEST_NAME_ARCH}_x86_64 + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "x86_64" "-c2" "${TEST_TYPE}" "${REL_HEADER}" ${ARGS}) + + add_test( + NAME ${TEST_NAME_ARCH}_aarch64 + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "aarch64" "-c2" "${TEST_TYPE}" "${REL_HEADER}" ${ARGS}) + + set_property(TEST ${TEST_NAME}_x86_64 APPEND PROPERTY DEPENDS "${HEADER}") + set_property(TEST ${TEST_NAME}_aarch64 APPEND PROPERTY DEPENDS "${HEADER}") + set_property(TEST ${TEST_NAME_ARCH}_x86_64 APPEND PROPERTY DEPENDS "${HEADER}") + set_property(TEST ${TEST_NAME_ARCH}_aarch64 APPEND PROPERTY DEPENDS "${HEADER}") +endforeach() + +execute_process(COMMAND "nproc" OUTPUT_VARIABLE CORES) +string(STRIP ${CORES} CORES) + +add_custom_target( + struct_verifier + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + USES_TERMINAL + COMMAND "ctest" "--timeout" "302" "-j${CORES}" "-R" "Test_verify*") diff --git a/Source/Tests/LinuxSyscalls/x32/Types.h b/Source/Tests/LinuxSyscalls/x32/Types.h index e2a82e9a3..9009400cd 100644 --- a/Source/Tests/LinuxSyscalls/x32/Types.h +++ b/Source/Tests/LinuxSyscalls/x32/Types.h @@ -22,6 +22,18 @@ using compat_long_t = int32_t; using compat_uptr_t = uint32_t; using compat_size_t = uint32_t; using compat_off_t = uint32_t; +using compat_loff_t = int64_t; +using compat_pid_t = int32_t; +using compat_dev_t = uint16_t; +using compat_ino_t = uint32_t; +using compat_mode_t = uint16_t; +using compat_nlink_t = uint16_t; +using compat_uid_t = uint16_t; +using compat_gid_t = uint16_t; + +// Can't use using with aligned attributes, clang doesn't honour it +typedef __attribute__((aligned(4))) uint64_t compat_uint64_t; +typedef __attribute__((aligned(4))) int64_t compat_int64_t; template class compat_ptr { @@ -81,7 +93,10 @@ static_assert(sizeof(compat_ptr) == 4, "Incorrect size"); * Provides conversation operators for the host version * @{ */ -struct timespec32 { +struct +__attribute__((annotate("alias-x86_32-timespec"))) +__attribute__((annotate("fex-match"))) +timespec32 { int32_t tv_sec; int32_t tv_nsec; @@ -111,7 +126,10 @@ static_assert(sizeof(timespec32) == 8, "Incorrect size"); * Provides conversation operators for the host version * @{ */ -struct timeval32 { +struct +__attribute__((annotate("alias-x86_32-timeval"))) +__attribute__((annotate("fex-match"))) +timeval32 { int32_t tv_sec; int32_t tv_usec; @@ -141,7 +159,10 @@ static_assert(sizeof(timeval32) == 8, "Incorrect size"); * Provides conversation operators for the host version * @{ */ -struct iovec32 { +struct +__attribute__((annotate("alias-x86_32-iovec"))) +__attribute__((annotate("fex-match"))) +iovec32 { uint32_t iov_base; uint32_t iov_len; @@ -164,17 +185,23 @@ static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(iovec32) == 8, "Incorrect size"); /** @} */ -struct cmsghdr32 { +struct +__attribute__((annotate("alias-x86_32-cmsghdr"))) +__attribute__((annotate("fex-match"))) +cmsghdr32 { uint32_t cmsg_len; int32_t cmsg_level; int32_t cmsg_type; - char cmsg_data[0]; + char cmsg_data[]; }; static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(cmsghdr32) == 12, "Incorrect size"); -struct msghdr32 { +struct +__attribute__((annotate("alias-x86_32-msghdr"))) +__attribute__((annotate("fex-match"))) +msghdr32 { compat_ptr msg_name; socklen_t msg_namelen; @@ -189,7 +216,10 @@ struct msghdr32 { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(msghdr32) == 28, "Incorrect size"); -struct mmsghdr_32 { +struct +__attribute__((annotate("alias-x86_32-mmsghdr"))) +__attribute__((annotate("fex-match"))) +mmsghdr_32 { msghdr32 msg_hdr; uint32_t msg_len; }; @@ -197,7 +227,10 @@ struct mmsghdr_32 { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(mmsghdr_32) == 32, "Incorrect size"); -struct stack_t32 { +struct +__attribute__((annotate("alias-x86_32-stack_t"))) +__attribute__((annotate("fex-match"))) +stack_t32 { compat_ptr ss_sp; compat_size_t ss_size; int32_t ss_flags; @@ -222,17 +255,22 @@ struct stack_t32 { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(stack_t32) == 12, "Incorrect size"); -struct stat32 { - uint32_t st_dev; - uint32_t st_ino; - uint32_t st_nlink; +struct +// This does not match the glibc implementation of stat +// Matches the definition of `struct compat_stat` in `arch/x86/include/asm/compat.h` +__attribute__((annotate("fex-match"))) +stat32 { + compat_dev_t st_dev; + uint16_t __pad1; + compat_ino_t st_ino; + compat_mode_t st_mode; + compat_nlink_t st_nlink; - uint16_t st_mode; - uint16_t st_uid; - uint16_t st_gid; - uint16_t __pad0; + compat_uid_t st_uid; + compat_gid_t st_gid; + compat_dev_t st_rdev; - uint32_t st_rdev; + uint16_t __pad2; uint32_t st_size; uint32_t st_blksize; uint32_t st_blocks; /* Number 512-byte blocks allocated. */ @@ -242,7 +280,8 @@ struct stat32 { uint32_t st_mtime_nsec; uint32_t st_ctime_; uint32_t st_ctime_nsec; - uint32_t __unused[3]; + uint32_t __unused4; + uint32_t __unused5; stat32() = delete; @@ -250,13 +289,13 @@ struct stat32 { #define COPY(x) x = host.x COPY(st_dev); COPY(st_ino); + COPY(st_mode); COPY(st_nlink); - COPY(st_mode); COPY(st_uid); COPY(st_gid); - COPY(st_rdev); + COPY(st_size); COPY(st_blksize); COPY(st_blocks); @@ -273,11 +312,16 @@ struct stat32 { } }; static_assert(std::is_trivial::value, "Needs to be trivial"); -static_assert(sizeof(stat32) == 72, "Incorrect size"); +static_assert(sizeof(stat32) == 64, "Incorrect size"); -struct __attribute__((packed)) stat64_32 { - uint64_t st_dev; - uint32_t pad0; +struct +// This does not match the glibc implementation of stat +// Matches the definition of `struct stat64` in `x86_64-linux-gnu/asm/stat.h` +__attribute__((annotate("fex-match"))) +__attribute__((packed)) +stat64_32 { + compat_uint64_t st_dev; + uint8_t __pad0[4]; uint32_t __st_ino; uint32_t st_mode; @@ -286,18 +330,18 @@ struct __attribute__((packed)) stat64_32 { uint32_t st_uid; uint32_t st_gid; - uint64_t st_rdev; - uint32_t pad3; - int64_t st_size; + compat_uint64_t st_rdev; + uint8_t __pad3[4]; + compat_int64_t st_size; uint32_t st_blksize; - uint64_t st_blocks; /* Number 512-byte blocks allocated. */ + compat_uint64_t st_blocks; /* Number 512-byte blocks allocated. */ uint32_t st_atime_; uint32_t st_atime_nsec; uint32_t st_mtime_; uint32_t st_mtime_nsec; uint32_t st_ctime_; uint32_t st_ctime_nsec; - uint64_t st_ino; + compat_uint64_t st_ino; stat64_32() = delete; @@ -360,14 +404,18 @@ struct __attribute__((packed)) stat64_32 { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(stat64_32) == 96, "Incorrect size"); -struct __attribute__((packed,aligned(4))) statfs64_32 { +struct +__attribute__((packed,aligned(4))) +__attribute__((annotate("alias-x86_32-statfs64"))) +__attribute__((annotate("fex-match"))) +statfs64_32 { uint32_t f_type; uint32_t f_bsize; - uint64_t f_blocks; - uint64_t f_bfree; - uint64_t f_bavail; - uint64_t f_files; - uint64_t f_ffree; + compat_uint64_t f_blocks; + compat_uint64_t f_bfree; + compat_uint64_t f_bavail; + compat_uint64_t f_files; + compat_uint64_t f_ffree; __kernel_fsid_t f_fsid; uint32_t f_namelen; uint32_t f_frsize; @@ -413,7 +461,10 @@ struct __attribute__((packed,aligned(4))) statfs64_32 { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(statfs64_32) == 84, "Incorrect size"); -struct flock_32 { +struct +__attribute__((annotate("alias-x86_32-flock"))) +__attribute__((annotate("fex-match"))) +flock_32 { int16_t l_type; int16_t l_whence; int32_t l_start; @@ -445,12 +496,17 @@ static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(flock_32) == 16, "Incorrect size"); // glibc doesn't pack flock64 while the kernel does -struct flock64_32 { +// This does not match glibc flock64 definition +// Matches the definition of `struct compat_flock64` in `arch/x86/include/asm/compat.h` +struct +__attribute__((annotate("fex-match"))) +__attribute__((packed)) +flock64_32 { int16_t l_type; int16_t l_whence; - int32_t l_start; - int32_t l_len; - int32_t l_pid; + compat_loff_t l_start; + compat_loff_t l_len; + compat_pid_t l_pid; flock64_32() = delete; @@ -473,9 +529,13 @@ struct flock64_32 { } }; static_assert(std::is_trivial::value, "Needs to be trivial"); -static_assert(sizeof(flock64_32) == 16, "Incorrect size"); +static_assert(sizeof(flock64_32) == 24, "Incorrect size"); -struct linux_dirent { +// There is no public definition of this struct +// Matches the definition of `struct linux_dirent` in fs/readdir.c +struct +__attribute__((annotate("fex-match"))) +linux_dirent { uint64_t d_ino; int64_t d_off; uint16_t d_reclen; @@ -485,18 +545,25 @@ struct linux_dirent { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(linux_dirent) == 24, "Incorrect size"); -struct linux_dirent_32 { - uint32_t d_ino; - int32_t d_off; +// There is no public definition of this struct +// Matches the definition of `struct compat_linux_dirent` in fs/readdir.c +struct +__attribute__((annotate("fex-match"))) +linux_dirent_32 { + compat_ulong_t d_ino; + compat_ulong_t d_off; uint16_t d_reclen; - uint8_t _pad[2]; - char d_name[]; + char d_name[1]; /* Has hidden null character and d_type */ }; static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(linux_dirent_32) == 12, "Incorrect size"); -struct linux_dirent_64 { +// There is no public definition of this struct +// Matches the definition of `struct linux_dirent64` in include/linux/dirent.h +struct +__attribute__((annotate("fex-match"))) +linux_dirent_64 { uint64_t d_ino; uint64_t d_off; uint16_t d_reclen; @@ -507,31 +574,80 @@ struct linux_dirent_64 { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(linux_dirent_64) == 24, "Incorrect size"); -struct sigset_argpack32 { +// There is no public definition of this struct +// Matches `struct compat_sigset_argpack` +struct +__attribute__((annotate("fex-match"))) +sigset_argpack32 { compat_ptr sigset; - size_t size; + compat_size_t size; }; static_assert(std::is_trivial::value, "Needs to be trivial"); -static_assert(sizeof(sigset_argpack32) == 16, "Incorrect size"); +static_assert(sizeof(sigset_argpack32) == 8, "Incorrect size"); -struct rusage_32 { +struct +__attribute__((annotate("alias-x86_32-rusage"))) +__attribute__((annotate("fex-match"))) +rusage_32 { timeval32 ru_utime; timeval32 ru_stime; - compat_long_t ru_maxrss; - compat_long_t ru_ixrss; - compat_long_t ru_idrss; - compat_long_t ru_isrss; - compat_long_t ru_minflt; - compat_long_t ru_majflt; - compat_long_t ru_nswap; - compat_long_t ru_inblock; - compat_long_t ru_oublock; - compat_long_t ru_msgsnd; - compat_long_t ru_msgrcv; - compat_long_t ru_nsignals; - compat_long_t ru_nvcsw; - compat_long_t ru_nivcsw; + union { + compat_long_t ru_maxrss; + compat_long_t __ru_maxrss_word; + }; + union { + compat_long_t ru_ixrss; + compat_long_t __ru_ixrss_word; + }; + union { + compat_long_t ru_idrss; + compat_long_t __ru_idrss_word; + }; + union { + compat_long_t ru_isrss; + compat_long_t __ru_isrss_word; + }; + union { + compat_long_t ru_minflt; + compat_long_t __ru_minflt_word; + }; + union { + compat_long_t ru_majflt; + compat_long_t __ru_majflt_word; + }; + union { + compat_long_t ru_nswap; + compat_long_t __ru_nswap_word; + }; + union { + compat_long_t ru_inblock; + compat_long_t __ru_inblock_word; + }; + union { + compat_long_t ru_oublock; + compat_long_t __ru_oublock_word; + }; + union { + compat_long_t ru_msgsnd; + compat_long_t __ru_msgsnd_word; + }; + union { + compat_long_t ru_msgrcv; + compat_long_t __ru_msgrcv_word; + }; + union { + compat_long_t ru_nsignals; + compat_long_t __ru_nsignals_word; + }; + union { + compat_long_t ru_nvcsw; + compat_long_t __ru_nvcsw_word; + }; + union { + compat_long_t ru_nivcsw; + compat_long_t __ru_nivcsw_word; + }; rusage_32() = delete; rusage_32(struct rusage usage) @@ -579,10 +695,16 @@ struct rusage_32 { static_assert(std::is_trivial::value, "Needs to be trivial"); static_assert(sizeof(rusage_32) == 72, "Incorrect size"); -struct __attribute__((packed)) GuestSigAction_32 { +// This definition isn't public +// This is for rt_sigaction +// Matches the definition for `struct compat_sigaction` in `include/linux/compat.h` +struct +__attribute__((packed)) +__attribute__((annotate("fex-match"))) +GuestSigAction_32 { FEX::HLE::x32::compat_ptr handler_32; - uint64_t sa_flags; + uint32_t sa_flags; FEX::HLE::x32::compat_ptr restorer_32; FEXCore::GuestSAMask sa_mask; @@ -607,6 +729,6 @@ struct __attribute__((packed)) GuestSigAction_32 { }; static_assert(std::is_trivial::value, "Needs to be trivial"); -static_assert(sizeof(GuestSigAction_32) == 24, "Incorrect size"); +static_assert(sizeof(GuestSigAction_32) == 20, "Incorrect size"); }