Merge pull request #850 from Sonicadvance1/struct_verifier

libclang based Struct verifier written in python

.github/workflows/ccpp.yml

@@ -113,13 +113,24 @@ jobs:
       working-directory: ${{runner.workspace}}/build
       run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_GCC64.log || true
 
+    - name: Struct verifier tests
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE --target struct_verifier
+
+    - name: Struct verifier Test Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_StructVerifier.log || true
+
     - name: Truncate test results
       if: ${{ always() }}
       shell: bash
       working-directory: ${{runner.workspace}}/build
       # Cap out the log files at 20M in case something crash spins and dumps fault text
       # ASM tests get quite close to 10MB
-      run: truncate --size=20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true
+      run: truncate --size=<20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true
 
     - name: Set runner name
       if: ${{ always() }}

CMakeLists.txt

@@ -234,14 +234,16 @@ endif()
 
 add_compile_options(-Wall)
 
+if (BUILD_TESTS)
+  include(CTest)
+  enable_testing()
+  message(STATUS "Unit tests are enabled")
+endif()
 add_subdirectory(External/FEXCore)
 
 add_subdirectory(Source/)
 
 if (BUILD_TESTS)
-  include(CTest)
-  enable_testing()
-  message(STATUS "Unit tests are enabled")
   add_subdirectory(unittests/)
 endif()
 

Scripts/StructPackVerifier.py

@@ -0,0 +1,661 @@
+#!/usr/bin/python3
+import clang.cindex
+from clang.cindex import CursorKind
+from clang.cindex import TypeKind
+from clang.cindex import TranslationUnit
+import sys
+from dataclasses import dataclass, field
+import subprocess
+import logging
+logger = logging.getLogger()
+logger.setLevel(logging.WARNING)
+
+@dataclass
+class TypeDefinition:
+    TYPE_UNKNOWN = 0
+    TYPE_STRUCT = 1
+    TYPE_UNION = 2
+    TYPE_FIELD = 3
+
+    name: str
+    type: int
+    def __init__(self, Name, Type):
+        self.name = Name
+        self.type = Type
+
+    @property
+    def Name(self):
+        return self.name
+    @property
+    def Type(self):
+        return self.type
+
+@dataclass
+class AliasType:
+    ALIAS_X86_32  = 0
+    ALIAS_X86_64  = 1
+    ALIAS_AARCH64 = 2
+    ALIAS_WIN32   = 3
+    ALIAS_WIN64   = 4
+    Name: str
+    AliasType: int
+    def __init__(self, Name, Type):
+        self.Name = Name
+        self.AliasType = Type
+
+@dataclass
+class StructDefinition(TypeDefinition):
+    Size: int
+    Aliases: list
+    Members: list
+    ExpectFEXMatch: bool
+
+    def __init__(self, Name, Size):
+        super(StructDefinition, self).__init__(Name, TypeDefinition.TYPE_STRUCT)
+        self.Size = Size
+        self.Aliases = []
+        self.Members = []
+        self.ExpectFEXMatch = False
+
+@dataclass
+class UnionDefinition(TypeDefinition):
+    Size: int
+    Aliases: list
+    Members: list
+    ExpectFEXMatch: bool
+
+    def __init__(self, Name, Size):
+        super(UnionDefinition, self).__init__(Name, TypeDefinition.TYPE_UNION)
+        self.Size = Size
+        self.Aliases = []
+        self.Members = []
+        self.ExpectFEXMatch = False
+
+@dataclass
+class FieldDefinition(TypeDefinition):
+    Size: int
+    OffsetOf: int
+    Alignment: int
+    def __init__(self, Name, Size, OffsetOf, Alignment):
+        super(FieldDefinition, self).__init__(Name, TypeDefinition.TYPE_FIELD)
+        self.Size = Size
+        self.OffsetOf = OffsetOf
+        self.Alignment = Alignment
+
+@dataclass
+class ArchDB:
+    Parsed: bool
+    ArchName: str
+    NamespaceScope: list
+    CurrentNamespace: str
+    TU: TranslationUnit
+    Structs: dict
+    Unions: dict
+    FieldDecls: list
+    def __init__(self, ArchName):
+        self.Parsed = True
+        self.ArchName = ArchName
+        self.NamespaceScope = []
+        self.CurrentNamespace = ""
+        self.TU = None
+        self.Structs = {}
+        self.Unions = {}
+        self.FieldDecls = []
+
+class DBList:
+    DBs: list
+    def __init__(self, DB32, DB64, DBAArch64, DBWin32, DBWin64):
+        self.DBs = [DB32, DB64, DBAArch64, DBWin32, DBWin64]
+
+def FindClangArguments(OriginalArguments):
+    AddedArguments = ["clang"]
+    AddedArguments.extend(OriginalArguments)
+    AddedArguments.extend(["-v", "-x", "c++", "-S", "-"])
+    Proc = subprocess.Popen(AddedArguments, stderr = subprocess.PIPE, stdin = subprocess.DEVNULL)
+    NewIncludes = []
+    BeginSearch = False
+    while True:
+        Line = Proc.stderr.readline().strip()
+
+        if not Line:
+            Proc.terminate()
+            break
+
+        if (Line == b"End of search list."):
+            BeginSearch = False
+            Proc.terminate()
+            break
+
+        if (BeginSearch == True):
+            NewIncludes.append("-I" + Line.decode('ascii'))
+
+        if (Line == b"#include <...> search starts here:"):
+            BeginSearch = True
+
+    # Add back original arguments
+    NewIncludes.extend(OriginalArguments)
+    return NewIncludes
+
+def SetNamespace(Arch):
+    Arch.CurrentNamespace = ""
+    for Namespace in Arch.NamespaceScope:
+        Arch.CurrentNamespace = Arch.CurrentNamespace + Namespace + "::"
+
+def HandleStructDeclCursor(Arch, Cursor, NameOverride = ""):
+    # Append namespace
+    CursorName = ""
+    StructType = Cursor.type
+    if (len(StructType.spelling) == 0):
+        CursorName = NameOverride
+    else:
+        CursorName = StructType.spelling
+
+    if (len(CursorName) != 0):
+        Arch.NamespaceScope.append(CursorName)
+        SetNamespace(Arch)
+
+    Struct = StructDefinition(
+        Name = CursorName,
+        Size = StructType.get_size())
+
+    # Handle children
+    Arch.Structs[Struct.Name] = HandleStructElements(Arch, Struct, Cursor)
+
+    # Pop namespace off
+    if (len(CursorName) != 0):
+        Arch.NamespaceScope.pop()
+        SetNamespace(Arch)
+
+    return Arch
+
+def HandleUnionDeclCursor(Arch, Cursor, NameOverride = ""):
+    # Append namespace
+    CursorName = ""
+
+    if (len(Cursor.spelling) == 0):
+        CursorName = NameOverride
+    else:
+        CursorName = Cursor.spelling
+
+    if (len(CursorName) != 0):
+        Arch.NamespaceScope.append(CursorName)
+        SetNamespace(Arch)
+
+    UnionType = Cursor.type
+    Union = UnionDefinition(
+        Name = CursorName,
+        Size = UnionType.get_size())
+    Arch.Unions[Union.Name] = Union
+
+    # Handle children
+    Arch.Unions[Union.Name] = HandleStructElements(Arch, Union, Cursor)
+
+    # Pop namespace off
+    if (len(CursorName) != 0):
+        Arch.NamespaceScope.pop()
+        SetNamespace(Arch)
+
+    return Arch
+
+def HandleTypeDefDeclCursor(Arch, Cursor):
+    TypeDefType = Cursor.underlying_typedef_type
+    CanonicalType = TypeDefType.get_canonical()
+
+    if (TypeDefType.kind == TypeKind.ELABORATED and CanonicalType.kind == TypeKind.RECORD):
+        TypeDefName = Cursor.type.get_typedef_name()
+        if (len(TypeDefName) != 0):
+            logging.info ("Found Typedef Decl'{0}'".format(TypeDefName))
+            logging.info ("\tSize of type: {0}".format(CanonicalType.get_size()));
+            HandleTypeDefDecl(Arch, Cursor, TypeDefName)
+
+            # Append namespace
+            Arch.NamespaceScope.append(TypeDefName)
+            SetNamespace(Arch)
+
+            Arch = HandleCursor(Arch, Cursor)
+            #StructType = Cursor.type
+            #Struct = StructDefinition(
+            #    Name = TypeDefName,
+            #    Size = CanonicalType.get_size())
+            #Arch.Structs[TypeDefName] = Struct
+
+            ## Handle children
+            #Arch.Structs[TypeDefName] = HandleStructElements(Arch, Struct, Cursor)
+
+            # Pop namespace off
+            Arch.NamespaceScope.pop()
+            SetNamespace(Arch)
+    return Arch
+
+def HandleStructElements(Arch, Struct, Cursor):
+    for Child in Cursor.get_children():
+        logging.info ("\t\tStruct/Union Children: Cursor \"{0}{1}\" of kind {2}".format(Arch.CurrentNamespace, Child.spelling, Child.kind))
+        if (Child.kind == CursorKind.ANNOTATE_ATTR):
+            if (Child.spelling.startswith("alias-")):
+                Sections = Child.spelling.split("-")
+                if (Sections[1] == "x86_32"):
+                    Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_X86_32))
+                elif (Sections[1] == "x86_64"):
+                    Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_X86_64))
+                elif (Sections[1] == "aarch64"):
+                    Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_AARCH64))
+                elif (Sections[1] == "win32"):
+                    Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_WIN32))
+                elif (Sections[1] == "win64"):
+                    Struct.Aliases.append(AliasType(Sections[2], AliasType.ALIAS_WIN64))
+                else:
+                    logging.critical ("Can't handle alias type '{0}'".format(Child.spelling))
+            elif (Child.spelling == "fex-match"):
+                Struct.ExpectedFEXMatch = True
+            else:
+                # Unknown annotation
+                pass
+        elif (Child.kind == CursorKind.FIELD_DECL):
+            ParentType = Cursor.type
+            FieldType = Child.type
+            Field = FieldDefinition(
+                Name = Child.spelling,
+                Size = FieldType.get_size(),
+                OffsetOf = ParentType.get_offset(Child.spelling),
+                Alignment = FieldType.get_align())
+
+            logging.info ("\t{0}".format(Child.spelling))
+            logging.info ("\t\tSize of type: {0}".format(FieldType.get_size()));
+            logging.info ("\t\tAlignment of type: {0}".format(FieldType.get_align()));
+            logging.info ("\t\tOffsetof of type: {0}".format(ParentType.get_offset(Child.spelling)));
+            Struct.Members.append(Field)
+            Arch.FieldDecls.append(Field)
+        elif (Child.kind == CursorKind.STRUCT_DECL):
+            ParentType = Cursor.type
+            FieldType = Child.type
+            Field = FieldDefinition(
+                Name = Child.spelling,
+                Size = FieldType.get_size(),
+                OffsetOf = ParentType.get_offset(Child.spelling),
+                Alignment = FieldType.get_align())
+
+            logging.info ("\t{0}".format(Child.spelling))
+            logging.info ("\t\tSize of type: {0}".format(FieldType.get_size()));
+            logging.info ("\t\tAlignment of type: {0}".format(FieldType.get_align()));
+            logging.info ("\t\tOffsetof of type: {0}".format(ParentType.get_offset(Child.spelling)));
+            Struct.Members.append(Field)
+            Arch.FieldDecls.append(Field)
+            Arch = HandleStructDeclCursor(Arch, Child)
+        elif (Child.kind == CursorKind.UNION_DECL):
+            Struct = HandleStructElements(Arch, Struct, Child)
+            #ParentType = Cursor.type
+            #FieldType = Child.type
+            #Field = FieldDefinition(
+            #    Name = Child.spelling,
+            #    Size = FieldType.get_size(),
+            #    OffsetOf = ParentType.get_offset(Child.spelling),
+            #    Alignment = FieldType.get_align())
+
+            #logging.info ("\t{0}".format(Child.spelling))
+            #logging.info ("\t\tSize of type: {0}".format(FieldType.get_size()));
+            #logging.info ("\t\tAlignment of type: {0}".format(FieldType.get_align()));
+            #logging.info ("\t\tOffsetof of type: {0}".format(ParentType.get_offset(Child.spelling)));
+            #Struct.Members.append(Field)
+            #Arch.FieldDecls.append(Field)
+            #Arch = HandleUnionDeclCursor(Arch, Child)
+        elif (Child.kind == CursorKind.TYPEDEF_DECL):
+            Arch = HandleTypeDefDeclCursor(Arch, Child)
+        else:
+            Arch = HandleCursor(Arch, Child)
+
+    return Struct
+
+def HandleTypeDefDecl(Arch, Cursor, Name):
+    for Child in Cursor.get_children():
+        if (Child.kind == CursorKind.UNION_DECL):
+            pass
+        elif (Child.kind == CursorKind.STRUCT_DECL):
+            Arch = HandleStructDeclCursor(Arch, Child, Name)
+        elif (Child.kind == CursorKind.UNION_DECL):
+            Arch = HandleUnionDeclCursor(Arch, Child, Name)
+        elif (Child.kind == CursorKind.TYPEDEF_DECL):
+            Arch = HandleTypeDefDeclCursor(Arch, Child)
+        elif (Child.kind == CursorKind.TYPE_REF or
+              Child.kind == CursorKind.NAMESPACE_REF or
+              Child.kind == CursorKind.TEMPLATE_REF):
+            # Safe to pass on
+            pass
+        else:
+            logging.critical ("Unhandled TypedefDecl {0}-{1}-{2}".format(Child.kind, Child.type.spelling, Child.spelling))
+
+def HandleCursor(Arch, Cursor):
+    if (Cursor.kind.is_invalid()):
+        Diags = TU.diagnostics
+        for Diag in Diags:
+            logging.warning (Diag.format())
+
+        Arch.Parsed = False
+        return
+
+    for Child in Cursor.get_children():
+        logging.info ("\tCursor \"{0}\" of kind {1}".format(Child.spelling, Child.kind))
+        if (Child.kind == CursorKind.TRANSLATION_UNIT):
+            Arch = HandleCursor(Arch, Child)
+        elif (Child.kind == CursorKind.FIELD_DECL):
+            pass
+        elif (Child.kind == CursorKind.UNION_DECL):
+            Arch = HandleUnionDeclCursor(Arch, Child)
+        elif (Child.kind == CursorKind.STRUCT_DECL):
+            Arch = HandleStructDeclCursor(Arch, Child)
+        elif (Child.kind == CursorKind.TYPEDEF_DECL):
+            Arch = HandleTypeDefDeclCursor(Arch, Child)
+        elif (Child.kind == CursorKind.NAMESPACE):
+            # Append namespace
+            Arch.NamespaceScope.append(Child.spelling)
+            SetNamespace(Arch)
+
+            # Handle children
+            Arch = HandleCursor(Arch, Child)
+
+            # Pop namespace off
+            Arch.NamespaceScope.pop()
+            SetNamespace(Arch)
+        elif (Child.kind == CursorKind.TYPE_REF):
+            # Safe to pass on
+            pass
+        else:
+            Arch = HandleCursor(Arch, Child)
+
+    return Arch
+
+def GetDB(Arch, filename, args):
+    Index = clang.cindex.Index.create()
+    try:
+        TU = Index.parse(filename, args=args, options=TranslationUnit.PARSE_INCOMPLETE)
+    except TranslationUnitLoadError:
+        Arch.Parsed = False
+        Diags = TU.diagnostics
+        for Diag in Diags:
+            logging.warning (Diag.format())
+
+        return
+
+    Arch.TU = TU
+    HandleCursor(Arch, TU.cursor)
+
+    # Get diagnostics
+    Diags = TU.diagnostics
+    if (len(Diags) != 0):
+        logging.warning ("Diagnostics from Arch: {0}".format(Arch.ArchName))
+
+    for Diag in Diags:
+        logging.warning (Diag.format())
+
+    return Arch
+
+def GetCompar(ComparisonName, DBs):
+    if (ComparisonName.lower() == "x86_32"):
+        return DBs.DBs[AliasType.ALIAS_X86_32]
+    elif (ComparisonName.lower() == "x86_64"):
+        return DBs.DBs[AliasType.ALIAS_X86_64]
+    elif (ComparisonName.lower() == "win32"):
+        return DBs.DBs[AliasType.ALIAS_WIN32]
+    elif (ComparisonName.lower() == "win64"):
+        return DBs.DBs[AliasType.ALIAS_WIN64]
+    elif (ComparisonName.lower() == "aarch64"):
+        return DBs.DBs[AliasType.ALIAS_AARCH64]
+
+def PrintMissingMembers(Struct1, Struct2):
+    for Member1 in Struct1.Members:
+        WasMissing = True
+        for Member2 in Struct2.Members:
+            if (Member1.Name == Member2.Name):
+                WasMissing = False
+                break
+        if (WasMissing):
+            logging.error ("\t'{0}' member '{1}' Doesn't exist in '{2}'".format(Struct1.Name, Member1.Name, Struct2.Name));
+
+def CompareStructs(Struct1, Struct2):
+    HadWarning = False
+    HadError = False
+
+    # Check if the struct size is a mismatch
+    if (Struct1.Size != Struct2.Size):
+        logging.warning ("\t#### Warning: Struct size mismatch. {0} != {1}".format(Struct1.Size, Struct2.Size))
+        logging.warning ("\t\tMight not be a problem if struct isn't in used inside another struct, or end of object")
+        HadWarning = True
+
+    # Check if the number of members differ
+    if (len(Struct1.Members) != len(Struct2.Members)):
+        logging.error ("@@@@ ERROR: Struct fields mismatch! Number of fields don't match! {0} != {1}".format(len(Struct1.Members), len(Struct2.Members)));
+        PrintMissingMembers(Struct1, Struct2)
+        PrintMissingMembers(Struct2, Struct1)
+        HadError = True
+    else:
+        # Compare the members themselves
+        for StructMemberIndex in range(0, len(Struct1.Members)):
+            Member1 = Struct1.Members[StructMemberIndex]
+            Member2 = Struct2.Members[StructMemberIndex]
+            if (Member1.Type == TypeDefinition.TYPE_FIELD):
+                if (Member1.Size != Member2.Size):
+                    logging.error ("\t@@@@ ERROR: Member '{0}' mismatch Size! {1} != {2}".format(Member1.Name, Member1.Size, Member2.Size));
+                    HadError = True
+                if (Member1.OffsetOf != Member2.OffsetOf):
+                    logging.error ("\t@@@@ ERROR: Member '{0}' mismatch OffsetOf! {1} != {2}".format(Member1.Name, Member1.OffsetOf, Member2.OffsetOf));
+                    HadError = True
+                if (Member1.Alignment != Member2.Alignment):
+                    logging.error ("\t@@@@ ERROR: Member '{0}' mismatch Alignment! {1} != {2}".format(Member1.Name, Member1.Alignment, Member2.Alignment));
+                    logging.error ("\t\tProbably not a problem if offset and size matches");
+                    HadWarning = True
+            else:
+                logging.critical ("Oops, didn't handle member type {0}".format(Member1.Type))
+        pass
+
+    return not (HadWarning or HadError)
+
+def CompareAliases(DB, DBs):
+    Passed = True
+    for StructKey, StructDef in DB.Structs.items():
+        if (len(StructKey) == 0):
+            # XXX: Oops, shouldn't have anonymous structs
+            continue
+
+        logging.info ("Comparing Aliases {0}".format(StructDef.Name))
+
+        for Alias in StructDef.Aliases:
+            OtherDB = DBs.DBs[Alias.AliasType]
+            OtherStruct = OtherDB.Structs.get(Alias.Name)
+            if (OtherStruct == None):
+                logging.critical ("Couldn't find alias {0} in {1} DB".format(Alias.Name, OtherDB.ArchName))
+                Passed = False
+                continue
+
+            ThisAlias = CompareStructs(StructDef, OtherStruct)
+            if not (ThisAlias):
+                logging.error ("Couldn't Alias to Arch {0} successfully".format(OtherDB.ArchName))
+            Passed &= ThisAlias
+    return Passed
+
+def CompareCrossArch(DB1, DB2):
+    Passed = True
+    for StructKey, StructDef in DB1.Structs.items():
+        if (len(StructKey) == 0):
+            # XXX: Oops, shouldn't have anonymous structs
+            continue
+
+        logging.info ("Comparing crossArch {0}".format(StructDef.Name))
+        if (StructDef.ExpectFEXMatch):
+            Struct2 = DB2.Structs.get(StructDef.Name)
+            if (Struct2 == None):
+                logging.critical ("Couldn't find Struct {0} in {1} DB".format(StructDef.Name, DB2.ArchName))
+                Passed = False
+                continue
+
+            Passed &= CompareStructs(StructDef, Struct2)
+
+    return Passed
+
+def main():
+    if sys.version_info[0] < 3:
+        logging.critical ("Python 3 or a more recent version is required.")
+
+    if (len(sys.argv) < 2):
+        print ("usage: %s <options> <Header.hpp> <clang arguments...>" % (sys.argv[0]))
+        print ("\t-c1 <Type1>: Base Comparison Type");
+        print ("\t-c2 <Type2>: Second Comparison Type");
+        print ("\t-win: Parse Windows");
+        sys.exit ("\t-no-linux: Do not parse Linux");
+
+    ParseLinux = True
+    ParseWindows = False
+
+    Header = ""
+    Comparison1 = ""
+    Comparison2 = ""
+    BaseArgs = []
+
+    StartOfArgs = 0
+
+    # Parse our arguments
+    ArgIndex = 1
+    while ArgIndex < len(sys.argv):
+        Arg = sys.argv[ArgIndex]
+        if (Arg == "--"):
+            StartOfArgs = ArgIndex + 1
+            break;
+
+        if (Arg == "-c1"):
+            ArgIndex += 1
+            Comparison1 = sys.argv[ArgIndex]
+        elif (Arg == "-c2"):
+            ArgIndex += 1
+            Comparison2 = sys.argv[ArgIndex]
+        elif (Arg == "-win"):
+            ParseWindows = True
+        elif (Arg == "-no-linux"):
+           ParseLinux = False
+        else:
+            Header = Arg
+            StartOfArgs = ArgIndex + 1
+            break
+
+        # Increment
+        ArgIndex += 1
+
+    # Add arguments for clang
+    for ArgIndex in range(StartOfArgs, len(sys.argv)):
+        BaseArgs.append(sys.argv[ArgIndex])
+
+    args_x86_32 = [
+        "-I/usr/i686-linux-gnu/include/c++/10/i686-linux-gnu/",
+        "-I/usr/i686-linux-gnu/include/",
+        "-O2",
+        "-m32",
+        "--target=i686-linux-unknown",
+    ]
+
+    args_x86_64 = [
+        "-I/usr/include/x86_64-linux-gnu",
+        "-I/usr/x86_64-linux-gnu/include/c++/10/x86_64-linux-gnu/",
+        "-I/usr/x86_64-linux-gnu/include/",
+        "-O2",
+        "--target=x86_64-linux-unknown",
+    ]
+
+    args_aarch64 = [
+        "-I/usr/aarch64-linux-gnu/include/c++/10/aarch64-linux-gnu/",
+        "-I/usr/aarch64-linux-gnu/include/",
+        "-O2",
+        "--target=aarch64-linux-unknown",
+    ]
+
+    args_x86_win32 = [
+        "-I/usr/lib/gcc/i686-w64-mingw32/10-win32/include/c++/",
+        "-I/usr/lib/gcc/i686-w64-mingw32/10-win32/include/c++/i686-w64-mingw32/",
+        "-O2",
+        "-m32",
+        "--target=i686-pc-win32",
+    ]
+
+    args_x86_win64 = [
+        "-I/usr/lib/gcc/x86_64-w64-mingw32/10-win32/include/c++/",
+        "-I/usr/lib/gcc/x86_64-w64-mingw32/10-win32/include/c++/x86_64-w64-mingw32/",
+        "-O2",
+        "--target=x86_64-pc-win32",
+    ]
+
+    # Add all the arguments to the different lists
+    args_x86_32.extend(BaseArgs)
+    args_x86_64.extend(BaseArgs)
+    args_aarch64.extend(BaseArgs)
+    args_x86_win32.extend(BaseArgs)
+    args_x86_win64.extend(BaseArgs)
+
+    # We need to find the default arguments through clang invocations
+    args_x86_32 = FindClangArguments(args_x86_32)
+    args_x86_64 = FindClangArguments(args_x86_64)
+    args_aarch64 = FindClangArguments(args_aarch64)
+    args_x86_win32 = FindClangArguments(args_x86_win32)
+    args_x86_win64 = FindClangArguments(args_x86_win64)
+
+    Arch_x86_32 = ArchDB("x86_32")
+    Arch_x86_64 = ArchDB("x86_64")
+    Arch_aarch64 = ArchDB("aarch64")
+    Arch_x86_win32 = ArchDB("win32")
+    Arch_x86_win64 = ArchDB("win64")
+
+    if (ParseLinux):
+        Arch_x86_32 = GetDB(Arch_x86_32, Header, args_x86_32)
+        Arch_x86_64 = GetDB(Arch_x86_64, Header, args_x86_64)
+        Arch_aarch64 = GetDB(Arch_aarch64, Header, args_aarch64)
+
+        if not (Arch_x86_32.Parsed):
+            logging.critical ("Couldn't parse:{0}".format(Arch_x86_32.ArchName))
+
+        if not (Arch_x86_64.Parsed):
+            logging.critical ("Couldn't parse:{0}".format(Arch_x86_64.ArchName))
+
+        if not (Arch_aarch64.Parsed):
+            logging.critical ("Couldn't parse:{0}".format(Arch_aarch64.ArchName))
+
+    if (ParseWindows):
+        Arch_x86_win32 = GetDB(Arch_x86_win32, Header, args_x86_win32)
+        Arch_x86_win64 = GetDB(Arch_x86_win64, Header, args_x86_win64)
+
+        if not (Arch_x86_win32.Parsed):
+            logging.critical ("Couldn't parse:{0}".format(Arch_x86_win32.ArchName))
+
+        if not (Arch_x86_win64.Parsed):
+            logging.critical ("Couldn't parse:{0}".format(Arch_x86_win64.ArchName))
+
+    DBs = DBList(Arch_x86_32,
+        Arch_x86_64,
+        Arch_aarch64,
+        Arch_x86_win32,
+        Arch_x86_win64)
+
+    Result = 0
+    if (len(Comparison1) != 0 and len(Comparison2) != 0):
+        CompDB1 = GetCompar(Comparison1, DBs)
+        CompDB2 = GetCompar(Comparison2, DBs)
+
+        # Now compare across the two compared architectures
+        Result = 0 if CompareCrossArch(CompDB1, CompDB2) else 1
+    elif (len(Comparison1) != 0):
+        CompDB1 = GetCompar(Comparison1, DBs)
+
+        # First compare the aliases to make sure we are matching
+        Result = 0 if CompareAliases(CompDB1, DBs) else 1
+
+    if (Result == 1):
+        logging.error("Execution environment")
+        Args = "[ "
+        for Arg in sys.argv:
+            Args += Arg + ", "
+        Args += " ]"
+        logging.error(Args)
+        Args = ""
+        for Arg in sys.argv:
+            Args += "\"" + Arg + "\" "
+
+        logging.error(Args)
+    return Result
+
+if __name__ == "__main__":
+# execute only if run as a script
+    sys.exit(main())

Source/Tests/LinuxSyscalls/CMakeLists.txt

@@ -55,3 +55,68 @@ add_library(LinuxEmulation STATIC
 
 target_link_libraries(LinuxEmulation FEXCore pthread numa)
 target_include_directories(LinuxEmulation PRIVATE ${CMAKE_BINARY_DIR}/generated)
+
+set(HEADERS_TO_VERIFY
+  x32/Types.h x86_32 # This needs to match structs to 32bit structs
+  )
+
+list(LENGTH HEADERS_TO_VERIFY ARG_COUNT)
+math(EXPR ARG_COUNT "${ARG_COUNT}-1")
+
+set (ARGS
+  "-x" "c++"
+  "-std=c++20")
+# Global include directories
+get_directory_property (INC_DIRS INCLUDE_DIRECTORIES)
+list(TRANSFORM INC_DIRS PREPEND "-I")
+list(APPEND ARGS ${INC_DIRS})
+
+# FEXCore directories
+get_target_property(INC_DIRS FEXCore INTERFACE_INCLUDE_DIRECTORIES)
+list(TRANSFORM INC_DIRS PREPEND "-I")
+list(APPEND ARGS ${INC_DIRS})
+
+foreach(Index RANGE 0 ${ARG_COUNT} 2)
+  math(EXPR TEST_TYPE_INDEX "${Index}+1")
+
+  list(GET HEADERS_TO_VERIFY ${Index} HEADER)
+  list(GET HEADERS_TO_VERIFY ${TEST_TYPE_INDEX} TEST_TYPE)
+
+  file(RELATIVE_PATH REL_HEADER ${CMAKE_BINARY_DIR} "${CMAKE_CURRENT_SOURCE_DIR}/${HEADER}")
+  set(TEST_NAME "${TEST_DESC}/Test_verify_${HEADER}")
+  set(TEST_NAME_ARCH "${TEST_DESC}/Test_verify_arch_${HEADER}")
+
+  add_test(
+    NAME ${TEST_NAME}_x86_64
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "x86_64" "${REL_HEADER}" ${ARGS})
+
+  add_test(
+    NAME ${TEST_NAME}_aarch64
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "aarch64" "${REL_HEADER}" ${ARGS})
+
+  add_test(
+    NAME ${TEST_NAME_ARCH}_x86_64
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "x86_64" "-c2" "${TEST_TYPE}" "${REL_HEADER}" ${ARGS})
+
+  add_test(
+    NAME ${TEST_NAME_ARCH}_aarch64
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py" "-c1" "aarch64" "-c2" "${TEST_TYPE}" "${REL_HEADER}" ${ARGS})
+
+  set_property(TEST ${TEST_NAME}_x86_64 APPEND PROPERTY DEPENDS "${HEADER}")
+  set_property(TEST ${TEST_NAME}_aarch64 APPEND PROPERTY DEPENDS "${HEADER}")
+  set_property(TEST ${TEST_NAME_ARCH}_x86_64 APPEND PROPERTY DEPENDS "${HEADER}")
+  set_property(TEST ${TEST_NAME_ARCH}_aarch64 APPEND PROPERTY DEPENDS "${HEADER}")
+endforeach()
+
+execute_process(COMMAND "nproc" OUTPUT_VARIABLE CORES)
+string(STRIP ${CORES} CORES)
+
+add_custom_target(
+  struct_verifier
+  WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+  USES_TERMINAL
+  COMMAND "ctest" "--timeout" "302" "-j${CORES}" "-R" "Test_verify*")

Source/Tests/LinuxSyscalls/x32/Types.h

@@ -22,6 +22,18 @@ using compat_long_t = int32_t;
 using compat_uptr_t = uint32_t;
 using compat_size_t = uint32_t;
 using compat_off_t = uint32_t;
+using compat_loff_t = int64_t;
+using compat_pid_t = int32_t;
+using compat_dev_t = uint16_t;
+using compat_ino_t = uint32_t;
+using compat_mode_t = uint16_t;
+using compat_nlink_t = uint16_t;
+using compat_uid_t = uint16_t;
+using compat_gid_t = uint16_t;
+
+// Can't use using with aligned attributes, clang doesn't honour it
+typedef __attribute__((aligned(4))) uint64_t compat_uint64_t;
+typedef __attribute__((aligned(4))) int64_t compat_int64_t;
 
 template<typename T>
 class compat_ptr {
@@ -81,7 +93,10 @@ static_assert(sizeof(compat_ptr<void>) == 4, "Incorrect size");
  * Provides conversation operators for the host version
  * @{ */
 
-struct timespec32 {
+struct
+__attribute__((annotate("alias-x86_32-timespec")))
+__attribute__((annotate("fex-match")))
+timespec32 {
   int32_t tv_sec;
   int32_t tv_nsec;
 
@@ -111,7 +126,10 @@ static_assert(sizeof(timespec32) == 8, "Incorrect size");
  * Provides conversation operators for the host version
  * @{ */
 
-struct timeval32 {
+struct
+__attribute__((annotate("alias-x86_32-timeval")))
+__attribute__((annotate("fex-match")))
+timeval32 {
   int32_t tv_sec;
   int32_t tv_usec;
 
@@ -141,7 +159,10 @@ static_assert(sizeof(timeval32) == 8, "Incorrect size");
  * Provides conversation operators for the host version
  * @{ */
 
-struct iovec32 {
+struct
+__attribute__((annotate("alias-x86_32-iovec")))
+__attribute__((annotate("fex-match")))
+iovec32 {
   uint32_t iov_base;
   uint32_t iov_len;
 
@@ -164,17 +185,23 @@ static_assert(std::is_trivial<iovec32>::value, "Needs to be trivial");
 static_assert(sizeof(iovec32) == 8, "Incorrect size");
 /**  @} */
 
-struct cmsghdr32 {
+struct
+__attribute__((annotate("alias-x86_32-cmsghdr")))
+__attribute__((annotate("fex-match")))
+cmsghdr32 {
   uint32_t cmsg_len;
   int32_t cmsg_level;
   int32_t cmsg_type;
-  char    cmsg_data[0];
+  char    cmsg_data[];
 };
 
 static_assert(std::is_trivial<cmsghdr32>::value, "Needs to be trivial");
 static_assert(sizeof(cmsghdr32) == 12, "Incorrect size");
 
-struct msghdr32 {
+struct
+__attribute__((annotate("alias-x86_32-msghdr")))
+__attribute__((annotate("fex-match")))
+msghdr32 {
   compat_ptr<void> msg_name;
   socklen_t msg_namelen;
 
@@ -189,7 +216,10 @@ struct msghdr32 {
 static_assert(std::is_trivial<msghdr32>::value, "Needs to be trivial");
 static_assert(sizeof(msghdr32) == 28, "Incorrect size");
 
-struct mmsghdr_32 {
+struct
+__attribute__((annotate("alias-x86_32-mmsghdr")))
+__attribute__((annotate("fex-match")))
+mmsghdr_32 {
   msghdr32 msg_hdr;
   uint32_t msg_len;
 };
@@ -197,7 +227,10 @@ struct mmsghdr_32 {
 static_assert(std::is_trivial<mmsghdr_32>::value, "Needs to be trivial");
 static_assert(sizeof(mmsghdr_32) == 32, "Incorrect size");
 
-struct stack_t32 {
+struct
+__attribute__((annotate("alias-x86_32-stack_t")))
+__attribute__((annotate("fex-match")))
+stack_t32 {
   compat_ptr<void> ss_sp;
   compat_size_t ss_size;
   int32_t ss_flags;
@@ -222,17 +255,22 @@ struct stack_t32 {
 static_assert(std::is_trivial<stack_t32>::value, "Needs to be trivial");
 static_assert(sizeof(stack_t32) == 12, "Incorrect size");
 
-struct stat32 {
-  uint32_t st_dev;
-  uint32_t st_ino;
-  uint32_t st_nlink;
+struct
+// This does not match the glibc implementation of stat
+// Matches the definition of `struct compat_stat` in `arch/x86/include/asm/compat.h`
+__attribute__((annotate("fex-match")))
+stat32 {
+  compat_dev_t st_dev;
+  uint16_t __pad1;
+  compat_ino_t st_ino;
+  compat_mode_t st_mode;
+  compat_nlink_t st_nlink;
 
-  uint16_t st_mode;
-  uint16_t st_uid;
-  uint16_t st_gid;
-  uint16_t __pad0;
+  compat_uid_t st_uid;
+  compat_gid_t st_gid;
+  compat_dev_t st_rdev;
 
-  uint32_t st_rdev;
+  uint16_t __pad2;
   uint32_t st_size;
   uint32_t st_blksize;
   uint32_t st_blocks;  /* Number 512-byte blocks allocated. */
@@ -242,7 +280,8 @@ struct stat32 {
   uint32_t st_mtime_nsec;
   uint32_t st_ctime_;
   uint32_t st_ctime_nsec;
-  uint32_t __unused[3];
+  uint32_t __unused4;
+  uint32_t __unused5;
 
   stat32() = delete;
 
@@ -250,13 +289,13 @@ struct stat32 {
     #define COPY(x) x = host.x
     COPY(st_dev);
     COPY(st_ino);
+    COPY(st_mode);
     COPY(st_nlink);
 
-    COPY(st_mode);
     COPY(st_uid);
     COPY(st_gid);
-
     COPY(st_rdev);
+
     COPY(st_size);
     COPY(st_blksize);
     COPY(st_blocks);
@@ -273,11 +312,16 @@ struct stat32 {
   }
 };
 static_assert(std::is_trivial<stat32>::value, "Needs to be trivial");
-static_assert(sizeof(stat32) == 72, "Incorrect size");
+static_assert(sizeof(stat32) == 64, "Incorrect size");
 
-struct __attribute__((packed)) stat64_32 {
-  uint64_t st_dev;
-  uint32_t pad0;
+struct
+// This does not match the glibc implementation of stat
+// Matches the definition of `struct stat64` in `x86_64-linux-gnu/asm/stat.h`
+__attribute__((annotate("fex-match")))
+__attribute__((packed))
+stat64_32 {
+  compat_uint64_t st_dev;
+  uint8_t  __pad0[4];
   uint32_t __st_ino;
 
   uint32_t st_mode;
@@ -286,18 +330,18 @@ struct __attribute__((packed)) stat64_32 {
   uint32_t st_uid;
   uint32_t st_gid;
 
-  uint64_t st_rdev;
-  uint32_t pad3;
-  int64_t st_size;
+  compat_uint64_t st_rdev;
+  uint8_t  __pad3[4];
+  compat_int64_t st_size;
   uint32_t st_blksize;
-  uint64_t st_blocks;  /* Number 512-byte blocks allocated. */
+  compat_uint64_t st_blocks;  /* Number 512-byte blocks allocated. */
   uint32_t st_atime_;
   uint32_t st_atime_nsec;
   uint32_t st_mtime_;
   uint32_t st_mtime_nsec;
   uint32_t st_ctime_;
   uint32_t st_ctime_nsec;
-  uint64_t st_ino;
+  compat_uint64_t st_ino;
 
   stat64_32() = delete;
 
@@ -360,14 +404,18 @@ struct __attribute__((packed)) stat64_32 {
 static_assert(std::is_trivial<stat64_32>::value, "Needs to be trivial");
 static_assert(sizeof(stat64_32) == 96, "Incorrect size");
 
-struct __attribute__((packed,aligned(4))) statfs64_32 {
+struct
+__attribute__((packed,aligned(4)))
+__attribute__((annotate("alias-x86_32-statfs64")))
+__attribute__((annotate("fex-match")))
+statfs64_32 {
   uint32_t f_type;
   uint32_t f_bsize;
-  uint64_t f_blocks;
-  uint64_t f_bfree;
-  uint64_t f_bavail;
-  uint64_t f_files;
-  uint64_t f_ffree;
+  compat_uint64_t f_blocks;
+  compat_uint64_t f_bfree;
+  compat_uint64_t f_bavail;
+  compat_uint64_t f_files;
+  compat_uint64_t f_ffree;
   __kernel_fsid_t f_fsid;
   uint32_t f_namelen;
   uint32_t f_frsize;
@@ -413,7 +461,10 @@ struct __attribute__((packed,aligned(4))) statfs64_32 {
 static_assert(std::is_trivial<statfs64_32>::value, "Needs to be trivial");
 static_assert(sizeof(statfs64_32) == 84, "Incorrect size");
 
-struct flock_32 {
+struct
+__attribute__((annotate("alias-x86_32-flock")))
+__attribute__((annotate("fex-match")))
+flock_32 {
   int16_t l_type;
   int16_t l_whence;
   int32_t l_start;
@@ -445,12 +496,17 @@ static_assert(std::is_trivial<flock_32>::value, "Needs to be trivial");
 static_assert(sizeof(flock_32) == 16, "Incorrect size");
 
 // glibc doesn't pack flock64 while the kernel does
-struct flock64_32 {
+// This does not match glibc flock64 definition
+// Matches the definition of `struct compat_flock64` in `arch/x86/include/asm/compat.h`
+struct
+__attribute__((annotate("fex-match")))
+__attribute__((packed))
+flock64_32 {
   int16_t l_type;
   int16_t l_whence;
-  int32_t l_start;
-  int32_t l_len;
-  int32_t l_pid;
+  compat_loff_t l_start;
+  compat_loff_t l_len;
+  compat_pid_t l_pid;
 
   flock64_32() = delete;
 
@@ -473,9 +529,13 @@ struct flock64_32 {
   }
 };
 static_assert(std::is_trivial<flock64_32>::value, "Needs to be trivial");
-static_assert(sizeof(flock64_32) == 16, "Incorrect size");
+static_assert(sizeof(flock64_32) == 24, "Incorrect size");
 
-struct linux_dirent {
+// There is no public definition of this struct
+// Matches the definition of `struct linux_dirent` in fs/readdir.c
+struct
+__attribute__((annotate("fex-match")))
+linux_dirent {
   uint64_t d_ino;
   int64_t  d_off;
   uint16_t d_reclen;
@@ -485,18 +545,25 @@ struct linux_dirent {
 static_assert(std::is_trivial<linux_dirent>::value, "Needs to be trivial");
 static_assert(sizeof(linux_dirent) == 24, "Incorrect size");
 
-struct linux_dirent_32 {
-  uint32_t d_ino;
-  int32_t d_off;
+// There is no public definition of this struct
+// Matches the definition of `struct compat_linux_dirent` in fs/readdir.c
+struct
+__attribute__((annotate("fex-match")))
+linux_dirent_32 {
+  compat_ulong_t d_ino;
+  compat_ulong_t d_off;
   uint16_t d_reclen;
-  uint8_t _pad[2];
-  char d_name[];
+  char d_name[1];
   /* Has hidden null character and d_type */
 };
 static_assert(std::is_trivial<linux_dirent_32>::value, "Needs to be trivial");
 static_assert(sizeof(linux_dirent_32) == 12, "Incorrect size");
 
-struct linux_dirent_64 {
+// There is no public definition of this struct
+// Matches the definition of `struct linux_dirent64` in include/linux/dirent.h
+struct
+__attribute__((annotate("fex-match")))
+linux_dirent_64 {
   uint64_t d_ino;
   uint64_t d_off;
   uint16_t d_reclen;
@@ -507,31 +574,80 @@ struct linux_dirent_64 {
 static_assert(std::is_trivial<linux_dirent_64>::value, "Needs to be trivial");
 static_assert(sizeof(linux_dirent_64) == 24, "Incorrect size");
 
-struct sigset_argpack32 {
+// There is no public definition of this struct
+// Matches `struct compat_sigset_argpack`
+struct
+__attribute__((annotate("fex-match")))
+sigset_argpack32 {
   compat_ptr<uint64_t> sigset;
-  size_t size;
+  compat_size_t size;
 };
 
 static_assert(std::is_trivial<sigset_argpack32>::value, "Needs to be trivial");
-static_assert(sizeof(sigset_argpack32) == 16, "Incorrect size");
+static_assert(sizeof(sigset_argpack32) == 8, "Incorrect size");
 
-struct rusage_32 {
+struct
+__attribute__((annotate("alias-x86_32-rusage")))
+__attribute__((annotate("fex-match")))
+rusage_32 {
   timeval32 ru_utime;
   timeval32 ru_stime;
-  compat_long_t ru_maxrss;
-  compat_long_t ru_ixrss;
-  compat_long_t ru_idrss;
-  compat_long_t ru_isrss;
-  compat_long_t ru_minflt;
-  compat_long_t ru_majflt;
-  compat_long_t ru_nswap;
-  compat_long_t ru_inblock;
-  compat_long_t ru_oublock;
-  compat_long_t ru_msgsnd;
-  compat_long_t ru_msgrcv;
-  compat_long_t ru_nsignals;
-  compat_long_t ru_nvcsw;
-  compat_long_t ru_nivcsw;
+  union {
+    compat_long_t ru_maxrss;
+    compat_long_t __ru_maxrss_word;
+  };
+  union {
+    compat_long_t ru_ixrss;
+    compat_long_t __ru_ixrss_word;
+  };
+  union {
+    compat_long_t ru_idrss;
+    compat_long_t __ru_idrss_word;
+  };
+  union {
+    compat_long_t ru_isrss;
+    compat_long_t __ru_isrss_word;
+  };
+  union {
+    compat_long_t ru_minflt;
+    compat_long_t __ru_minflt_word;
+  };
+  union {
+    compat_long_t ru_majflt;
+    compat_long_t __ru_majflt_word;
+  };
+  union {
+    compat_long_t ru_nswap;
+    compat_long_t __ru_nswap_word;
+  };
+  union {
+    compat_long_t ru_inblock;
+    compat_long_t __ru_inblock_word;
+  };
+  union {
+    compat_long_t ru_oublock;
+    compat_long_t __ru_oublock_word;
+  };
+  union {
+    compat_long_t ru_msgsnd;
+    compat_long_t __ru_msgsnd_word;
+  };
+  union {
+    compat_long_t ru_msgrcv;
+    compat_long_t __ru_msgrcv_word;
+  };
+  union {
+    compat_long_t ru_nsignals;
+    compat_long_t __ru_nsignals_word;
+  };
+  union {
+    compat_long_t ru_nvcsw;
+    compat_long_t __ru_nvcsw_word;
+  };
+  union {
+    compat_long_t ru_nivcsw;
+    compat_long_t __ru_nivcsw_word;
+  };
 
   rusage_32() = delete;
   rusage_32(struct rusage usage)
@@ -579,10 +695,16 @@ struct rusage_32 {
 static_assert(std::is_trivial<rusage_32>::value, "Needs to be trivial");
 static_assert(sizeof(rusage_32) == 72, "Incorrect size");
 
-struct __attribute__((packed)) GuestSigAction_32 {
+// This definition isn't public
+// This is for rt_sigaction
+// Matches the definition for `struct compat_sigaction` in `include/linux/compat.h`
+struct
+__attribute__((packed))
+__attribute__((annotate("fex-match")))
+GuestSigAction_32 {
   FEX::HLE::x32::compat_ptr<void> handler_32;
 
-  uint64_t sa_flags;
+  uint32_t sa_flags;
   FEX::HLE::x32::compat_ptr<void> restorer_32;
   FEXCore::GuestSAMask sa_mask;
 
@@ -607,6 +729,6 @@ struct __attribute__((packed)) GuestSigAction_32 {
 };
 
 static_assert(std::is_trivial<GuestSigAction_32>::value, "Needs to be trivial");
-static_assert(sizeof(GuestSigAction_32) == 24, "Incorrect size");
+static_assert(sizeof(GuestSigAction_32) == 20, "Incorrect size");
 
 }