From ed3af580c595fd0890e1fbcc256f30d5a2391064 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 29 Mar 2024 15:26:02 -0700 Subject: [PATCH] FEXCore: Move nearly all IR definitions to internal It has been a long time coming that FEX no longer needed to leak IR implementation details to the frontend, this was legacy due to IR CI and various other problems. Now that the last bits of IR leaking has been removed, move everything that we can internally to the implementation. We still have a couple of minor details in the exposed IR.h to the frontend, but these are limited to a few enums and some thunking struct information rather than all the implementation details. No functional change with this, just moving headers around. --- FEXCore/Source/Interface/Context/Context.h | 1 + FEXCore/Source/Interface/Core/Core.cpp | 4 +- .../Core/Interpreter/Fallbacks/F80Fallbacks.h | 3 +- .../Core/Interpreter/InterpreterOps.h | 1 - .../Interface/Core/JIT/Arm64/JITClass.h | 5 +- .../Interface/Core/OpcodeDispatcher.cpp | 1 - .../Source/Interface/Core/OpcodeDispatcher.h | 1 - .../Interface/Core/OpcodeDispatcher/Flags.cpp | 1 - .../Core/OpcodeDispatcher/Vector.cpp | 1 - .../Source/Interface/HLE/Thunks/Thunks.cpp | 2 +- FEXCore/Source/Interface/HLE/Thunks/Thunks.h | 3 +- FEXCore/Source/Interface/IR/AOTIR.cpp | 4 +- FEXCore/Source/Interface/IR/AOTIR.h | 3 +- FEXCore/Source/Interface/IR/IR.h | 617 +++++++++++++++++ FEXCore/Source/Interface/IR/IRDumper.cpp | 5 +- FEXCore/Source/Interface/IR/IREmitter.cpp | 1 - FEXCore/Source/Interface/IR/IREmitter.h | 5 +- FEXCore/Source/Interface/IR/IRParser.cpp | 2 +- .../Interface}/IR/IntrusiveIRList.h | 3 +- .../Source/Interface/IR/Passes/ConstProp.cpp | 1 - .../IR/Passes/DeadCodeElimination.cpp | 1 - .../IR/Passes/DeadContextStoreElimination.cpp | 2 +- .../IR/Passes/DeadStoreElimination.cpp | 1 - .../Interface/IR/Passes/IRCompaction.cpp | 1 - .../Interface/IR/Passes/IRValidation.cpp | 3 +- .../IR/Passes/InlineCallOptimization.cpp | 1 - .../IR/Passes/LongDivideRemovalPass.cpp | 1 - .../Interface/IR/Passes/RAValidation.cpp | 3 +- .../RedundantFlagCalculationElimination.cpp | 1 - .../IR/Passes/RegisterAllocationPass.cpp | 4 +- .../IR/Passes/ValueDominanceValidation.cpp | 1 - .../Interface}/IR/RegisterAllocationData.h | 0 FEXCore/include/FEXCore/Core/CoreState.h | 4 +- .../FEXCore/Debug/InternalThreadState.h | 2 - FEXCore/include/FEXCore/IR/IR.h | 644 +----------------- 35 files changed, 658 insertions(+), 675 deletions(-) rename FEXCore/{include/FEXCore => Source/Interface}/IR/IntrusiveIRList.h (99%) rename FEXCore/{include/FEXCore => Source/Interface}/IR/RegisterAllocationData.h (100%) diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index 1887890d2..b7ec385b8 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index f262cb6ef..331c38145 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -25,6 +25,7 @@ $end_info$ #include "Interface/IR/Passes/RegisterAllocationPass.h" #include "Interface/IR/Passes.h" #include "Interface/IR/PassManager.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Utils/Allocator.h" #include "Utils/Allocator/HostAllocator.h" @@ -39,9 +40,6 @@ $end_info$ #include #include #include -#include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h index 69ee572d3..7bc5ce745 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h +++ b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h @@ -2,9 +2,8 @@ #include "Common/SoftFloat.h" #include "Common/SoftFloat-3e/softfloat.h" -#include - #include "Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h" +#include "Interface/IR/IR.h" namespace FEXCore::CPU { FEXCORE_PRESERVE_ALL_ATTR diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index c103dd118..9b7727f57 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -7,7 +7,6 @@ #include #include -#include namespace FEXCore::IR { class IRListView; diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index 6d0838ac6..5dde2dde4 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -10,6 +10,9 @@ $end_info$ #include "Interface/Core/ArchHelpers/Arm64Emitter.h" #include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/Dispatcher/Dispatcher.h" +#include "Interface/IR/IR.h" +#include "Interface/IR/IntrusiveIRList.h" +#include "Interface/IR/RegisterAllocationData.h" #include #include @@ -17,8 +20,6 @@ $end_info$ #include #include #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index 12091359d..13e60b928 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -18,7 +18,6 @@ $end_info$ #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index d9f8fb08b..e059a92ce 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp index b8aa8371f..5c8ebbd6b 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp @@ -13,7 +13,6 @@ $end_info$ #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp index 3ee2d7515..2274ec4a9 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp @@ -13,7 +13,6 @@ $end_info$ #include #include #include -#include #include #include diff --git a/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp b/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp index 48faeee99..6842b0e71 100644 --- a/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp +++ b/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp @@ -6,13 +6,13 @@ tags: glue|thunks $end_info$ */ +#include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include #include #include #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/HLE/Thunks/Thunks.h b/FEXCore/Source/Interface/HLE/Thunks/Thunks.h index d7933e5dd..6d6437f57 100644 --- a/FEXCore/Source/Interface/HLE/Thunks/Thunks.h +++ b/FEXCore/Source/Interface/HLE/Thunks/Thunks.h @@ -7,7 +7,8 @@ $end_info$ #pragma once -#include +#include "Interface/IR/IR.h" + #include #include diff --git a/FEXCore/Source/Interface/IR/AOTIR.cpp b/FEXCore/Source/Interface/IR/AOTIR.cpp index c16ab231a..a09bdfffe 100644 --- a/FEXCore/Source/Interface/IR/AOTIR.cpp +++ b/FEXCore/Source/Interface/IR/AOTIR.cpp @@ -2,9 +2,9 @@ #include "FEXHeaderUtils/Filesystem.h" #include "Interface/Context/Context.h" #include "Interface/IR/AOTIR.h" +#include "Interface/IR/IntrusiveIRList.h" +#include "Interface/IR/RegisterAllocationData.h" -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/AOTIR.h b/FEXCore/Source/Interface/IR/AOTIR.h index 4dfc5319c..b128b8ab6 100644 --- a/FEXCore/Source/Interface/IR/AOTIR.h +++ b/FEXCore/Source/Interface/IR/AOTIR.h @@ -1,7 +1,8 @@ // SPDX-License-Identifier: MIT #pragma once -#include "FEXCore/IR/RegisterAllocationData.h" +#include "Interface/IR/RegisterAllocationData.h" + #include #include #include diff --git a/FEXCore/Source/Interface/IR/IR.h b/FEXCore/Source/Interface/IR/IR.h index c92a7a433..77d1930d2 100644 --- a/FEXCore/Source/Interface/IR/IR.h +++ b/FEXCore/Source/Interface/IR/IR.h @@ -9,11 +9,628 @@ namespace FEXCore::IR { +class OrderedNode; +class RegisterAllocationPass; class RegisterAllocationData; +/** + * @brief The IROp_Header is an dynamically sized array + * At the end it contains a uint8_t for the number of arguments that Op has + * Then there is an unsized array of NodeWrapper arguments for the number of arguments this op has + * The op structures that are including the header must ensure that they pad themselves correctly to the number of arguments used + */ +struct IROp_Header; + +/** + * @brief Represents the ID of a given IR node. + * + * Intended to provide strong typing from other integer values + * to prevent passing incorrect values to certain API functions. + */ +struct NodeID final { + using value_type = uint32_t; + + constexpr NodeID() noexcept = default; + constexpr explicit NodeID(value_type Value_) noexcept : Value{Value_} {} + + constexpr NodeID(const NodeID&) noexcept = default; + constexpr NodeID& operator=(const NodeID&) noexcept = default; + + constexpr NodeID(NodeID&&) noexcept = default; + constexpr NodeID& operator=(NodeID&&) noexcept = default; + + [[nodiscard]] constexpr bool IsValid() const noexcept { + return Value != 0; + } + [[nodiscard]] constexpr bool IsInvalid() const noexcept { + return !IsValid(); + } + constexpr void Invalidate() noexcept { + Value = 0; + } + + [[nodiscard]] friend constexpr bool operator==(NodeID, NodeID) noexcept = default; + + [[nodiscard]] friend constexpr bool operator<(NodeID lhs, NodeID rhs) noexcept { + return lhs.Value < rhs.Value; + } + [[nodiscard]] friend constexpr bool operator>(NodeID lhs, NodeID rhs) noexcept { + return operator<(rhs, lhs); + } + [[nodiscard]] friend constexpr bool operator<=(NodeID lhs, NodeID rhs) noexcept { + return !operator>(lhs, rhs); + } + [[nodiscard]] friend constexpr bool operator>=(NodeID lhs, NodeID rhs) noexcept { + return !operator<(lhs, rhs); + } + + friend std::ostream& operator<<(std::ostream& out, NodeID ID) { + out << ID.Value; + return out; + } + friend std::istream& operator>>(std::istream& in, NodeID& ID) { + in >> ID.Value; + return in; + } + + value_type Value{}; +}; + +/** + * @brief This is a very simple wrapper for our node pointers + * You probably don't want to use this directly + * Use OpNodeWrapper and OrderedNodeWrapper types below instead + * + * This is necessary to allow two things + * - Reduce memory usage by having the pointer be an 32bit offset rather than the whole 64bit pointer + * - Actually use an offset from a base so we aren't storing pointers for everything + * - Makes IR list copying be as cheap as a memcpy + * Downsides + * - The IR nodes have to be allocated out of a linear array of memory + * - We currently only allow a 32bit offset, so *only* 4 million nodes per list + * - We have to have the base offset live somewhere else + * - Has to be POD and trivially copyable + * - Makes every real node access turn in to a [Base + Offset] access + * - Can be confusing if you're mixing OpNodeWrapper and OrderedNodeWrapper usage + */ +template +struct NodeWrapperBase final { + // On x86-64 using a uint64_t type is more efficient since RIP addressing gives you [ + + ] + // On AArch64 using uint32_t is just more memory efficient. 32bit or 64bit offset doesn't matter + // We use uint32_t to be more memory efficient (Cuts our node list size in half) + using NodeOffsetType = uint32_t; + NodeOffsetType NodeOffset; + + explicit NodeWrapperBase() = default; + + [[nodiscard]] static NodeWrapperBase WrapOffset(NodeOffsetType Offset) { + NodeWrapperBase Wrapped; + Wrapped.NodeOffset = Offset; + return Wrapped; + } + + [[nodiscard]] static NodeWrapperBase WrapPtr(uintptr_t Base, uintptr_t Value) { + NodeWrapperBase Wrapped; + Wrapped.SetOffset(Base, Value); + return Wrapped; + } + + [[nodiscard]] static void *UnwrapNode(uintptr_t Base, NodeWrapperBase Node) { + return Node.GetNode(Base); + } + + [[nodiscard]] NodeID ID() const; + + [[nodiscard]] bool IsInvalid() const { return NodeOffset == 0; } + + [[nodiscard]] Type *GetNode(uintptr_t Base) { + return reinterpret_cast(Base + NodeOffset); + } + [[nodiscard]] const Type *GetNode(uintptr_t Base) const { + return reinterpret_cast(Base + NodeOffset); + } + + void SetOffset(uintptr_t Base, uintptr_t Value) { NodeOffset = Value - Base; } + + [[nodiscard]] friend constexpr bool operator==(const NodeWrapperBase&, const NodeWrapperBase&) = default; +}; + +static_assert(std::is_trivial_v>); + +static_assert(sizeof(NodeWrapperBase) == sizeof(uint32_t)); + +using OpNodeWrapper = NodeWrapperBase; +using OrderedNodeWrapper = NodeWrapperBase; + +struct OrderedNodeHeader { + OpNodeWrapper Value; + OrderedNodeWrapper Next; + OrderedNodeWrapper Previous; +}; + +static_assert(sizeof(OrderedNodeHeader) == sizeof(uint32_t) * 3); + +/** + * @brief This is a node in our IR representation + * Is a doubly linked list node that lives in a representation of a linearly allocated node list + * The links in the nodes can live in a list independent of the data IR data + * + * ex. + * Region1 : ... <-> <-> <-> ... + * | * | + * v v + * Region2 : ...... + * + * In this example the OrderedNodes are allocated in one linear memory region (Not necessarily contiguous with one another linking) + * The second region is contiguous but they don't have any relationship with one another directly + */ +class OrderedNode final { + friend class NodeWrapperIterator; + friend class OrderedList; + public: + // These three values are laid out very specifically to make it fast to access the NodeWrappers specifically + OrderedNodeHeader Header; + uint32_t NumUses; + + using value_type = OrderedNodeWrapper; + + OrderedNode() = default; + + /** + * @brief Appends a node to this current node + * + * Before. <-> <-> + * After. <-> <-> <-> Next + * + * @return Pointer to the node being added + */ + value_type append(uintptr_t Base, value_type Node) { + // Set Next Node's Previous to incoming node + SetPrevious(Base, Header.Next, Node); + + // Set Incoming node's links to this node's links + SetPrevious(Base, Node, Wrapped(Base)); + SetNext(Base, Node, Header.Next); + + // Set this node's next to the incoming node + SetNext(Base, Wrapped(Base), Node); + + // Return the node we are appending + return Node; + } + + OrderedNode *append(uintptr_t Base, OrderedNode *Node) { + value_type WNode = Node->Wrapped(Base); + // Set Next Node's Previous to incoming node + SetPrevious(Base, Header.Next, WNode); + + // Set Incoming node's links to this node's links + SetPrevious(Base, WNode, Wrapped(Base)); + SetNext(Base, WNode, Header.Next); + + // Set this node's next to the incoming node + SetNext(Base, Wrapped(Base), WNode); + + // Return the node we are appending + return Node; + } + + /** + * @brief Prepends a node to the current node + * Before. <-> <-> + * After. <-> <-> <-> Next + * + * @return Pointer to the node being added + */ + value_type prepend(uintptr_t Base, value_type Node) { + // Set the previous node's next to the incoming node + SetNext(Base, Header.Previous, Node); + + // Set the incoming node's links + SetPrevious(Base, Node, Header.Previous); + SetNext(Base, Node, Wrapped(Base)); + + // Set the current node's link + SetPrevious(Base, Wrapped(Base), Node); + + // Return the node we are prepending + return Node; + } + + OrderedNode *prepend(uintptr_t Base, OrderedNode *Node) { + value_type WNode = Node->Wrapped(Base); + // Set the previous node's next to the incoming node + SetNext(Base, Header.Previous, WNode); + + // Set the incoming node's links + SetPrevious(Base, WNode, Header.Previous); + SetNext(Base, WNode, Wrapped(Base)); + + // Set the current node's link + SetPrevious(Base, Wrapped(Base), WNode); + + // Return the node we are prepending + return Node; + } + + /** + * @brief Gets the remaining size of the blocks from this point onward + * + * Doesn't find the head of the list + * + */ + [[nodiscard]] size_t size(uintptr_t Base) const { + size_t Size = 1; + // Walk the list forward until we hit a sentinel + value_type Current = Header.Next; + while (Current.NodeOffset != 0) { + ++Size; + OrderedNode *RealNode = Current.GetNode(Base); + Current = RealNode->Header.Next; + } + return Size; + } + + void Unlink(uintptr_t Base) { + // This removes the node from the list. Orphaning it + // Before: <-> <-> + // After: + SetNext(Base, Header.Previous, Header.Next); + SetPrevious(Base, Header.Next, Header.Previous); + } + + [[nodiscard]] IROp_Header const* Op(uintptr_t Base) const { + return Header.Value.GetNode(Base); + } + [[nodiscard]] IROp_Header *Op(uintptr_t Base) { + return Header.Value.GetNode(Base); + } + + [[nodiscard]] uint32_t GetUses() const { return NumUses; } + + void AddUse() { ++NumUses; } + void RemoveUse() { --NumUses; } + + [[nodiscard]] value_type Wrapped(uintptr_t Base) const { + value_type Tmp; + Tmp.SetOffset(Base, reinterpret_cast(this)); + return Tmp; + } + + private: + [[nodiscard]] value_type WrappedOffset(uint32_t Offset) const { + value_type Tmp; + Tmp.NodeOffset = Offset; + return Tmp; + } + + static void SetPrevious(uintptr_t Base, value_type Node, value_type New) { + OrderedNode *RealNode = Node.GetNode(Base); + RealNode->Header.Previous = New; + } + + static void SetNext(uintptr_t Base, value_type Node, value_type New) { + OrderedNode *RealNode = Node.GetNode(Base); + RealNode->Header.Next = New; + } + + void SetUses(uint32_t Uses) { NumUses = Uses; } +}; + +static_assert(std::is_trivial_v); +static_assert(std::is_trivially_copyable_v); +static_assert(offsetof(OrderedNode, Header) == 0); +static_assert(sizeof(OrderedNode) == (sizeof(OrderedNodeHeader) + sizeof(uint32_t))); + +struct RegisterClassType final { + using value_type = uint32_t; + + value_type Val; + [[nodiscard]] constexpr operator value_type() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const RegisterClassType&, const RegisterClassType&) = default; +}; + +struct CondClassType final { + uint8_t Val; + [[nodiscard]] constexpr operator uint8_t() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const CondClassType&, const CondClassType&) = default; +}; + +struct MemOffsetType final { + uint8_t Val; + [[nodiscard]] constexpr operator uint8_t() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const MemOffsetType&, const MemOffsetType&) = default; +}; + +struct TypeDefinition final { + uint16_t Val; + + [[nodiscard]] constexpr operator uint16_t() const { + return Val; + } + + [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes) { + TypeDefinition Type{}; + Type.Val = Bytes << 8; + return Type; + } + + [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes, uint8_t Elements) { + TypeDefinition Type{}; + Type.Val = (Bytes << 8) | (Elements & 255); + return Type; + } + + [[nodiscard]] constexpr uint8_t Bytes() const { + return Val >> 8; + } + + [[nodiscard]] constexpr uint8_t Elements() const { + return Val & 255; + } + + [[nodiscard]] friend constexpr bool operator==(const TypeDefinition&, const TypeDefinition&) = default; +}; + +static_assert(std::is_trivial_v); + +struct FenceType final { + using value_type = uint8_t; + + value_type Val; + [[nodiscard]] constexpr operator value_type() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const FenceType&, const FenceType&) = default; +}; + +struct RoundType final { + uint8_t Val; + [[nodiscard]] constexpr operator uint8_t() const { + return Val; + } + [[nodiscard]] friend constexpr bool operator==(const RoundType&, const RoundType&) = default; +}; + +class NodeIterator; + +/* This iterator can be used to step though nodes. + * Due to how our IR is laid out, this can be used to either step + * though the CodeBlocks or though the code within a single block. + */ +class NodeIterator { +public: + using value_type = std::tuple; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator = NodeIterator; + using const_iterator = const NodeIterator; + using reverse_iterator = iterator; + using const_reverse_iterator = const_iterator; + using iterator_category = std::bidirectional_iterator_tag; + + NodeIterator(uintptr_t Base, uintptr_t IRBase) : BaseList {Base}, IRList{ IRBase } {} + explicit NodeIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : BaseList {Base}, IRList{ IRBase }, Node {Ptr} {} + + [[nodiscard]] bool operator==(const NodeIterator &rhs) const { + return Node.NodeOffset == rhs.Node.NodeOffset; + } + + [[nodiscard]] bool operator!=(const NodeIterator &rhs) const { + return !operator==(rhs); + } + + NodeIterator operator++() { + OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); + Node = RealNode->Next; + return *this; + } + + NodeIterator operator--() { + OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); + Node = RealNode->Previous; + return *this; + } + + [[nodiscard]] value_type operator*() { + OrderedNode *RealNode = Node.GetNode(BaseList); + return { RealNode, RealNode->Op(IRList) }; + } + + [[nodiscard]] value_type operator()() { + OrderedNode *RealNode = Node.GetNode(BaseList); + return { RealNode, RealNode->Op(IRList) }; + } + + [[nodiscard]] NodeID ID() const { + return Node.ID(); + } + + [[nodiscard]] static NodeIterator Invalid() { + return NodeIterator(0, 0); + } + +protected: + uintptr_t BaseList{}; + uintptr_t IRList{}; + OrderedNodeWrapper Node{}; +}; + +// This must directly match bytes to the named opsize. +// Implicit sized IR operations does math to get between sizes. +enum OpSize : uint8_t { + i8Bit = 1, + i16Bit = 2, + i32Bit = 4, + i64Bit = 8, + i128Bit = 16, + i256Bit = 32, +}; + +enum class FloatCompareOp : uint8_t { + EQ = 0, + LT, + LE, + UNO, + NEQ, + ORD, +}; + +enum class ShiftType : uint8_t { + LSL = 0, + LSR, + ASR, + ROR, +}; + + +// Converts a size stored as an integer in to an OpSize enum. +// This is a nop operation and will be eliminated by the compiler. +static inline OpSize SizeToOpSize(uint8_t Size) { + switch (Size) { + case 1: return OpSize::i8Bit; + case 2: return OpSize::i16Bit; + case 4: return OpSize::i32Bit; + case 8: return OpSize::i64Bit; + case 16: return OpSize::i128Bit; + case 32: return OpSize::i256Bit; + default: FEX_UNREACHABLE; + } +} + +#define IROP_ENUM +#define IROP_STRUCTS +#define IROP_SIZES +#define IROP_REG_CLASSES +#include + +/* This iterator can be used to step though every single node in a multi-block in SSA order. + * + * Iterates in the order of: + * + * end <-- CodeBlockA <--> BlockAInst1 <--> BlockAInst2 <--> CodeBlockB <--> BlockBInst1 <--> BlockBInst2 --> end + */ +class AllNodesIterator : public NodeIterator { +public: + AllNodesIterator(uintptr_t Base, uintptr_t IRBase) : NodeIterator(Base, IRBase) {} + explicit AllNodesIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : NodeIterator(Base, IRBase, Ptr) {} + AllNodesIterator(NodeIterator other) : NodeIterator(other) {} // Allow NodeIterator to be upgraded + + AllNodesIterator operator++() { + OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); + auto IROp = Node.GetNode(BaseList)->Op(IRList); + + // If this is the last node of a codeblock, we need to continue to the next block + if (IROp->Op == OP_ENDBLOCK) { + auto EndBlock = IROp->C(); + + auto CurrentBlock = EndBlock->BlockHeader.GetNode(BaseList); + Node = CurrentBlock->Header.Next; + } else if (IROp->Op == OP_CODEBLOCK) { + auto CodeBlock = IROp->C(); + + Node = CodeBlock->Begin; + } else { + Node = RealNode->Next; + } + + return *this; + } + + AllNodesIterator operator--() { + auto IROp = Node.GetNode(BaseList)->Op(IRList); + + if (IROp->Op == OP_BEGINBLOCK) { + auto BeginBlock = IROp->C(); + + Node = BeginBlock->BlockHeader; + } else if (IROp->Op == OP_CODEBLOCK) { + auto PrevBlockWrapper = Node.GetNode(BaseList)->Header.Previous; + auto PrevCodeBlock = PrevBlockWrapper.GetNode(BaseList)->Op(IRList)->C(); + + Node = PrevCodeBlock->Last; + } else { + Node = Node.GetNode(BaseList)->Header.Previous; + } + + return *this; + } + + [[nodiscard]] static AllNodesIterator Invalid() { + return AllNodesIterator(0, 0); + } +}; + class IRListView; class IREmitter; +template +inline NodeID NodeWrapperBase::ID() const { + return NodeID(NodeOffset / sizeof(IR::OrderedNode)); +} + +bool IsFragmentExit(FEXCore::IR::IROps Op); +bool IsBlockExit(FEXCore::IR::IROps Op); + void Dump(fextl::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData); fextl::unique_ptr Parse(FEXCore::Utils::IntrusivePooledAllocator &ThreadAllocator, fextl::stringstream &MapsStream); } + +template <> +struct std::hash { + size_t operator()(const FEXCore::IR::NodeID& ID) const noexcept { + return std::hash{}(ID.Value); + } +}; + +template <> +struct fmt::formatter : fmt::formatter { + using Base = fmt::formatter; + + // Pass-through the underlying value, so IDs can + // be formatted like any integral value. + template + auto format(const FEXCore::IR::NodeID& ID, FormatContext& ctx) const { + return Base::format(ID.Value, ctx); + } +}; + +template <> +struct fmt::formatter : fmt::formatter { + using Base = fmt::formatter; + + template + auto format(const FEXCore::IR::RegisterClassType& Class, FormatContext& ctx) const { + return Base::format(Class.Val, ctx); + } +}; + +template <> +struct fmt::formatter : fmt::formatter { + using Base = fmt::formatter; + + template + auto format(const FEXCore::IR::FenceType& Fence, FormatContext& ctx) const { + return Base::format(Fence.Val, ctx); + } +}; + +template <> +struct fmt::formatter : fmt::formatter> { + using Base = fmt::formatter>; + + template + auto format(const FEXCore::IR::OpSize& OpSize, FormatContext& ctx) const { + return Base::format(FEXCore::ToUnderlying(OpSize), ctx); + } +}; diff --git a/FEXCore/Source/Interface/IR/IRDumper.cpp b/FEXCore/Source/Interface/IR/IRDumper.cpp index 5cd4dcce0..b95e34832 100644 --- a/FEXCore/Source/Interface/IR/IRDumper.cpp +++ b/FEXCore/Source/Interface/IR/IRDumper.cpp @@ -6,9 +6,10 @@ tags: ir|dumper $end_info$ */ +#include "Interface/IR/IntrusiveIRList.h" +#include "Interface/IR/RegisterAllocationData.h" + #include -#include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/IREmitter.cpp b/FEXCore/Source/Interface/IR/IREmitter.cpp index d811ce4ac..0da4e8e41 100644 --- a/FEXCore/Source/Interface/IR/IREmitter.cpp +++ b/FEXCore/Source/Interface/IR/IREmitter.cpp @@ -9,7 +9,6 @@ $end_info$ #include "Interface/IR/IREmitter.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/IREmitter.h b/FEXCore/Source/Interface/IR/IREmitter.h index 35ab50221..ebb974671 100644 --- a/FEXCore/Source/Interface/IR/IREmitter.h +++ b/FEXCore/Source/Interface/IR/IREmitter.h @@ -1,7 +1,10 @@ // SPDX-License-Identifier: MIT #pragma once + +#include "Interface/IR/IR.h" +#include "Interface/IR/IntrusiveIRList.h" + #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/IRParser.cpp b/FEXCore/Source/Interface/IR/IRParser.cpp index 94c16b4c8..1c941de86 100644 --- a/FEXCore/Source/Interface/IR/IRParser.cpp +++ b/FEXCore/Source/Interface/IR/IRParser.cpp @@ -7,8 +7,8 @@ $end_info$ */ #include "Interface/IR/IREmitter.h" + #include -#include #include #include #include diff --git a/FEXCore/include/FEXCore/IR/IntrusiveIRList.h b/FEXCore/Source/Interface/IR/IntrusiveIRList.h similarity index 99% rename from FEXCore/include/FEXCore/IR/IntrusiveIRList.h rename to FEXCore/Source/Interface/IR/IntrusiveIRList.h index 3b626d337..c5f1e7e8e 100644 --- a/FEXCore/include/FEXCore/IR/IntrusiveIRList.h +++ b/FEXCore/Source/Interface/IR/IntrusiveIRList.h @@ -1,7 +1,8 @@ // SPDX-License-Identifier: MIT #pragma once -#include "FEXCore/IR/IR.h" +#include "Interface/IR/IR.h" + #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp b/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp index 192ae6520..affa4468b 100644 --- a/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp +++ b/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp @@ -17,7 +17,6 @@ $end_info$ #include "Interface/IR/PassManager.h" #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp b/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp index 28ee963c1..14a234a37 100644 --- a/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp @@ -9,7 +9,6 @@ $end_info$ #include "Interface/IR/PassManager.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp b/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp index ca95b8333..1fd7eca1b 100644 --- a/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp @@ -6,6 +6,7 @@ desc: Transforms ContextLoad/Store to temporaries, similar to mem2reg $end_info$ */ +#include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include "Interface/IR/Passes.h" #include "Interface/IR/PassManager.h" @@ -13,7 +14,6 @@ $end_info$ #include #include #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp b/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp index 9bd5c6232..4b3a2d46b 100644 --- a/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp @@ -12,7 +12,6 @@ $end_info$ #include #include #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp b/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp index 79ae90c7c..a146ed557 100644 --- a/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp +++ b/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp @@ -11,7 +11,6 @@ $end_info$ #include "Interface/Core/OpcodeDispatcher.h" #include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp b/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp index 655b0a96c..29b696ef8 100644 --- a/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp +++ b/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp @@ -9,12 +9,11 @@ $end_info$ #include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include "Interface/IR/PassManager.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Interface/IR/Passes/IRValidation.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp b/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp index 8aef6ccc0..8a0e7f4d0 100644 --- a/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp +++ b/FEXCore/Source/Interface/IR/Passes/InlineCallOptimization.cpp @@ -11,7 +11,6 @@ $end_info$ #include "Interface/IR/PassManager.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp b/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp index 00a3c446b..03d3cf14f 100644 --- a/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp +++ b/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp @@ -9,7 +9,6 @@ $end_info$ #include "Interface/IR/IREmitter.h" #include "Interface/IR/PassManager.h" #include -#include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp b/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp index f506d6c61..6f95ecbab 100644 --- a/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp +++ b/FEXCore/Source/Interface/IR/Passes/RAValidation.cpp @@ -3,12 +3,11 @@ #include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" #include "Interface/IR/PassManager.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Interface/IR/Passes/IRValidation.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp b/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp index 4a0fde231..c3489fd99 100644 --- a/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp +++ b/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp @@ -10,7 +10,6 @@ $end_info$ #include "Interface/IR/IREmitter.h" #include -#include #include #include "Interface/IR/PassManager.h" diff --git a/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp b/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp index 3c6e82641..d48aecda3 100644 --- a/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp +++ b/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp @@ -9,12 +9,12 @@ $end_info$ #include "Interface/IR/Passes/RegisterAllocationPass.h" #include "FEXCore/Core/X86Enums.h" +#include "Interface/IR/IR.h" #include "Interface/IR/IREmitter.h" +#include "Interface/IR/RegisterAllocationData.h" #include "Interface/IR/Passes.h" #include #include -#include -#include #include #include #include diff --git a/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp b/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp index 39f83e8fe..adf731a88 100644 --- a/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp +++ b/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp @@ -11,7 +11,6 @@ $end_info$ #include "Interface/IR/PassManager.h" #include -#include #include #include #include diff --git a/FEXCore/include/FEXCore/IR/RegisterAllocationData.h b/FEXCore/Source/Interface/IR/RegisterAllocationData.h similarity index 100% rename from FEXCore/include/FEXCore/IR/RegisterAllocationData.h rename to FEXCore/Source/Interface/IR/RegisterAllocationData.h diff --git a/FEXCore/include/FEXCore/Core/CoreState.h b/FEXCore/include/FEXCore/Core/CoreState.h index a11d677a4..47af9a1de 100644 --- a/FEXCore/include/FEXCore/Core/CoreState.h +++ b/FEXCore/include/FEXCore/Core/CoreState.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT #pragma once -#include "FEXCore/Core/X86Enums.h" -#include "FEXCore/IR/IR.h" +#include +#include #include #include #include diff --git a/FEXCore/include/FEXCore/Debug/InternalThreadState.h b/FEXCore/include/FEXCore/Debug/InternalThreadState.h index f5bdfe2a6..ac95a7a28 100644 --- a/FEXCore/include/FEXCore/Debug/InternalThreadState.h +++ b/FEXCore/include/FEXCore/Debug/InternalThreadState.h @@ -4,8 +4,6 @@ #include #include #include -#include -#include #include #include #include diff --git a/FEXCore/include/FEXCore/IR/IR.h b/FEXCore/include/FEXCore/IR/IR.h index 1ca505867..a09c26885 100644 --- a/FEXCore/include/FEXCore/IR/IR.h +++ b/FEXCore/include/FEXCore/IR/IR.h @@ -8,13 +8,8 @@ #include #include -#include -#include #include #include -#include -#include -#include #include @@ -24,477 +19,6 @@ class OrderedNode; class RegisterAllocationPass; class RegisterAllocationData; -/** - * @brief The IROp_Header is an dynamically sized array - * At the end it contains a uint8_t for the number of arguments that Op has - * Then there is an unsized array of NodeWrapper arguments for the number of arguments this op has - * The op structures that are including the header must ensure that they pad themselves correctly to the number of arguments used - */ -struct IROp_Header; - -/** - * @brief Represents the ID of a given IR node. - * - * Intended to provide strong typing from other integer values - * to prevent passing incorrect values to certain API functions. - */ -struct NodeID final { - using value_type = uint32_t; - - constexpr NodeID() noexcept = default; - constexpr explicit NodeID(value_type Value_) noexcept : Value{Value_} {} - - constexpr NodeID(const NodeID&) noexcept = default; - constexpr NodeID& operator=(const NodeID&) noexcept = default; - - constexpr NodeID(NodeID&&) noexcept = default; - constexpr NodeID& operator=(NodeID&&) noexcept = default; - - [[nodiscard]] constexpr bool IsValid() const noexcept { - return Value != 0; - } - [[nodiscard]] constexpr bool IsInvalid() const noexcept { - return !IsValid(); - } - constexpr void Invalidate() noexcept { - Value = 0; - } - - [[nodiscard]] friend constexpr bool operator==(NodeID, NodeID) noexcept = default; - - [[nodiscard]] friend constexpr bool operator<(NodeID lhs, NodeID rhs) noexcept { - return lhs.Value < rhs.Value; - } - [[nodiscard]] friend constexpr bool operator>(NodeID lhs, NodeID rhs) noexcept { - return operator<(rhs, lhs); - } - [[nodiscard]] friend constexpr bool operator<=(NodeID lhs, NodeID rhs) noexcept { - return !operator>(lhs, rhs); - } - [[nodiscard]] friend constexpr bool operator>=(NodeID lhs, NodeID rhs) noexcept { - return !operator<(lhs, rhs); - } - - friend std::ostream& operator<<(std::ostream& out, NodeID ID) { - out << ID.Value; - return out; - } - friend std::istream& operator>>(std::istream& in, NodeID& ID) { - in >> ID.Value; - return in; - } - - value_type Value{}; -}; - -/** - * @brief This is a very simple wrapper for our node pointers - * You probably don't want to use this directly - * Use OpNodeWrapper and OrderedNodeWrapper types below instead - * - * This is necessary to allow two things - * - Reduce memory usage by having the pointer be an 32bit offset rather than the whole 64bit pointer - * - Actually use an offset from a base so we aren't storing pointers for everything - * - Makes IR list copying be as cheap as a memcpy - * Downsides - * - The IR nodes have to be allocated out of a linear array of memory - * - We currently only allow a 32bit offset, so *only* 4 million nodes per list - * - We have to have the base offset live somewhere else - * - Has to be POD and trivially copyable - * - Makes every real node access turn in to a [Base + Offset] access - * - Can be confusing if you're mixing OpNodeWrapper and OrderedNodeWrapper usage - */ -template -struct NodeWrapperBase final { - // On x86-64 using a uint64_t type is more efficient since RIP addressing gives you [ + + ] - // On AArch64 using uint32_t is just more memory efficient. 32bit or 64bit offset doesn't matter - // We use uint32_t to be more memory efficient (Cuts our node list size in half) - using NodeOffsetType = uint32_t; - NodeOffsetType NodeOffset; - - explicit NodeWrapperBase() = default; - - [[nodiscard]] static NodeWrapperBase WrapOffset(NodeOffsetType Offset) { - NodeWrapperBase Wrapped; - Wrapped.NodeOffset = Offset; - return Wrapped; - } - - [[nodiscard]] static NodeWrapperBase WrapPtr(uintptr_t Base, uintptr_t Value) { - NodeWrapperBase Wrapped; - Wrapped.SetOffset(Base, Value); - return Wrapped; - } - - [[nodiscard]] static void *UnwrapNode(uintptr_t Base, NodeWrapperBase Node) { - return Node.GetNode(Base); - } - - [[nodiscard]] NodeID ID() const; - - [[nodiscard]] bool IsInvalid() const { return NodeOffset == 0; } - - [[nodiscard]] Type *GetNode(uintptr_t Base) { - return reinterpret_cast(Base + NodeOffset); - } - [[nodiscard]] const Type *GetNode(uintptr_t Base) const { - return reinterpret_cast(Base + NodeOffset); - } - - void SetOffset(uintptr_t Base, uintptr_t Value) { NodeOffset = Value - Base; } - - [[nodiscard]] friend constexpr bool operator==(const NodeWrapperBase&, const NodeWrapperBase&) = default; -}; - -static_assert(std::is_trivial_v>); - -static_assert(sizeof(NodeWrapperBase) == sizeof(uint32_t)); - -using OpNodeWrapper = NodeWrapperBase; -using OrderedNodeWrapper = NodeWrapperBase; - -struct OrderedNodeHeader { - OpNodeWrapper Value; - OrderedNodeWrapper Next; - OrderedNodeWrapper Previous; -}; - -static_assert(sizeof(OrderedNodeHeader) == sizeof(uint32_t) * 3); - -/** - * @brief This is a node in our IR representation - * Is a doubly linked list node that lives in a representation of a linearly allocated node list - * The links in the nodes can live in a list independent of the data IR data - * - * ex. - * Region1 : ... <-> <-> <-> ... - * | * | - * v v - * Region2 : ...... - * - * In this example the OrderedNodes are allocated in one linear memory region (Not necessarily contiguous with one another linking) - * The second region is contiguous but they don't have any relationship with one another directly - */ -class OrderedNode final { - friend class NodeWrapperIterator; - friend class OrderedList; - public: - // These three values are laid out very specifically to make it fast to access the NodeWrappers specifically - OrderedNodeHeader Header; - uint32_t NumUses; - - using value_type = OrderedNodeWrapper; - - OrderedNode() = default; - - /** - * @brief Appends a node to this current node - * - * Before. <-> <-> - * After. <-> <-> <-> Next - * - * @return Pointer to the node being added - */ - value_type append(uintptr_t Base, value_type Node) { - // Set Next Node's Previous to incoming node - SetPrevious(Base, Header.Next, Node); - - // Set Incoming node's links to this node's links - SetPrevious(Base, Node, Wrapped(Base)); - SetNext(Base, Node, Header.Next); - - // Set this node's next to the incoming node - SetNext(Base, Wrapped(Base), Node); - - // Return the node we are appending - return Node; - } - - OrderedNode *append(uintptr_t Base, OrderedNode *Node) { - value_type WNode = Node->Wrapped(Base); - // Set Next Node's Previous to incoming node - SetPrevious(Base, Header.Next, WNode); - - // Set Incoming node's links to this node's links - SetPrevious(Base, WNode, Wrapped(Base)); - SetNext(Base, WNode, Header.Next); - - // Set this node's next to the incoming node - SetNext(Base, Wrapped(Base), WNode); - - // Return the node we are appending - return Node; - } - - /** - * @brief Prepends a node to the current node - * Before. <-> <-> - * After. <-> <-> <-> Next - * - * @return Pointer to the node being added - */ - value_type prepend(uintptr_t Base, value_type Node) { - // Set the previous node's next to the incoming node - SetNext(Base, Header.Previous, Node); - - // Set the incoming node's links - SetPrevious(Base, Node, Header.Previous); - SetNext(Base, Node, Wrapped(Base)); - - // Set the current node's link - SetPrevious(Base, Wrapped(Base), Node); - - // Return the node we are prepending - return Node; - } - - OrderedNode *prepend(uintptr_t Base, OrderedNode *Node) { - value_type WNode = Node->Wrapped(Base); - // Set the previous node's next to the incoming node - SetNext(Base, Header.Previous, WNode); - - // Set the incoming node's links - SetPrevious(Base, WNode, Header.Previous); - SetNext(Base, WNode, Wrapped(Base)); - - // Set the current node's link - SetPrevious(Base, Wrapped(Base), WNode); - - // Return the node we are prepending - return Node; - } - - /** - * @brief Gets the remaining size of the blocks from this point onward - * - * Doesn't find the head of the list - * - */ - [[nodiscard]] size_t size(uintptr_t Base) const { - size_t Size = 1; - // Walk the list forward until we hit a sentinel - value_type Current = Header.Next; - while (Current.NodeOffset != 0) { - ++Size; - OrderedNode *RealNode = Current.GetNode(Base); - Current = RealNode->Header.Next; - } - return Size; - } - - void Unlink(uintptr_t Base) { - // This removes the node from the list. Orphaning it - // Before: <-> <-> - // After: - SetNext(Base, Header.Previous, Header.Next); - SetPrevious(Base, Header.Next, Header.Previous); - } - - [[nodiscard]] IROp_Header const* Op(uintptr_t Base) const { - return Header.Value.GetNode(Base); - } - [[nodiscard]] IROp_Header *Op(uintptr_t Base) { - return Header.Value.GetNode(Base); - } - - [[nodiscard]] uint32_t GetUses() const { return NumUses; } - - void AddUse() { ++NumUses; } - void RemoveUse() { --NumUses; } - - [[nodiscard]] value_type Wrapped(uintptr_t Base) const { - value_type Tmp; - Tmp.SetOffset(Base, reinterpret_cast(this)); - return Tmp; - } - - private: - [[nodiscard]] value_type WrappedOffset(uint32_t Offset) const { - value_type Tmp; - Tmp.NodeOffset = Offset; - return Tmp; - } - - static void SetPrevious(uintptr_t Base, value_type Node, value_type New) { - OrderedNode *RealNode = Node.GetNode(Base); - RealNode->Header.Previous = New; - } - - static void SetNext(uintptr_t Base, value_type Node, value_type New) { - OrderedNode *RealNode = Node.GetNode(Base); - RealNode->Header.Next = New; - } - - void SetUses(uint32_t Uses) { NumUses = Uses; } -}; - -static_assert(std::is_trivial_v); -static_assert(std::is_trivially_copyable_v); -static_assert(offsetof(OrderedNode, Header) == 0); -static_assert(sizeof(OrderedNode) == (sizeof(OrderedNodeHeader) + sizeof(uint32_t))); - -struct RegisterClassType final { - using value_type = uint32_t; - - value_type Val; - [[nodiscard]] constexpr operator value_type() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const RegisterClassType&, const RegisterClassType&) = default; -}; - -struct CondClassType final { - uint8_t Val; - [[nodiscard]] constexpr operator uint8_t() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const CondClassType&, const CondClassType&) = default; -}; - -struct MemOffsetType final { - uint8_t Val; - [[nodiscard]] constexpr operator uint8_t() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const MemOffsetType&, const MemOffsetType&) = default; -}; - -struct TypeDefinition final { - uint16_t Val; - - [[nodiscard]] constexpr operator uint16_t() const { - return Val; - } - - [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes) { - TypeDefinition Type{}; - Type.Val = Bytes << 8; - return Type; - } - - [[nodiscard]] static constexpr TypeDefinition Create(uint8_t Bytes, uint8_t Elements) { - TypeDefinition Type{}; - Type.Val = (Bytes << 8) | (Elements & 255); - return Type; - } - - [[nodiscard]] constexpr uint8_t Bytes() const { - return Val >> 8; - } - - [[nodiscard]] constexpr uint8_t Elements() const { - return Val & 255; - } - - [[nodiscard]] friend constexpr bool operator==(const TypeDefinition&, const TypeDefinition&) = default; -}; - -static_assert(std::is_trivial_v); - -struct FenceType final { - using value_type = uint8_t; - - value_type Val; - [[nodiscard]] constexpr operator value_type() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const FenceType&, const FenceType&) = default; -}; - -struct RoundType final { - uint8_t Val; - [[nodiscard]] constexpr operator uint8_t() const { - return Val; - } - [[nodiscard]] friend constexpr bool operator==(const RoundType&, const RoundType&) = default; -}; - -struct SHA256Sum final { - uint8_t data[32]; - [[nodiscard]] bool operator<(SHA256Sum const &rhs) const { - return memcmp(data, rhs.data, sizeof(data)) < 0; - } - - [[nodiscard]] bool operator==(SHA256Sum const &rhs) const { - return memcmp(data, rhs.data, sizeof(data)) == 0; - } -}; - -typedef void ThunkedFunction(void* ArgsRv); - -struct ThunkDefinition final { - SHA256Sum Sum; - ThunkedFunction *ThunkFunction; -}; - -class NodeIterator; - -/* This iterator can be used to step though nodes. - * Due to how our IR is laid out, this can be used to either step - * though the CodeBlocks or though the code within a single block. - */ -class NodeIterator { -public: - using value_type = std::tuple; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = value_type*; - using const_pointer = const value_type*; - using iterator = NodeIterator; - using const_iterator = const NodeIterator; - using reverse_iterator = iterator; - using const_reverse_iterator = const_iterator; - using iterator_category = std::bidirectional_iterator_tag; - - NodeIterator(uintptr_t Base, uintptr_t IRBase) : BaseList {Base}, IRList{ IRBase } {} - explicit NodeIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : BaseList {Base}, IRList{ IRBase }, Node {Ptr} {} - - [[nodiscard]] bool operator==(const NodeIterator &rhs) const { - return Node.NodeOffset == rhs.Node.NodeOffset; - } - - [[nodiscard]] bool operator!=(const NodeIterator &rhs) const { - return !operator==(rhs); - } - - NodeIterator operator++() { - OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); - Node = RealNode->Next; - return *this; - } - - NodeIterator operator--() { - OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); - Node = RealNode->Previous; - return *this; - } - - [[nodiscard]] value_type operator*() { - OrderedNode *RealNode = Node.GetNode(BaseList); - return { RealNode, RealNode->Op(IRList) }; - } - - [[nodiscard]] value_type operator()() { - OrderedNode *RealNode = Node.GetNode(BaseList); - return { RealNode, RealNode->Op(IRList) }; - } - - [[nodiscard]] NodeID ID() const { - return Node.ID(); - } - - [[nodiscard]] static NodeIterator Invalid() { - return NodeIterator(0, 0); - } - -protected: - uintptr_t BaseList{}; - uintptr_t IRList{}; - OrderedNodeWrapper Node{}; -}; - enum class SyscallFlags : uint8_t { DEFAULT = 0, // Syscalldoesn't care about CPUState being serialized up to the syscall instruction. @@ -555,168 +79,22 @@ enum IndexNamedVectorConstant : uint8_t { INDEXED_NAMED_VECTOR_MAX, }; -// This must directly match bytes to the named opsize. -// Implicit sized IR operations does math to get between sizes. -enum OpSize : uint8_t { - i8Bit = 1, - i16Bit = 2, - i32Bit = 4, - i64Bit = 8, - i128Bit = 16, - i256Bit = 32, -}; - -enum class FloatCompareOp : uint8_t { - EQ = 0, - LT, - LE, - UNO, - NEQ, - ORD, -}; - -enum class ShiftType : uint8_t { - LSL = 0, - LSR, - ASR, - ROR, -}; - -// Converts a size stored as an integer in to an OpSize enum. -// This is a nop operation and will be eliminated by the compiler. -static inline OpSize SizeToOpSize(uint8_t Size) { - switch (Size) { - case 1: return OpSize::i8Bit; - case 2: return OpSize::i16Bit; - case 4: return OpSize::i32Bit; - case 8: return OpSize::i64Bit; - case 16: return OpSize::i128Bit; - case 32: return OpSize::i256Bit; - default: FEX_UNREACHABLE; - } -} - -#define IROP_ENUM -#define IROP_STRUCTS -#define IROP_SIZES -#define IROP_REG_CLASSES -#include - -/* This iterator can be used to step though every single node in a multi-block in SSA order. - * - * Iterates in the order of: - * - * end <-- CodeBlockA <--> BlockAInst1 <--> BlockAInst2 <--> CodeBlockB <--> BlockBInst1 <--> BlockBInst2 --> end - */ -class AllNodesIterator : public NodeIterator { -public: - AllNodesIterator(uintptr_t Base, uintptr_t IRBase) : NodeIterator(Base, IRBase) {} - explicit AllNodesIterator(uintptr_t Base, uintptr_t IRBase, OrderedNodeWrapper Ptr) : NodeIterator(Base, IRBase, Ptr) {} - AllNodesIterator(NodeIterator other) : NodeIterator(other) {} // Allow NodeIterator to be upgraded - - AllNodesIterator operator++() { - OrderedNodeHeader *RealNode = reinterpret_cast(Node.GetNode(BaseList)); - auto IROp = Node.GetNode(BaseList)->Op(IRList); - - // If this is the last node of a codeblock, we need to continue to the next block - if (IROp->Op == OP_ENDBLOCK) { - auto EndBlock = IROp->C(); - - auto CurrentBlock = EndBlock->BlockHeader.GetNode(BaseList); - Node = CurrentBlock->Header.Next; - } else if (IROp->Op == OP_CODEBLOCK) { - auto CodeBlock = IROp->C(); - - Node = CodeBlock->Begin; - } else { - Node = RealNode->Next; - } - - return *this; +struct SHA256Sum final { + uint8_t data[32]; + [[nodiscard]] bool operator<(SHA256Sum const &rhs) const { + return memcmp(data, rhs.data, sizeof(data)) < 0; } - AllNodesIterator operator--() { - auto IROp = Node.GetNode(BaseList)->Op(IRList); - - if (IROp->Op == OP_BEGINBLOCK) { - auto BeginBlock = IROp->C(); - - Node = BeginBlock->BlockHeader; - } else if (IROp->Op == OP_CODEBLOCK) { - auto PrevBlockWrapper = Node.GetNode(BaseList)->Header.Previous; - auto PrevCodeBlock = PrevBlockWrapper.GetNode(BaseList)->Op(IRList)->C(); - - Node = PrevCodeBlock->Last; - } else { - Node = Node.GetNode(BaseList)->Header.Previous; - } - - return *this; - } - - [[nodiscard]] static AllNodesIterator Invalid() { - return AllNodesIterator(0, 0); + [[nodiscard]] bool operator==(SHA256Sum const &rhs) const { + return memcmp(data, rhs.data, sizeof(data)) == 0; } }; -class IRListView; -class IREmitter; +typedef void ThunkedFunction(void* ArgsRv); -template -inline NodeID NodeWrapperBase::ID() const { - return NodeID(NodeOffset / sizeof(IR::OrderedNode)); -} - -bool IsFragmentExit(FEXCore::IR::IROps Op); -bool IsBlockExit(FEXCore::IR::IROps Op); +struct ThunkDefinition final { + SHA256Sum Sum; + ThunkedFunction *ThunkFunction; +}; } // namespace FEXCore::IR - -template <> -struct std::hash { - size_t operator()(const FEXCore::IR::NodeID& ID) const noexcept { - return std::hash{}(ID.Value); - } -}; - -template <> -struct fmt::formatter : fmt::formatter { - using Base = fmt::formatter; - - // Pass-through the underlying value, so IDs can - // be formatted like any integral value. - template - auto format(const FEXCore::IR::NodeID& ID, FormatContext& ctx) const { - return Base::format(ID.Value, ctx); - } -}; - -template <> -struct fmt::formatter : fmt::formatter { - using Base = fmt::formatter; - - template - auto format(const FEXCore::IR::RegisterClassType& Class, FormatContext& ctx) const { - return Base::format(Class.Val, ctx); - } -}; - -template <> -struct fmt::formatter : fmt::formatter { - using Base = fmt::formatter; - - template - auto format(const FEXCore::IR::FenceType& Fence, FormatContext& ctx) const { - return Base::format(Fence.Val, ctx); - } -}; - -template <> -struct fmt::formatter : fmt::formatter> { - using Base = fmt::formatter>; - - template - auto format(const FEXCore::IR::OpSize& OpSize, FormatContext& ctx) const { - return Base::format(FEXCore::ToUnderlying(OpSize), ctx); - } -};