#pragma once #include "Interface/Core/Frontend.h" #include #include #include #include #include "LogManager.h" #include #include #include #include namespace FEXCore::IR { class Pass; class PassManager; class OpDispatchBuilder final { friend class FEXCore::IR::Pass; friend class FEXCore::IR::PassManager; public: bool ShouldDump {false}; struct JumpTargetInfo { OrderedNode* BlockEntry; bool HaveEmitted; }; std::map JumpTargets; OrderedNode* GetNewJumpBlock(uint64_t RIP) { auto it = JumpTargets.find(RIP); LogMan::Throw::A(it != JumpTargets.end(), "Couldn't find block generated for 0x%lx", RIP); return it->second.BlockEntry; } void SetNewBlockIfChanged(uint64_t RIP) { auto it = JumpTargets.find(RIP); if (it == JumpTargets.end()) return; it->second.HaveEmitted = true; if (CurrentCodeBlock->Wrapped(ListData.Begin()).ID() == it->second.BlockEntry->Wrapped(ListData.Begin()).ID()) return; // We have hit a RIP that is a jump target // Thus we need to end up in a new block SetCurrentCodeBlock(it->second.BlockEntry); } bool FinishOp(uint64_t NextRIP, bool LastOp) { // If we are switching to a new block and this current block has yet to set a RIP // Then we need to insert an unconditional jump from the current block to the one we are going to // This happens most frequently when an instruction jumps backwards to another location // eg: // // nop dword [rax], eax // .label: // rdi, 0x8 // cmp qword [rdi-8], 0 // jne .label if (!BlockSetRIP) { auto it = JumpTargets.find(NextRIP); if (it == JumpTargets.end() && LastOp) { // If we don't have a jump target to a new block then we have to leave // Set the RIP to the next instruction and leave _StoreContext(8, offsetof(FEXCore::Core::CPUState, rip), _Constant(NextRIP)); _ExitFunction(); } else if (it != JumpTargets.end()) { _Jump(it->second.BlockEntry); return true; } } BlockSetRIP = false; return false; } OpDispatchBuilder(); IRListView ViewIR() { return IRListView(&Data, &ListData); } IRListView *CreateIRCopy() { return new IRListView(&Data, &ListData); } void ResetWorkingList(); bool HadDecodeFailure() { return DecodeFailure; } void BeginFunction(uint64_t RIP, std::vector const *Blocks); void ExitFunction(); void Finalize(); // Dispatch builder functions #define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op void UnhandledOp(OpcodeArgs); void MOVOp(OpcodeArgs); void ALUOp(OpcodeArgs); void INTOp(OpcodeArgs); void SyscallOp(OpcodeArgs); void LEAOp(OpcodeArgs); void NOPOp(OpcodeArgs); void RETOp(OpcodeArgs); void SecondaryALUOp(OpcodeArgs); void ADCOp(OpcodeArgs); void SBBOp(OpcodeArgs); void PUSHOp(OpcodeArgs); void POPOp(OpcodeArgs); void LEAVEOp(OpcodeArgs); void CALLOp(OpcodeArgs); void CALLAbsoluteOp(OpcodeArgs); void CondJUMPOp(OpcodeArgs); void JUMPOp(OpcodeArgs); void JUMPAbsoluteOp(OpcodeArgs); void TESTOp(OpcodeArgs); void MOVSXDOp(OpcodeArgs); void MOVSXOp(OpcodeArgs); void MOVZXOp(OpcodeArgs); void CMPOp(OpcodeArgs); void SETccOp(OpcodeArgs); void CQOOp(OpcodeArgs); void CDQOp(OpcodeArgs); void XCHGOp(OpcodeArgs); void SAHFOp(OpcodeArgs); void LAHFOp(OpcodeArgs); void MOVSegOp(OpcodeArgs); void FLAGControlOp(OpcodeArgs); void MOVOffsetOp(OpcodeArgs); void CMOVOp(OpcodeArgs); void CPUIDOp(OpcodeArgs); template void SHLOp(OpcodeArgs); template void SHROp(OpcodeArgs); void ASHROp(OpcodeArgs); void ROROp(OpcodeArgs); void ROLOp(OpcodeArgs); void BTOp(OpcodeArgs); void IMUL1SrcOp(OpcodeArgs); void IMUL2SrcOp(OpcodeArgs); void IMULOp(OpcodeArgs); void STOSOp(OpcodeArgs); void MOVSOp(OpcodeArgs); void CMPSOp(OpcodeArgs); void BSWAPOp(OpcodeArgs); void RDTSCOp(OpcodeArgs); void INCOp(OpcodeArgs); void DECOp(OpcodeArgs); void NEGOp(OpcodeArgs); void DIVOp(OpcodeArgs); void IDIVOp(OpcodeArgs); void BSFOp(OpcodeArgs); void BSROp(OpcodeArgs); void CMPXCHGOp(OpcodeArgs); void MULOp(OpcodeArgs); void NOTOp(OpcodeArgs); // SSE void MOVUPSOp(OpcodeArgs); void MOVLHPSOp(OpcodeArgs); void MOVHPDOp(OpcodeArgs); void VectorALUOp(OpcodeArgs); void MOVQOp(OpcodeArgs); void PADDQOp(OpcodeArgs); void PSUBQOp(OpcodeArgs); template void PMINUOp(OpcodeArgs); void PMINSWOp(OpcodeArgs); void PMOVMSKBOp(OpcodeArgs); void PUNPCKLOp(OpcodeArgs); void PUNPCKHOp(OpcodeArgs); template void PSHUFDOp(OpcodeArgs); void PCMPEQOp(OpcodeArgs); template void PCMPGTOp(OpcodeArgs); void MOVDOp(OpcodeArgs); template void PSRLD(OpcodeArgs); template void PSLL(OpcodeArgs); void PSRLDQ(OpcodeArgs); void PSLLDQ(OpcodeArgs); void MOVDDUPOp(OpcodeArgs); template void SHUFOp(OpcodeArgs); void FXSaveOp(OpcodeArgs); void FXRStoreOp(OpcodeArgs); void PAlignrOp(OpcodeArgs); void UnimplementedOp(OpcodeArgs); #undef OpcodeArgs /** * @name IR allocation routines * * @{ */ // These handlers add cost to the constructor and destructor // If it becomes an issue then blow them away // GCC also generates some pretty atrocious code around these // Use Clang! #define IROP_ALLOCATE_HELPERS #define IROP_DISPATCH_HELPERS #include IRPair _Constant(uint8_t Size, uint64_t Constant) { auto Op = AllocateOp(); Op.first->Constant = Constant; Op.first->Header.Size = Size / 8; Op.first->Header.Elements = 1; Op.first->Header.NumArgs = 0; Op.first->Header.HasDest = true; return Op; } IRPair _Bfe(uint8_t Width, uint8_t lsb, OrderedNode *ssa0) { return _Bfe(ssa0, Width, lsb); } IRPair _Bfi(uint8_t Width, uint8_t lsb, OrderedNode *ssa0, OrderedNode *ssa1) { return _Bfi(ssa0, ssa1, Width, lsb); } IRPair _StoreMem(uint8_t Size, OrderedNode *ssa0, OrderedNode *ssa1, uint8_t Align = 1) { return _StoreMem(ssa0, ssa1, Size, Align); } IRPair _LoadMem(uint8_t Size, OrderedNode *ssa0, uint8_t Align = 1) { return _LoadMem(ssa0, Size, Align); } IRPair _StoreContext(uint8_t Size, uint32_t Offset, OrderedNode *ssa0) { return _StoreContext(ssa0, Size, Offset); } IRPair _Select(uint8_t Cond, OrderedNode *ssa0, OrderedNode *ssa1, OrderedNode *ssa2, OrderedNode *ssa3) { return _Select(ssa0, ssa1, ssa2, ssa3, {Cond}); } IRPair _Sext(uint8_t SrcSize, OrderedNode *ssa0) { return _Sext(ssa0, SrcSize); } IRPair _Zext(uint8_t SrcSize, OrderedNode *ssa0) { return _Zext(ssa0, SrcSize); } IRPair _VInsElement(uint8_t RegisterSize, uint8_t ElementSize, uint8_t DestIdx, uint8_t SrcIdx, OrderedNode *ssa0, OrderedNode *ssa1) { return _VInsElement(ssa0, ssa1, RegisterSize, ElementSize, DestIdx, SrcIdx); } IRPair _VAdd(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VAdd(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VSub(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VSub(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VUMin(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VUMin(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VSMin(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VSMin(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VZip(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VZip(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VZip2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VZip2(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VCMPEQ(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VCMPEQ(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VCMPGT(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VCMPGT(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VUShl(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VUShl(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VUShlS(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VUShlS(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VUShr(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VUShr(ssa0, ssa1, RegisterSize, ElementSize); } IRPair _VExtr(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1, uint8_t Index) { return _VExtr(ssa0, ssa1, RegisterSize, ElementSize, Index); } IRPair _VSLI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t ByteShift) { return _VSLI(ssa0, RegisterSize, ElementSize, ByteShift); } IRPair _VSRI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t ByteShift) { return _VSRI(ssa0, RegisterSize, ElementSize, ByteShift); } IRPair _Jump() { return _Jump(InvalidNode); } IRPair _CondJump(OrderedNode *ssa0) { return _CondJump(ssa0, InvalidNode, InvalidNode); } void SetJumpTarget(IR::IROp_Jump *Op, OrderedNode *Target) { LogMan::Throw::A(Target->Op(Data.Begin())->Op == OP_CODEBLOCK, "Tried setting Jump target to %%ssa%d %s", Target->Wrapped(ListData.Begin()).ID(), std::string(IR::GetName(Target->Op(Data.Begin())->Op)).c_str()); Op->Header.Args[0].NodeOffset = Target->Wrapped(ListData.Begin()).NodeOffset; } void SetTrueJumpTarget(IR::IROp_CondJump *Op, OrderedNode *Target) { LogMan::Throw::A(Target->Op(Data.Begin())->Op == OP_CODEBLOCK, "Tried setting CondJump target to %%ssa%d %s", Target->Wrapped(ListData.Begin()).ID(), std::string(IR::GetName(Target->Op(Data.Begin())->Op)).c_str()); Op->Header.Args[1].NodeOffset = Target->Wrapped(ListData.Begin()).NodeOffset; } void SetFalseJumpTarget(IR::IROp_CondJump *Op, OrderedNode *Target) { LogMan::Throw::A(Target->Op(Data.Begin())->Op == OP_CODEBLOCK, "Tried setting CondJump target to %%ssa%d %s", Target->Wrapped(ListData.Begin()).ID(), std::string(IR::GetName(Target->Op(Data.Begin())->Op)).c_str()); Op->Header.Args[2].NodeOffset = Target->Wrapped(ListData.Begin()).NodeOffset; } void SetJumpTarget(IRPair Op, OrderedNode *Target) { LogMan::Throw::A(Target->Op(Data.Begin())->Op == OP_CODEBLOCK, "Tried setting Jump target to %%ssa%d %s", Target->Wrapped(ListData.Begin()).ID(), std::string(IR::GetName(Target->Op(Data.Begin())->Op)).c_str()); Op.first->Header.Args[0].NodeOffset = Target->Wrapped(ListData.Begin()).NodeOffset; } void SetTrueJumpTarget(IRPair Op, OrderedNode *Target) { LogMan::Throw::A(Target->Op(Data.Begin())->Op == OP_CODEBLOCK, "Tried setting CondJump target to %%ssa%d %s", Target->Wrapped(ListData.Begin()).ID(), std::string(IR::GetName(Target->Op(Data.Begin())->Op)).c_str()); Op.first->Header.Args[1].NodeOffset = Target->Wrapped(ListData.Begin()).NodeOffset; } void SetFalseJumpTarget(IRPair Op, OrderedNode *Target) { LogMan::Throw::A(Target->Op(Data.Begin())->Op == OP_CODEBLOCK, "Tried setting CondJump target to %%ssa%d %s", Target->Wrapped(ListData.Begin()).ID(), std::string(IR::GetName(Target->Op(Data.Begin())->Op)).c_str()); Op.first->Header.Args[2].NodeOffset = Target->Wrapped(ListData.Begin()).NodeOffset; } /** @} */ bool IsValueConstant(OrderedNodeWrapper ssa, uint64_t *Constant) { OrderedNode *RealNode = ssa.GetNode(ListData.Begin()); FEXCore::IR::IROp_Header *IROp = RealNode->Op(Data.Begin()); if (IROp->Op == OP_CONSTANT) { auto Op = IROp->C(); *Constant = Op->Constant; return true; } return false; } // This is fairly special in how it operates // Since the node is decoupled from the backing op then we can swap out the backing op without much overhead // This can potentially cause problems where multiple nodes are pointing to the same IROp OrderedNode *ReplaceAllUsesWith(OrderedNode *Node, IROp_Header *Op) { RemoveArgUses(Node); Node->Header.Value.SetOffset(Data.Begin(), reinterpret_cast(Op)); return Node; } // This is similar to the previous op except that we pass in a node // This takes the op backing in the new node and replaces the node in the other node // Again can cause problems where things are pointing to NewNode and haven't been decoupled OrderedNode *ReplaceAllUsesWith(OrderedNode *Node, OrderedNode *NewNode) { RemoveArgUses(Node); Node->Header.Value.NodeOffset = NewNode->Header.Value.NodeOffset; return Node; } void ReplaceAllUsesWithInclusive(OrderedNode *Node, OrderedNode *NewNode, IR::NodeWrapperIterator After, IR::NodeWrapperIterator End); void Remove(OrderedNode *Node); void SetPackedRFLAG(bool Lower8, OrderedNode *Src); OrderedNode *GetPackedRFLAG(bool Lower8); void CopyData(OpDispatchBuilder const &rhs) { LogMan::Throw::A(rhs.Data.BackingSize() <= Data.BackingSize(), "Trying to take ownership of data that is too large"); LogMan::Throw::A(rhs.ListData.BackingSize() <= ListData.BackingSize(), "Trying to take ownership of data that is too large"); Data.CopyData(rhs.Data); ListData.CopyData(rhs.ListData); InvalidNode = rhs.InvalidNode; CurrentWriteCursor = rhs.CurrentWriteCursor; CodeBlocks = rhs.CodeBlocks; } void SetWriteCursor(OrderedNode *Node) { CurrentWriteCursor = Node; } OrderedNode *GetWriteCursor() { return CurrentWriteCursor; } /** * @brief This creates an orphaned code node * The IROp backing is in the correct list but the OrderedNode lives outside of the list * * XXX: This is because we don't want code blocks to interleave with current instruction IR ops currently * We can change this behaviour once we remove the old BeginBlock/EndBlock types * * @return OrderedNode */ IRPair CreateCodeNode() { auto CodeNode = _CodeBlock(InvalidNode, InvalidNode, InvalidNode); CodeBlocks.emplace_back(CodeNode); return CodeNode; } void SetCodeNodeBegin(OrderedNode *CodeNode, OrderedNode *Begin) { FEXCore::IR::IROp_CodeBlock *IROp = CodeNode->Op(Data.Begin())->CW(); LogMan::Throw::A(IROp->Header.Op == IROps::OP_CODEBLOCK, "Invalid"); IROp->Begin = Begin->Wrapped(ListData.Begin()); } void SetCodeNodeLast(OrderedNode *CodeNode, OrderedNode *Last) { FEXCore::IR::IROp_CodeBlock *IROp = CodeNode->Op(Data.Begin())->CW(); LogMan::Throw::A(IROp->Header.Op == IROps::OP_CODEBLOCK, "Invalid"); IROp->Last = Last->Wrapped(ListData.Begin()); } /** * @name Links codeblocks together * Codeblocks are singly linked so we need to walk the list forward if the linked block isn't isn't the last * * eq. * CodeNode->Next -> Next * to * CodeNode->Next -> New -> Next * * @{ */ /** @} */ void LinkCodeBlocks(OrderedNode *CodeNode, OrderedNode *Next) { FEXCore::IR::IROp_CodeBlock *CurrentIROp = CodeNode->Op(Data.Begin())->CW(); LogMan::Throw::A(CurrentIROp->Header.Op == IROps::OP_CODEBLOCK, "Invalid"); OrderedNodeWrapper OldNext = CurrentIROp->Next; // First thing is to assign CodeNode->Next to the incoming node { CurrentIROp->Next = Next->Wrapped(ListData.Begin()); } // Second thing is to assign the incoming node's Next to what was in CodeNode->Next { FEXCore::IR::IROp_CodeBlock *NextIROp = Next->Op(Data.Begin())->CW(); auto NewOldNext = NextIROp->Next; NextIROp->Next = OldNext; OldNext = NewOldNext; } } IRPair CreateNewCodeBlock(); void SetCurrentCodeBlock(OrderedNode *Node); void SetMultiblock(bool _Multiblock) { Multiblock = _Multiblock; } bool GetMultiblock() { return Multiblock; } private: void RemoveArgUses(OrderedNode *Node); bool DecodeFailure{false}; OrderedNode *LoadSource(FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false); OrderedNode *LoadSource_WithOpSize(FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false); void StoreResult_WithOpSize(FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, uint8_t OpSize, int8_t Align); void StoreResult(FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align); void StoreResult(FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align); uint8_t GetDstSize(FEXCore::X86Tables::DecodedOp Op); uint8_t GetSrcSize(FEXCore::X86Tables::DecodedOp Op); template void SetRFLAG(OrderedNode *Value); void SetRFLAG(OrderedNode *Value, unsigned BitOffset); OrderedNode *GetRFLAG(unsigned BitOffset); void GenerateFlags_ADC(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2, OrderedNode *CF); void GenerateFlags_SBB(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2, OrderedNode *CF); void GenerateFlags_SUB(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2); void GenerateFlags_ADD(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2); void GenerateFlags_MUL(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *High); void GenerateFlags_UMUL(FEXCore::X86Tables::DecodedOp Op, OrderedNode *High); void GenerateFlags_Logical(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2); void GenerateFlags_Shift(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2); void GenerateFlags_Rotate(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2); OrderedNode *CreateNode(IROp_Header *Op) { uintptr_t ListBegin = ListData.Begin(); size_t Size = sizeof(OrderedNode); void *Ptr = ListData.Allocate(Size); OrderedNode *Node = new (Ptr) OrderedNode(); Node->Header.Value.SetOffset(Data.Begin(), reinterpret_cast(Op)); if (CurrentWriteCursor) { CurrentWriteCursor->append(ListBegin, Node); } CurrentWriteCursor = Node; return Node; } OrderedNode *GetNode(uint32_t SSANode) { uintptr_t ListBegin = ListData.Begin(); OrderedNode *Node = reinterpret_cast(ListBegin + SSANode * sizeof(OrderedNode)); return Node; } OrderedNode *EmplaceOrphanedNode(OrderedNode *OldNode) { size_t Size = sizeof(OrderedNode); OrderedNode *Ptr = reinterpret_cast(ListData.Allocate(Size)); memcpy(Ptr, OldNode, Size); return Ptr; } void CreateJumpBlocks(std::vector const *Blocks); bool BlockSetRIP {false}; OrderedNode *CurrentWriteCursor = nullptr; // These could be combined with a little bit of work to be more efficient with memory usage. Isn't a big deal IntrusiveAllocator Data; IntrusiveAllocator ListData; OrderedNode *InvalidNode; OrderedNode *CurrentCodeBlock{}; std::vector CodeBlocks; bool Multiblock{}; uint64_t Entry; }; void InstallOpcodeHandlers(); }