From 513112919b56cb78336595de4efe934761cafb2d Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Thu, 28 Jan 2021 17:41:27 +0200 Subject: [PATCH 1/4] RA: Move allocation data to RAData --- External/FEXCore/Scripts/json_ir_generator.py | 2 +- .../FEXCore/Source/Interface/Core/Core.cpp | 15 ++++++---- .../Core/Interpreter/InterpreterClass.h | 2 +- .../Core/Interpreter/InterpreterCore.cpp | 2 +- .../Source/Interface/Core/JIT/Arm64/JIT.cpp | 28 ++++++++++--------- .../Interface/Core/JIT/Arm64/JITClass.h | 3 +- .../Source/Interface/Core/JIT/x86_64/JIT.cpp | 15 +++++----- .../Interface/Core/JIT/x86_64/JITClass.h | 3 +- .../FEXCore/Source/Interface/IR/IRDumper.cpp | 16 +++++------ External/FEXCore/Source/Interface/IR/Passes.h | 1 + .../Interface/IR/Passes/IRValidation.cpp | 10 +++---- .../IR/Passes/RegisterAllocationPass.cpp | 11 ++++++-- .../IR/Passes/RegisterAllocationPass.h | 13 +++++++-- .../FEXCore/include/FEXCore/Core/CPUBackend.h | 3 +- External/FEXCore/include/FEXCore/IR/IR.h | 3 +- Source/CommonCore/HostFactory.cpp | 4 +-- 16 files changed, 78 insertions(+), 53 deletions(-) diff --git a/External/FEXCore/Scripts/json_ir_generator.py b/External/FEXCore/Scripts/json_ir_generator.py index 6d725eb95..b5f75c899 100644 --- a/External/FEXCore/Scripts/json_ir_generator.py +++ b/External/FEXCore/Scripts/json_ir_generator.py @@ -236,7 +236,7 @@ def print_ir_arg_printer(ops, defines): if (SSAArgs != 0): for i in range(0, SSAArgs): LastArg = (SSAArgs - i - 1) == 0 and not HasArgs - output_file.write("\tPrintArg(out, IR, Op->Header.Args[%d], RAPass);\n" % i) + output_file.write("\tPrintArg(out, IR, Op->Header.Args[%d], RAData);\n" % i) if not (LastArg): output_file.write("\t*out << \", \";\n") diff --git a/External/FEXCore/Source/Interface/Core/Core.cpp b/External/FEXCore/Source/Interface/Core/Core.cpp index d40a9d2f9..82fe563ce 100644 --- a/External/FEXCore/Source/Interface/Core/Core.cpp +++ b/External/FEXCore/Source/Interface/Core/Core.cpp @@ -111,7 +111,7 @@ namespace DefaultFallbackCore { void Initialize() override {} bool NeedsOpDispatch() override { return false; } - void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData) override { + void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override { LogMan::Msg::E("Fell back to default code handler at RIP: 0x%lx", ThreadState->State.State.rip); return nullptr; } @@ -590,6 +590,7 @@ namespace FEXCore::Context { auto IR = Thread->IRLists.find(GuestRIP); FEXCore::IR::IRListView *IRList {}; FEXCore::Core::DebugData *DebugData {}; + FEXCore::IR::RegisterAllocationData *RAData {}; if (IR == Thread->IRLists.end()) { bool HadDispatchError {false}; @@ -703,7 +704,7 @@ namespace FEXCore::Context { - auto IRDumper = [Thread, GuestRIP](IR::RegisterAllocationPass* RA) { + auto IRDumper = [Thread, GuestRIP](IR::RegisterAllocationData* RA) { FILE* f = nullptr; bool CloseAfter = false; @@ -764,16 +765,18 @@ namespace FEXCore::Context { Thread->PassManager->Run(Thread->OpDispatcher.get()); if (Thread->CTX->Config.DumpIR != "no") { - IRDumper(Thread->PassManager->GetRAPass()); + IRDumper(Thread->PassManager->GetRAPass()->GetAllocationData()); } if (Thread->OpDispatcher->ShouldDump) { std::stringstream out; auto NewIR = Thread->OpDispatcher->ViewIR(); - FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass()); + FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass()->GetAllocationData()); printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str()); } + RAData = Thread->PassManager->GetRAPass()->GetAllocationData(); + // Create a copy of the IR and place it in this thread's IR cache auto AddedIR = Thread->IRLists.try_emplace(GuestRIP, Thread->OpDispatcher->CreateIRCopy()); Thread->OpDispatcher->ResetWorkingList(); @@ -795,7 +798,7 @@ namespace FEXCore::Context { } // Attempt to get the CPU backend to compile this code - return { Thread->CPUBackend->CompileCode(IRList, DebugData), DebugData }; + return { Thread->CPUBackend->CompileCode(IRList, DebugData, RAData), DebugData }; } uintptr_t Context::CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { @@ -861,7 +864,7 @@ namespace FEXCore::Context { // We have ONE more chance to try and fallback to the fallback CPU backend // This will most likely fail since regular code use won't be using a fallback core. // It's mainly for testing new instruction encodings - void *CodePtr = Thread->FallbackBackend->CompileCode(nullptr, nullptr); + void *CodePtr = Thread->FallbackBackend->CompileCode(nullptr, nullptr, nullptr); AddBlockMapping(Thread, GuestRIP, CodePtr); return (uintptr_t)CodePtr; } diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterClass.h b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterClass.h index e2ee39da1..ab4cbf92b 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterClass.h +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterClass.h @@ -22,7 +22,7 @@ public: explicit InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread); ~InterpreterCore() override; std::string GetName() override { return "Interpreter"; } - void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData) override; + void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override; void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; } diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp index 6b85db1ce..1f4a97956 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp @@ -476,7 +476,7 @@ Res InterpreterCore::GetSrc(void* SSAData, IR::OrderedNodeWrapper Src) { return reinterpret_cast(DstPtr); } -void *InterpreterCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData) { +void *InterpreterCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) { return reinterpret_cast(InterpreterExecution); } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index e07e88f89..9df7bf682 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -685,8 +685,8 @@ void JITCore::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant) { struct PhysReg { uint32_t Class; uint32_t VId; }; -static PhysReg GetPhys(IR::RegisterAllocationPass *RAPass, uint32_t Node) { - uint64_t Reg = RAPass->GetNodeRegister(Node); +static PhysReg GetPhys(IR::RegisterAllocationData *RAData, uint32_t Node) { + uint64_t Reg = RAData->GetNodeRegister(Node); auto rv = PhysReg {uint32_t(Reg>>32), (uint32_t)Reg}; if (rv.VId != ~0U) @@ -699,7 +699,7 @@ static PhysReg GetPhys(IR::RegisterAllocationPass *RAPass, uint32_t Node) { template<> aarch64::Register JITCore::GetReg(uint32_t Node) { -auto Reg = GetPhys(RAPass, Node); +auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::GPRFixedClass.Val) { return SRA64[Reg.VId].W(); } else if (Reg.Class == IR::GPRClass.Val) { @@ -711,7 +711,7 @@ auto Reg = GetPhys(RAPass, Node); template<> aarch64::Register JITCore::GetReg(uint32_t Node) { - auto Reg = GetPhys(RAPass, Node); + auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::GPRFixedClass.Val) { return SRA64[Reg.VId]; } else if (Reg.Class == IR::GPRClass.Val) { @@ -723,18 +723,18 @@ aarch64::Register JITCore::GetReg(uint32_t Node) { template<> std::pair JITCore::GetSrcPair(uint32_t Node) { - uint32_t Reg = GetPhys(RAPass, Node).VId; + uint32_t Reg = GetPhys(RAData, Node).VId; return RA32Pair[Reg]; } template<> std::pair JITCore::GetSrcPair(uint32_t Node) { - uint32_t Reg = GetPhys(RAPass, Node).VId; + uint32_t Reg = GetPhys(RAData, Node).VId; return RA64Pair[Reg]; } aarch64::VRegister JITCore::GetSrc(uint32_t Node) { - auto Reg = GetPhys(RAPass, Node); + auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::FPRFixedClass.Val) { return SRAFPR[Reg.VId]; } else if (Reg.Class == IR::FPRClass.Val) { @@ -745,7 +745,7 @@ aarch64::VRegister JITCore::GetSrc(uint32_t Node) { } aarch64::VRegister JITCore::GetDst(uint32_t Node) { - auto Reg = GetPhys(RAPass, Node); + auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::FPRFixedClass.Val) { return SRAFPR[Reg.VId]; } else if (Reg.Class == IR::FPRClass.Val) { @@ -770,7 +770,7 @@ bool JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Va } FEXCore::IR::RegisterClassType JITCore::GetRegClass(uint32_t Node) { - auto Class = static_cast(RAPass->GetNodeRegister(Node) >> 32); + auto Class = static_cast(RAData->GetNodeRegister(Node) >> 32); return FEXCore::IR::RegisterClassType {Class}; } @@ -787,11 +787,13 @@ bool JITCore::IsGPR(uint32_t Node) { return Class == IR::GPRClass || Class == IR::GPRFixedClass; } -void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData) { +void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) { using namespace aarch64; JumpTargets.clear(); uint32_t SSACount = IR->GetSSACount(); + this->RAData = RAData; + auto HeaderOp = IR->GetHeader(); #ifndef NDEBUG @@ -861,9 +863,9 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const LoadConstant(x0, ThreadSharedData.InterpreterFallbackHelperAddress); br(x0); } else { - LogMan::Throw::A(RAPass->HasFullRA(), "Arm64 JIT only works with RA"); + //LogMan::Throw::A(RAData->HasFullRA(), "Arm64 JIT only works with RA"); - SpillSlots = RAPass->SpillSlots(); + SpillSlots = RAData->SpillSlots(); if (SpillSlots) { sub(sp, sp, SpillSlots * 16); @@ -1097,7 +1099,7 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { Literal l_VirtualMemory {VirtualMemorySize}; Literal l_PagePtr {Thread->LookupCache->GetPagePointer()}; Literal l_CTX {reinterpret_cast(CTX)}; - Literal l_Interpreter {reinterpret_cast(State->IntBackend->CompileCode(nullptr, nullptr))}; + Literal l_Interpreter {reinterpret_cast(State->IntBackend->CompileCode(nullptr, nullptr, nullptr))}; Literal l_Sleep {reinterpret_cast(SleepThread)}; uintptr_t CompileBlockPtr{}; diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index 73591ad9d..211415c6c 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -78,7 +78,7 @@ public: ~JITCore() override; std::string GetName() override { return "JIT"; } - void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData) override; + void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override; void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; } @@ -232,6 +232,7 @@ private: CompilerSharedData ThreadSharedData; IR::RegisterAllocationPass *RAPass; + IR::RegisterAllocationData *RAData; uint32_t SpillSlots{}; diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp index 38787cf25..758ade1cb 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp @@ -461,7 +461,7 @@ void JITCore::ClearCache() { } uint32_t JITCore::GetPhys(uint32_t Node) { - uint64_t Reg = RAPass->GetNodeRegister(Node); + uint64_t Reg = RAData->GetNodeRegister(Node); if ((uint32_t)Reg != ~0U) return Reg; @@ -472,13 +472,13 @@ uint32_t JITCore::GetPhys(uint32_t Node) { } bool JITCore::IsFPR(uint32_t Node) { - auto Class = RAPass->GetNodeRegister(Node) >> 32; + auto Class = RAData->GetNodeRegister(Node) >> 32; return Class == IR::FPRClass.Val; } bool JITCore::IsGPR(uint32_t Node) { - auto Class = RAPass->GetNodeRegister(Node) >> 32; + auto Class = RAData->GetNodeRegister(Node) >> 32; return Class == IR::GPRClass.Val; } @@ -610,10 +610,11 @@ std::tuple JITCore::GetCC(IR::Con } } -void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData) { +void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) { JumpTargets.clear(); uint32_t SSACount = IR->GetSSACount(); + this->RAData = RAData; auto HeaderOp = IR->GetHeader(); // Fairly excessive buffer range to make sure we don't overflow @@ -650,9 +651,9 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const mov(rax, (uintptr_t)ThreadSharedData.InterpreterFallbackHelperAddress); jmp(rax); } else { - LogMan::Throw::A(RAPass->HasFullRA(), "Needs RA"); + LogMan::Throw::A(RAData != nullptr, "Needs RA"); - SpillSlots = RAPass->SpillSlots(); + SpillSlots = RAData->SpillSlots(); if (SpillSlots) { sub(rsp, SpillSlots * 16); @@ -1001,7 +1002,7 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { ThreadSharedData.InterpreterFallbackHelperAddress = getCurr(); // This will get called so our stack is now misaligned mov(rdi, STATE); - mov(rax, reinterpret_cast(ThreadState->IntBackend->CompileCode(nullptr, nullptr))); + mov(rax, reinterpret_cast(ThreadState->IntBackend->CompileCode(nullptr, nullptr, nullptr))); call(rax); diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h index 57aa826bc..ec83fa95e 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h @@ -58,7 +58,7 @@ public: explicit JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, CodeBuffer Buffer, bool CompileThread); ~JITCore() override; std::string GetName() override { return "JIT"; } - void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData) override; + void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override; void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; } @@ -128,6 +128,7 @@ private: void CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread); IR::RegisterAllocationPass *RAPass; + FEXCore::IR::RegisterAllocationData *RAData; #ifdef BLOCKSTATS bool GetSamplingData {true}; diff --git a/External/FEXCore/Source/Interface/IR/IRDumper.cpp b/External/FEXCore/Source/Interface/IR/IRDumper.cpp index 956e5a034..9ae363dd1 100644 --- a/External/FEXCore/Source/Interface/IR/IRDumper.cpp +++ b/External/FEXCore/Source/Interface/IR/IRDumper.cpp @@ -74,15 +74,15 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView *out << "Unknown Registerclass " << Arg; } -static void PrintArg(std::stringstream *out, IRListView const* IR, OrderedNodeWrapper Arg, IR::RegisterAllocationPass *RAPass) { +static void PrintArg(std::stringstream *out, IRListView const* IR, OrderedNodeWrapper Arg, IR::RegisterAllocationData *RAData) { auto [CodeNode, IROp] = IR->at(Arg)(); if (Arg.ID() == 0) { *out << "%Invalid"; } else { *out << "%ssa" << std::to_string(Arg.ID()); - if (RAPass) { - uint64_t RegClass = RAPass->GetNodeRegister(Arg.ID()); + if (RAData) { + uint64_t RegClass = RAData->GetNodeRegister(Arg.ID()); FEXCore::IR::RegisterClassType Class {uint32_t(RegClass >> 32)}; uint32_t Reg = RegClass; switch (Class) { @@ -147,7 +147,7 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView } } -void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationPass *RAPass) { +void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData) { auto HeaderOp = IR->GetHeader(); int8_t CurrentIndent = 0; @@ -204,8 +204,8 @@ void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAlloc *out << "%ssa" << std::to_string(ID); - if (RAPass) { - uint64_t RegClass = RAPass->GetNodeRegister(ID); + if (RAData) { + uint64_t RegClass = RAData->GetNodeRegister(ID); FEXCore::IR::RegisterClassType Class {uint32_t(RegClass >> 32)}; uint32_t Reg = RegClass; switch (Class) { @@ -264,9 +264,9 @@ void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAlloc auto [NodeNode, IROp] = NodeBegin(); auto PhiOp = IROp->C(); *out << "[ "; - PrintArg(out, IR, PhiOp->Value, RAPass); + PrintArg(out, IR, PhiOp->Value, RAData); *out << ", "; - PrintArg(out, IR, PhiOp->Block, RAPass); + PrintArg(out, IR, PhiOp->Block, RAData); *out << " ]"; if (PhiOp->Next.ID()) diff --git a/External/FEXCore/Source/Interface/IR/Passes.h b/External/FEXCore/Source/Interface/IR/Passes.h index 723bf2007..a5ab2a63d 100644 --- a/External/FEXCore/Source/Interface/IR/Passes.h +++ b/External/FEXCore/Source/Interface/IR/Passes.h @@ -3,6 +3,7 @@ namespace FEXCore::IR { class Pass; class RegisterAllocationPass; +class RegisterAllocationData; FEXCore::IR::Pass* CreateConstProp(bool InlineConstants); FEXCore::IR::Pass* CreateContextLoadStoreElimination(); diff --git a/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp b/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp index b21310e06..d964ed5a8 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp +++ b/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp @@ -50,9 +50,9 @@ bool IRValidation::Run(IREmitter *IREmit) { auto HeaderOp = CurrentIR.GetHeader(); LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader"); - IR::RegisterAllocationPass * RAPass{}; + IR::RegisterAllocationData * RAData{}; if (Manager->HasRAPass() && !HeaderOp->ShouldInterpret) { - RAPass = Manager->GetRAPass(); + RAData = Manager->GetRAPass()->GetAllocationData(); } for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) { @@ -81,9 +81,9 @@ bool IRValidation::Run(IREmitter *IREmit) { Warnings << "%ssa" << ID << ": Destination created but had no uses" << std::endl; } - if (RAPass) { + if (RAData) { // If we have a register allocator then the destination needs to be assigned a register and class - uint64_t Reg = RAPass->GetNodeRegister(ID); + uint64_t Reg = RAData->GetNodeRegister(ID); FEXCore::IR::RegisterClassType ExpectedClass = IR::GetRegClass(IROp->Op); FEXCore::IR::RegisterClassType AssignedClass = FEXCore::IR::RegisterClassType{uint32_t(Reg >> 32)}; @@ -260,7 +260,7 @@ bool IRValidation::Run(IREmitter *IREmit) { HadWarning = false; if (HadError || HadWarning) { - FEXCore::IR::Dump(&Out, &CurrentIR, RAPass); + FEXCore::IR::Dump(&Out, &CurrentIR, RAData); if (HadError) { Out << "Errors:" << std::endl << Errors.str() << std::endl; diff --git a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp index df2783981..1c83f8510 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp +++ b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp @@ -193,6 +193,7 @@ namespace { }; struct RegisterGraph { + FEXCore::IR::RegisterAllocationData RegAllocMap; RegisterSet Set; std::vector Nodes; uint32_t NodeCount; @@ -390,7 +391,7 @@ namespace FEXCore::IR { * @brief Returns the register and class encoded together * Top 32bits is the class, lower 32bits is the register */ - uint64_t GetNodeRegister(uint32_t Node) override; + RegisterAllocationData* GetAllocationData() override; private: bool OptimizeSRA; uint32_t SpillPointId; @@ -466,8 +467,12 @@ namespace FEXCore::IR { VirtualAddRegisterConflict(Graph, ClassConflict, RegConflict, Class, Reg); } - uint64_t ConstrainedRAPass::GetNodeRegister(uint32_t Node) { - return Graph->Nodes[Node].Head.RegAndClass; + RegisterAllocationData* ConstrainedRAPass::GetAllocationData() { + Graph->RegAllocMap.RegAndClass.resize(Graph->Nodes.size()); + for (size_t i = 0; i < Graph->Nodes.size(); i++) { + Graph->RegAllocMap.RegAndClass[i] = Graph->Nodes[i].Head.RegAndClass; + } + return &Graph->RegAllocMap; } void ConstrainedRAPass::RecursiveLiveRangeExpansion(FEXCore::IR::IRListView *IR, uint32_t Node, uint32_t DefiningBlockID, LiveRange *LiveRange, const std::unordered_set &Predecessors, std::unordered_set &VisitedPredecessors) { diff --git a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h index f8a2ad74f..3ebabf5c7 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h +++ b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h @@ -6,10 +6,19 @@ namespace FEXCore::IR { template class IRListView; +class RegisterAllocationData { + public: + std::vector RegAndClass; + uint32_t SpillSlotCount {}; + uint64_t GetNodeRegister(uint32_t Node) { + return RegAndClass[Node]; + } + uint32_t SpillSlots() const { return SpillSlotCount; } +}; + class RegisterAllocationPass : public FEXCore::IR::Pass { public: bool HasFullRA() const { return HadFullRA; } - uint32_t SpillSlots() const { return SpillSlotCount; } virtual void AllocateRegisterSet(uint32_t RegisterCount, uint32_t ClassCount) = 0; virtual void AddRegisters(FEXCore::IR::RegisterClassType Class, uint32_t RegisterCount) = 0; @@ -36,7 +45,7 @@ class RegisterAllocationPass : public FEXCore::IR::Pass { * @brief Returns the register and class encoded together * Top 32bits is the class, lower 32bits is the register */ - virtual uint64_t GetNodeRegister(uint32_t Node) = 0; + virtual RegisterAllocationData* GetAllocationData() = 0; /** @} */ protected: diff --git a/External/FEXCore/include/FEXCore/Core/CPUBackend.h b/External/FEXCore/include/FEXCore/Core/CPUBackend.h index 26373eccf..4b488008c 100644 --- a/External/FEXCore/include/FEXCore/Core/CPUBackend.h +++ b/External/FEXCore/include/FEXCore/Core/CPUBackend.h @@ -7,6 +7,7 @@ namespace FEXCore { namespace IR { template class IRListView; + class RegisterAllocationData; } namespace Core { @@ -44,7 +45,7 @@ class LLVMCore; * @return An executable function pointer that is theoretically compiled from this point. * Is actually a function pointer of type `void (FEXCore::Core::ThreadState *Thread) */ - virtual void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData) = 0; + virtual void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) = 0; /** * @brief Function for mapping memory in to the CPUBackend's visible space. Allows setting up virtual mappings if required diff --git a/External/FEXCore/include/FEXCore/IR/IR.h b/External/FEXCore/include/FEXCore/IR/IR.h index 1f927b804..2543b7be6 100644 --- a/External/FEXCore/include/FEXCore/IR/IR.h +++ b/External/FEXCore/include/FEXCore/IR/IR.h @@ -7,6 +7,7 @@ namespace FEXCore::IR { class RegisterAllocationPass; +class RegisterAllocationData; /** * @brief The IROp_Header is an dynamically sized array @@ -459,7 +460,7 @@ template class IRListView; class IREmitter; -void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationPass *RAPass); +void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData); IREmitter* Parse(std::istream *in); template diff --git a/Source/CommonCore/HostFactory.cpp b/Source/CommonCore/HostFactory.cpp index 6c516d319..a6c9a02b1 100644 --- a/Source/CommonCore/HostFactory.cpp +++ b/Source/CommonCore/HostFactory.cpp @@ -30,7 +30,7 @@ namespace HostFactory { explicit HostCore(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread, bool Fallback); ~HostCore() override; std::string GetName() override { return "Host Core"; } - void* CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData) override; + void* CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override; void *MapRegion(void *HostPtr, uint64_t VirtualGuestPtr, uint64_t Size) override { return HostPtr; @@ -177,7 +177,7 @@ namespace HostFactory { ready(); } - void* HostCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData) { + void* HostCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) { return nullptr; } From a45c2c34e91b6dc5e7b10cbf14b5dd8c8b3d6342 Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Thu, 28 Jan 2021 21:42:39 +0200 Subject: [PATCH 2/4] RA: Use RAData directly --- .../FEXCore/Source/Interface/Core/Core.cpp | 6 +-- .../Interface/IR/Passes/IRValidation.cpp | 2 +- .../IR/Passes/RegisterAllocationPass.cpp | 48 ++++++++++--------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/Core.cpp b/External/FEXCore/Source/Interface/Core/Core.cpp index 82fe563ce..2f1434648 100644 --- a/External/FEXCore/Source/Interface/Core/Core.cpp +++ b/External/FEXCore/Source/Interface/Core/Core.cpp @@ -765,17 +765,17 @@ namespace FEXCore::Context { Thread->PassManager->Run(Thread->OpDispatcher.get()); if (Thread->CTX->Config.DumpIR != "no") { - IRDumper(Thread->PassManager->GetRAPass()->GetAllocationData()); + IRDumper(Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr); } if (Thread->OpDispatcher->ShouldDump) { std::stringstream out; auto NewIR = Thread->OpDispatcher->ViewIR(); - FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass()->GetAllocationData()); + FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr); printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str()); } - RAData = Thread->PassManager->GetRAPass()->GetAllocationData(); + RAData = Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr; // Create a copy of the IR and place it in this thread's IR cache auto AddedIR = Thread->IRLists.try_emplace(GuestRIP, Thread->OpDispatcher->CreateIRCopy()); diff --git a/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp b/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp index d964ed5a8..757465f49 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp +++ b/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp @@ -52,7 +52,7 @@ bool IRValidation::Run(IREmitter *IREmit) { IR::RegisterAllocationData * RAData{}; if (Manager->HasRAPass() && !HeaderOp->ShouldInterpret) { - RAData = Manager->GetRAPass()->GetAllocationData(); + RAData = Manager->GetRAPass() ? Manager->GetRAPass()->GetAllocationData() : nullptr; } for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) { diff --git a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp index 1c83f8510..9fbd4dcd1 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp +++ b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp @@ -11,7 +11,7 @@ namespace { constexpr uint32_t INVALID_REG = 31; constexpr uint32_t INVALID_CLASS = 7; - constexpr uint32_t DEFAULT_INTERFERENCE_LIST_COUNT = 120; + constexpr uint32_t DEFAULT_INTERFERENCE_LIST_COUNT = 122; constexpr uint32_t DEFAULT_INTERFERENCE_SPAN_COUNT = 30; constexpr uint32_t DEFAULT_NODE_COUNT = 8192; @@ -158,11 +158,10 @@ namespace { struct RegisterNode { struct VolatileHeader { - uint64_t RegAndClass; uint32_t BlockID; uint32_t SpillSlot; RegisterNode *PhiPartner; - } Head { INVALID_REGCLASS, ~0U, ~0U, nullptr }; + } Head { ~0U, ~0U, nullptr }; BucketList Interferences; }; @@ -273,10 +272,11 @@ namespace { Graph->Nodes.resize(NodeCount); Graph->NodeCount = NodeCount; Graph->VisitedNodePredecessors.clear(); + Graph->RegAllocMap.RegAndClass.resize(NodeCount); } void SetNodeClass(RegisterGraph *Graph, uint32_t Node, FEXCore::IR::RegisterClassType Class) { - Graph->Nodes[Node].Head.RegAndClass = ((uint64_t)Class << 32) | (Graph->Nodes[Node].Head.RegAndClass & ~0U); + Graph->RegAllocMap.RegAndClass[Node] = ((uint64_t)Class << 32) | (Graph->RegAllocMap.RegAndClass[Node] & ~0U); } void SetNodePartner(RegisterGraph *Graph, uint32_t Node, uint32_t Partner) { @@ -284,8 +284,8 @@ namespace { } - bool DoesNodeConflictWithRegAndClass(RegisterGraph *Graph, RegisterNode const *InterferenceNode, uint64_t RegAndClass) { - return IsConflict(Graph, InterferenceNode->Head.RegAndClass, RegAndClass); + bool DoesNodeConflictWithRegAndClass(RegisterGraph *Graph, const uint64_t InterferenceRegAndClass, uint64_t RegAndClass) { + return IsConflict(Graph, InterferenceRegAndClass, RegAndClass); } /** @@ -294,8 +294,8 @@ namespace { bool DoesNodeInterfereWithRegister(RegisterGraph *Graph, RegisterNode const *Node, uint64_t RegAndClass) { // Walk the node's interference list and see if it interferes with this register return Node->Interferences.Find([Graph, RegAndClass](uint32_t InterferenceNodeId) { - RegisterNode *InterferenceNode = &Graph->Nodes[InterferenceNodeId]; - return DoesNodeConflictWithRegAndClass(Graph, InterferenceNode, RegAndClass); + auto InterferenceRegAndClass = Graph->RegAllocMap.RegAndClass[InterferenceNodeId]; + return DoesNodeConflictWithRegAndClass(Graph, InterferenceRegAndClass, RegAndClass); }); } @@ -468,10 +468,10 @@ namespace FEXCore::IR { } RegisterAllocationData* ConstrainedRAPass::GetAllocationData() { - Graph->RegAllocMap.RegAndClass.resize(Graph->Nodes.size()); - for (size_t i = 0; i < Graph->Nodes.size(); i++) { - Graph->RegAllocMap.RegAndClass[i] = Graph->Nodes[i].Head.RegAndClass; - } + // Graph->RegAllocMap.RegAndClass.resize(Graph->Nodes.size()); + // for (size_t i = 0; i < Graph->Nodes.size(); i++) { + // Graph->RegAllocMap.RegAndClass[i] = Graph->Nodes[i].Head.RegAndClass; + // } return &Graph->RegAllocMap; } @@ -910,7 +910,7 @@ namespace FEXCore::IR { if (LiveRanges[i].Begin != ~0U) { LogMan::Throw::A(LiveRanges[i].Begin < LiveRanges[i].End , "Span must Begin before Ending"); - auto Class = GetClass(Graph->Nodes[i].Head.RegAndClass); + auto Class = GetClass(Graph->RegAllocMap.RegAndClass[i]); SpanStart[LiveRanges[i].Begin].Append(INFO_MAKE(i, Class)); SpanEnd[LiveRanges[i].End] .Append(INFO_MAKE(i, Class)); } @@ -946,12 +946,13 @@ namespace FEXCore::IR { void ConstrainedRAPass::AllocateVirtualRegisters() { for (uint32_t i = 0; i < Graph->NodeCount; ++i) { RegisterNode *CurrentNode = &Graph->Nodes[i]; - if (CurrentNode->Head.RegAndClass == INVALID_REGCLASS) + auto &CurrentRegAndClass = Graph->RegAllocMap.RegAndClass[i]; + if (CurrentRegAndClass == INVALID_REGCLASS) continue; auto LiveRange = &LiveRanges[i]; - FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentNode->Head.RegAndClass >> 32)}; + FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentRegAndClass >> 32)}; uint64_t RegAndClass = INVALID_REGCLASS; RegisterClass *RAClass = &Graph->Set.Classes[RegClass]; @@ -996,7 +997,7 @@ namespace FEXCore::IR { } else { uint32_t RegisterConflicts = 0; CurrentNode->Interferences.Iterate([&](const uint32_t InterferenceNode) { - RegisterConflicts |= GetConflicts(Graph, Graph->Nodes[InterferenceNode].Head.RegAndClass, RegClass); + RegisterConflicts |= GetConflicts(Graph, Graph->RegAllocMap.RegAndClass[InterferenceNode], RegClass); }); RegisterConflicts = (~RegisterConflicts) & RAClass->CountMask; @@ -1014,12 +1015,12 @@ namespace FEXCore::IR { HadFullRA = false; SpillPointId = i; - CurrentNode->Head.RegAndClass = RegAndClass; + CurrentRegAndClass = RegAndClass; // Must spill and restart return; } - CurrentNode->Head.RegAndClass = RegAndClass; + CurrentRegAndClass = RegAndClass; } } } @@ -1313,7 +1314,7 @@ namespace FEXCore::IR { if (InterferenceIdToSpill == 0) { int j = 0; LogMan::Msg::D("node %%ssa%d, was dumped in to virtual reg %d. Live Range[%d, %d)", - CurrentLocation, RegisterNode->Head.RegAndClass, + CurrentLocation, -1, OpLiveRange->Begin, OpLiveRange->End); RegisterNode->Interferences.Iterate([&](uint32_t InterferenceNode) { @@ -1361,12 +1362,13 @@ namespace FEXCore::IR { uint32_t Node = IR.GetID(CodeNode); RegisterNode *CurrentNode = &Graph->Nodes[Node]; + auto &CurrentRegAndClass = Graph->RegAllocMap.RegAndClass[Node]; LiveRange *OpLiveRange = &LiveRanges[Node]; // If this node is allocated above the number of physical registers we have then we need to search the interference list and spill the one // that is cheapest - FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentNode->Head.RegAndClass >> 32)}; - bool NeedsToSpill = (uint32_t)CurrentNode->Head.RegAndClass == INVALID_REG; + FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentRegAndClass >> 32)}; + bool NeedsToSpill = (uint32_t)CurrentRegAndClass == INVALID_REG; if (NeedsToSpill) { bool Spilled = false; @@ -1396,11 +1398,11 @@ namespace FEXCore::IR { if (!Spilled) { uint32_t InterferenceNode = FindNodeToSpill(IREmit, CurrentNode, Node, OpLiveRange); if (InterferenceNode != ~0U) { - FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{uint32_t(Graph->Nodes[InterferenceNode].Head.RegAndClass >> 32)}; + FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{uint32_t(Graph->RegAllocMap.RegAndClass[InterferenceNode] >> 32)}; uint32_t SpillSlot = FindSpillSlot(InterferenceNode, InterferenceRegClass); RegisterNode *InterferenceRegisterNode = &Graph->Nodes[InterferenceNode]; LogMan::Throw::A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!"); - LogMan::Throw::A((InterferenceRegisterNode->Head.RegAndClass & ~0U) != ~0U, "Interference node never assigned a register?"); + //LogMan::Throw::A((InterferenceRegisterNode->Head.RegAndClass & ~0U) != ~0U, "Interference node never assigned a register?"); LogMan::Throw::A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?"); LogMan::Throw::A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently"); LogMan::Throw::A(InterferenceRegClass == RegClass, "Class doesn't match"); From 0e00d3dc15d4038979ca85b92e073c8c0c68dd5a Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Thu, 28 Jan 2021 23:58:30 +0200 Subject: [PATCH 3/4] RA: Optimize RAData --- .../Source/Interface/Core/JIT/Arm64/JIT.cpp | 37 ++--- .../Source/Interface/Core/JIT/x86_64/JIT.cpp | 57 +++---- .../Interface/Core/JIT/x86_64/JITClass.h | 3 +- External/FEXCore/Source/Interface/IR/IR.json | 2 + .../FEXCore/Source/Interface/IR/IRDumper.cpp | 21 +-- .../Interface/IR/Passes/IRValidation.cpp | 6 +- .../IR/Passes/RegisterAllocationPass.cpp | 154 ++++++++---------- .../IR/Passes/RegisterAllocationPass.h | 52 +++++- 8 files changed, 172 insertions(+), 160 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index 9df7bf682..ad422b2ea 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -683,27 +683,21 @@ void JITCore::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant) { } } -struct PhysReg { uint32_t Class; uint32_t VId; }; +static IR::PhysicalRegister GetPhys(IR::RegisterAllocationData *RAData, uint32_t Node) { + auto PhyReg = RAData->GetNodeRegister(Node); -static PhysReg GetPhys(IR::RegisterAllocationData *RAData, uint32_t Node) { - uint64_t Reg = RAData->GetNodeRegister(Node); - auto rv = PhysReg {uint32_t(Reg>>32), (uint32_t)Reg}; + LogMan::Throw::A(!PhyReg.IsInvalid(), "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class); - if (rv.VId != ~0U) - return rv; - else - LogMan::Msg::A("Couldn't Allocate register for node: ssa%d. Class: %d", Node, Reg >> 32); - - return PhysReg { ~0U, ~0U}; + return PhyReg; } template<> aarch64::Register JITCore::GetReg(uint32_t Node) { auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::GPRFixedClass.Val) { - return SRA64[Reg.VId].W(); + return SRA64[Reg.Reg].W(); } else if (Reg.Class == IR::GPRClass.Val) { - return RA64[Reg.VId].W(); + return RA64[Reg.Reg].W(); } else { LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class); } @@ -713,9 +707,9 @@ template<> aarch64::Register JITCore::GetReg(uint32_t Node) { auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::GPRFixedClass.Val) { - return SRA64[Reg.VId]; + return SRA64[Reg.Reg]; } else if (Reg.Class == IR::GPRClass.Val) { - return RA64[Reg.VId]; + return RA64[Reg.Reg]; } else { LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class); } @@ -723,22 +717,22 @@ aarch64::Register JITCore::GetReg(uint32_t Node) { template<> std::pair JITCore::GetSrcPair(uint32_t Node) { - uint32_t Reg = GetPhys(RAData, Node).VId; + uint32_t Reg = GetPhys(RAData, Node).Reg; return RA32Pair[Reg]; } template<> std::pair JITCore::GetSrcPair(uint32_t Node) { - uint32_t Reg = GetPhys(RAData, Node).VId; + uint32_t Reg = GetPhys(RAData, Node).Reg; return RA64Pair[Reg]; } aarch64::VRegister JITCore::GetSrc(uint32_t Node) { auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::FPRFixedClass.Val) { - return SRAFPR[Reg.VId]; + return SRAFPR[Reg.Reg]; } else if (Reg.Class == IR::FPRClass.Val) { - return RAFPR[Reg.VId]; + return RAFPR[Reg.Reg]; } else { LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class); } @@ -747,9 +741,9 @@ aarch64::VRegister JITCore::GetSrc(uint32_t Node) { aarch64::VRegister JITCore::GetDst(uint32_t Node) { auto Reg = GetPhys(RAData, Node); if (Reg.Class == IR::FPRFixedClass.Val) { - return SRAFPR[Reg.VId]; + return SRAFPR[Reg.Reg]; } else if (Reg.Class == IR::FPRClass.Val) { - return RAFPR[Reg.VId]; + return RAFPR[Reg.Reg]; } else { LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class); } @@ -770,8 +764,7 @@ bool JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Va } FEXCore::IR::RegisterClassType JITCore::GetRegClass(uint32_t Node) { - auto Class = static_cast(RAData->GetNodeRegister(Node) >> 32); - return FEXCore::IR::RegisterClassType {Class}; + return FEXCore::IR::RegisterClassType {GetPhys(RAData, Node).Class}; } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp index 758ade1cb..1647131ef 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp @@ -460,27 +460,20 @@ void JITCore::ClearCache() { } } -uint32_t JITCore::GetPhys(uint32_t Node) { - uint64_t Reg = RAData->GetNodeRegister(Node); +IR::PhysicalRegister JITCore::GetPhys(uint32_t Node) { + auto PhyReg = RAData->GetNodeRegister(Node); - if ((uint32_t)Reg != ~0U) - return Reg; - else - LogMan::Msg::A("Couldn't Allocate register for node: ssa%d. Class: %d", Node, Reg >> 32); + LogMan::Throw::A(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class); - return ~0U; + return PhyReg; } bool JITCore::IsFPR(uint32_t Node) { - auto Class = RAData->GetNodeRegister(Node) >> 32; - - return Class == IR::FPRClass.Val; + return RAData->GetNodeRegister(Node).Class == IR::FPRClass.Val; } bool JITCore::IsGPR(uint32_t Node) { - auto Class = RAData->GetNodeRegister(Node) >> 32; - - return Class == IR::GPRClass.Val; + return RAData->GetNodeRegister(Node).Class == IR::GPRClass.Val; } template @@ -489,17 +482,17 @@ Xbyak::Reg JITCore::GetSrc(uint32_t Node) { // r10 // Callee Saved // rbx, rbp, r12, r13, r14, r15 - uint32_t Reg = GetPhys(Node); + auto PhyReg = GetPhys(Node); if (RAType == RA_64) - return RA64[Reg].cvt64(); + return RA64[PhyReg.Reg].cvt64(); else if (RAType == RA_XMM) - return RAXMM[Reg]; + return RAXMM[PhyReg.Reg]; else if (RAType == RA_32) - return RA64[Reg].cvt32(); + return RA64[PhyReg.Reg].cvt32(); else if (RAType == RA_16) - return RA64[Reg].cvt16(); + return RA64[PhyReg.Reg].cvt16(); else if (RAType == RA_8) - return RA64[Reg].cvt8(); + return RA64[PhyReg.Reg].cvt8(); } template @@ -515,23 +508,23 @@ template Xbyak::Reg JITCore::GetSrc(uint32_t Node); Xbyak::Xmm JITCore::GetSrc(uint32_t Node) { - uint32_t Reg = GetPhys(Node); - return RAXMM_x[Reg]; + auto PhyReg = GetPhys(Node); + return RAXMM_x[PhyReg.Reg]; } template Xbyak::Reg JITCore::GetDst(uint32_t Node) { - uint32_t Reg = GetPhys(Node); + auto PhyReg = GetPhys(Node); if (RAType == RA_64) - return RA64[Reg].cvt64(); + return RA64[PhyReg.Reg].cvt64(); else if (RAType == RA_XMM) - return RAXMM[Reg]; + return RAXMM[PhyReg.Reg]; else if (RAType == RA_32) - return RA64[Reg].cvt32(); + return RA64[PhyReg.Reg].cvt32(); else if (RAType == RA_16) - return RA64[Reg].cvt16(); + return RA64[PhyReg.Reg].cvt16(); else if (RAType == RA_8) - return RA64[Reg].cvt8(); + return RA64[PhyReg.Reg].cvt8(); } template @@ -548,11 +541,11 @@ Xbyak::Reg JITCore::GetDst(uint32_t Node); template std::pair JITCore::GetSrcPair(uint32_t Node) { - uint32_t Reg = GetPhys(Node); + auto PhyReg = GetPhys(Node); if (RAType == RA_64) - return RA64Pair[Reg]; + return RA64Pair[PhyReg.Reg]; else if (RAType == RA_32) - return {RA64Pair[Reg].first.cvt32(), RA64Pair[Reg].second.cvt32()}; + return {RA64Pair[PhyReg.Reg].first.cvt32(), RA64Pair[PhyReg.Reg].second.cvt32()}; } template @@ -562,8 +555,8 @@ template std::pair JITCore::GetSrcPair(uint32_t Node); Xbyak::Xmm JITCore::GetDst(uint32_t Node) { - uint32_t Reg = GetPhys(Node); - return RAXMM_x[Reg]; + auto PhyReg = GetPhys(Node); + return RAXMM_x[PhyReg.Reg]; } bool JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) { diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h index ec83fa95e..85f3ceabf 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h @@ -14,6 +14,7 @@ using namespace Xbyak; #include #include #include +#include "Interface/IR/Passes/RegisterAllocationPass.h" #include @@ -106,7 +107,7 @@ private: constexpr static uint8_t RA_64 = 3; constexpr static uint8_t RA_XMM = 4; - uint32_t GetPhys(uint32_t Node); + IR::PhysicalRegister GetPhys(uint32_t Node); bool IsFPR(uint32_t Node); bool IsGPR(uint32_t Node); diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index ad5bad4c1..1272893ab 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -30,6 +30,8 @@ "static constexpr FEXCore::IR::RegisterClassType ComplexClass {5}", "static constexpr FEXCore::IR::RegisterClassType InvalidClass {7}", "", + "static constexpr uint8_t InvalidReg {31}", + "", "static const FEXCore::IR::TypeDefinition i8 {TypeDefinition::Create(1, 0)}", "static const FEXCore::IR::TypeDefinition i16 {TypeDefinition::Create(2, 0)}", "static const FEXCore::IR::TypeDefinition i32 {TypeDefinition::Create(4, 0)}", diff --git a/External/FEXCore/Source/Interface/IR/IRDumper.cpp b/External/FEXCore/Source/Interface/IR/IRDumper.cpp index 9ae363dd1..38a66312d 100644 --- a/External/FEXCore/Source/Interface/IR/IRDumper.cpp +++ b/External/FEXCore/Source/Interface/IR/IRDumper.cpp @@ -82,10 +82,9 @@ static void PrintArg(std::stringstream *out, IRListView const* IR, Ordere } else { *out << "%ssa" << std::to_string(Arg.ID()); if (RAData) { - uint64_t RegClass = RAData->GetNodeRegister(Arg.ID()); - FEXCore::IR::RegisterClassType Class {uint32_t(RegClass >> 32)}; - uint32_t Reg = RegClass; - switch (Class) { + auto PhyReg = RAData->GetNodeRegister(Arg.ID()); + + switch (PhyReg.Class) { case FEXCore::IR::GPRClass.Val: *out << "(GPR"; break; case FEXCore::IR::GPRFixedClass.Val: *out << "(GPRFixed"; break; case FEXCore::IR::FPRClass.Val: *out << "(FPR"; break; @@ -96,8 +95,8 @@ static void PrintArg(std::stringstream *out, IRListView const* IR, Ordere default: *out << "(Unknown"; break; } - if (Class != FEXCore::IR::InvalidClass.Val) { - *out << std::dec << Reg << ")"; + if (PhyReg.Class != FEXCore::IR::InvalidClass.Val) { + *out << std::dec << (uint32_t)PhyReg.Reg << ")"; } else { *out << ")"; } @@ -205,10 +204,8 @@ void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAlloc *out << "%ssa" << std::to_string(ID); if (RAData) { - uint64_t RegClass = RAData->GetNodeRegister(ID); - FEXCore::IR::RegisterClassType Class {uint32_t(RegClass >> 32)}; - uint32_t Reg = RegClass; - switch (Class) { + auto PhyReg = RAData->GetNodeRegister(ID); + switch (PhyReg.Class) { case FEXCore::IR::GPRClass.Val: *out << "(GPR"; break; case FEXCore::IR::GPRFixedClass.Val: *out << "(GPRFixed"; break; case FEXCore::IR::FPRClass.Val: *out << "(FPR"; break; @@ -218,8 +215,8 @@ void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAlloc case FEXCore::IR::InvalidClass.Val: *out << "(Invalid"; break; default: *out << "(Unknown"; break; } - if (Class != FEXCore::IR::InvalidClass.Val) { - *out << std::dec << Reg << ")"; + if (PhyReg.Class != FEXCore::IR::InvalidClass.Val) { + *out << std::dec << (uint32_t)PhyReg.Reg << ")"; } else { *out << ")"; } diff --git a/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp b/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp index 757465f49..6b5f1b63a 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp +++ b/External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp @@ -83,10 +83,10 @@ bool IRValidation::Run(IREmitter *IREmit) { if (RAData) { // If we have a register allocator then the destination needs to be assigned a register and class - uint64_t Reg = RAData->GetNodeRegister(ID); + auto PhyReg = RAData->GetNodeRegister(ID); FEXCore::IR::RegisterClassType ExpectedClass = IR::GetRegClass(IROp->Op); - FEXCore::IR::RegisterClassType AssignedClass = FEXCore::IR::RegisterClassType{uint32_t(Reg >> 32)}; + FEXCore::IR::RegisterClassType AssignedClass = FEXCore::IR::RegisterClassType{PhyReg.Class}; // If no register class was assigned if (AssignedClass == IR::InvalidClass) { @@ -95,7 +95,7 @@ bool IRValidation::Run(IREmitter *IREmit) { } // If no physical register was assigned - if ((uint32_t)Reg == ~0U) { + if (PhyReg.Reg == IR::InvalidReg) { HadError |= true; Errors << "%ssa" << ID << ": Had destination but with no register assigned" << std::endl; } diff --git a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp index 9fbd4dcd1..32d59d931 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp +++ b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp @@ -9,13 +9,16 @@ #define SRA_DEBUG(...) // printf(__VA_ARGS__) namespace { - constexpr uint32_t INVALID_REG = 31; - constexpr uint32_t INVALID_CLASS = 7; + using FEXCore::IR::PhysicalRegister; + + constexpr uint32_t INVALID_REG = FEXCore::IR::InvalidReg; + constexpr uint32_t INVALID_CLASS = FEXCore::IR::InvalidClass.Val; + constexpr uint32_t DEFAULT_INTERFERENCE_LIST_COUNT = 122; constexpr uint32_t DEFAULT_INTERFERENCE_SPAN_COUNT = 30; constexpr uint32_t DEFAULT_NODE_COUNT = 8192; - constexpr uint64_t INVALID_REGCLASS = (((uint64_t)INVALID_CLASS) << 32) | (INVALID_REG); + const PhysicalRegister INVALID_REGCLASS = PhysicalRegister::Invalid(); // BucketList is an optimized container, it includes an inline array of Size // and can overflow to a linked list of further buckets @@ -178,8 +181,8 @@ namespace { uint32_t Begin{~0U}; uint32_t End{~0U}; uint32_t RematCost{0}; - int64_t PrefferedRegister{-1}; uint32_t PreWritten{0}; + PhysicalRegister PrefferedRegister{INVALID_REGCLASS}; bool Written{false}; bool Global{false}; }; @@ -192,7 +195,7 @@ namespace { }; struct RegisterGraph { - FEXCore::IR::RegisterAllocationData RegAllocMap; + std::unique_ptr AllocData; RegisterSet Set; std::vector Nodes; uint32_t NodeCount; @@ -221,41 +224,27 @@ namespace { Graph->Set.Classes[Class].PhysicalCount = Count; } - void SetConflict(RegisterGraph *Graph, uint64_t RegAndClass, uint64_t RegAndClassConflict) { - uint32_t Reg = (uint32_t)RegAndClass; - uint32_t Class = RegAndClass >> 32; - uint32_t RegConflict = (uint32_t)RegAndClassConflict; - uint32_t ClassConflict = RegAndClassConflict >> 32; - - uint32_t Index = ((Class << 3) | ClassConflict << 5) | Reg; + void SetConflict(RegisterGraph *Graph, PhysicalRegister RegAndClass, PhysicalRegister ConflictRegAndClass) { + uint32_t Index = (ConflictRegAndClass.Class << 8) | RegAndClass.Raw; - Graph->Set.Conflicts[Index] |= 1 << RegConflict; + Graph->Set.Conflicts[Index] |= 1 << ConflictRegAndClass.Reg; } - bool IsConflict(RegisterGraph *Graph, uint64_t RegAndClass, uint64_t ConflictRegAndClass) { - uint32_t Reg = (uint32_t)RegAndClass; - uint32_t Class = RegAndClass >> 32; - uint32_t RegConflict = (uint32_t)ConflictRegAndClass; - uint32_t ClassConflict = ConflictRegAndClass >> 32; - - uint32_t Index = ((Class << 3) | ClassConflict << 5) | Reg; - - return (Graph->Set.Conflicts[Index] >> RegConflict) & 1; + bool IsConflict(RegisterGraph *Graph, PhysicalRegister RegAndClass, PhysicalRegister ConflictRegAndClass) { + uint32_t Index = (ConflictRegAndClass.Class << 8) | RegAndClass.Raw; + return (Graph->Set.Conflicts[Index] >> ConflictRegAndClass.Reg) & 1; } - uint32_t GetConflicts(RegisterGraph *Graph, uint64_t RegAndClass, uint32_t ConflictClass) { - uint32_t Reg = (uint32_t)RegAndClass; - uint32_t Class = RegAndClass >> 32; - - uint32_t Index = ((Class << 3) | ConflictClass << 5) | Reg; + uint32_t GetConflicts(RegisterGraph *Graph, PhysicalRegister RegAndClass, FEXCore::IR::RegisterClassType ConflictClass) { + uint32_t Index = (ConflictClass.Val << 8) | RegAndClass.Raw; return Graph->Set.Conflicts[Index]; } void VirtualAddRegisterConflict(RegisterGraph *Graph, FEXCore::IR::RegisterClassType ClassConflict, uint32_t RegConflict, FEXCore::IR::RegisterClassType Class, uint32_t Reg) { - auto RegAndClass = ((uint64_t)Class << 32) | Reg; - auto RegAndClassConflict = ((uint64_t)ClassConflict << 32) | RegConflict; + auto RegAndClass = PhysicalRegister(Class, Reg); + auto RegAndClassConflict = PhysicalRegister(ClassConflict, RegConflict); // Conflict must go both ways SetConflict(Graph, RegAndClass, RegAndClassConflict); @@ -270,13 +259,15 @@ namespace { NodeCount = AlignUp(NodeCount, sizeof(uint64_t)); Graph->Nodes.clear(); Graph->Nodes.resize(NodeCount); - Graph->NodeCount = NodeCount; Graph->VisitedNodePredecessors.clear(); - Graph->RegAllocMap.RegAndClass.resize(NodeCount); + Graph->AllocData.reset(); + Graph->AllocData.reset((FEXCore::IR::RegisterAllocationData*)malloc(FEXCore::IR::RegisterAllocationData::Size(NodeCount))); + memset(&Graph->AllocData->Map[0], INVALID_REGCLASS.Raw, NodeCount); + Graph->NodeCount = NodeCount; } void SetNodeClass(RegisterGraph *Graph, uint32_t Node, FEXCore::IR::RegisterClassType Class) { - Graph->RegAllocMap.RegAndClass[Node] = ((uint64_t)Class << 32) | (Graph->RegAllocMap.RegAndClass[Node] & ~0U); + Graph->AllocData->Map[Node].Class = Class.Val; } void SetNodePartner(RegisterGraph *Graph, uint32_t Node, uint32_t Partner) { @@ -284,25 +275,21 @@ namespace { } - bool DoesNodeConflictWithRegAndClass(RegisterGraph *Graph, const uint64_t InterferenceRegAndClass, uint64_t RegAndClass) { - return IsConflict(Graph, InterferenceRegAndClass, RegAndClass); - } - /** * @brief Individual node interference check */ - bool DoesNodeInterfereWithRegister(RegisterGraph *Graph, RegisterNode const *Node, uint64_t RegAndClass) { + bool DoesNodeInterfereWithRegister(RegisterGraph *Graph, RegisterNode const *Node, PhysicalRegister RegAndClass) { // Walk the node's interference list and see if it interferes with this register return Node->Interferences.Find([Graph, RegAndClass](uint32_t InterferenceNodeId) { - auto InterferenceRegAndClass = Graph->RegAllocMap.RegAndClass[InterferenceNodeId]; - return DoesNodeConflictWithRegAndClass(Graph, InterferenceRegAndClass, RegAndClass); + auto InterferenceRegAndClass = Graph->AllocData->Map[InterferenceNodeId]; + return IsConflict(Graph, InterferenceRegAndClass, RegAndClass); }); } /** * @brief Node set walking for PHI node interference checking */ - bool DoesNodeSetInterfereWithRegister(RegisterGraph *Graph, std::vector const &Nodes, uint64_t RegAndClass) { + bool DoesNodeSetInterfereWithRegister(RegisterGraph *Graph, std::vector const &Nodes, PhysicalRegister RegAndClass) { for (auto it : Nodes) { if (DoesNodeInterfereWithRegister(Graph, it, RegAndClass)) { return true; @@ -370,7 +357,9 @@ namespace { for (auto [CodeNode, IROp] : IR->GetAllCode()) { // If the destination hasn't yet been set then set it now if (IROp->HasDest) { - SetNodeClass(Graph, IR->GetID(CodeNode), GetRegClassFromNode(IR, IROp)); + Graph->AllocData->Map[IR->GetID(CodeNode)] = PhysicalRegister(GetRegClassFromNode(IR, IROp), INVALID_REG); + } else { + //Graph->AllocData->Map[IR->GetID(CodeNode)] = PhysicalRegister::Invalid(); } } } @@ -392,6 +381,7 @@ namespace FEXCore::IR { * Top 32bits is the class, lower 32bits is the register */ RegisterAllocationData* GetAllocationData() override; + std::unique_ptr &PullAllocationData() override; private: bool OptimizeSRA; uint32_t SpillPointId; @@ -468,11 +458,11 @@ namespace FEXCore::IR { } RegisterAllocationData* ConstrainedRAPass::GetAllocationData() { - // Graph->RegAllocMap.RegAndClass.resize(Graph->Nodes.size()); - // for (size_t i = 0; i < Graph->Nodes.size(); i++) { - // Graph->RegAllocMap.RegAndClass[i] = Graph->Nodes[i].Head.RegAndClass; - // } - return &Graph->RegAllocMap; + return Graph->AllocData.get(); + } + + std::unique_ptr& ConstrainedRAPass::PullAllocationData() { + return Graph->AllocData; } void ConstrainedRAPass::RecursiveLiveRangeExpansion(FEXCore::IR::IRListView *IR, uint32_t Node, uint32_t DefiningBlockID, LiveRange *LiveRange, const std::unordered_set &Predecessors, std::unordered_set &VisitedPredecessors) { @@ -622,13 +612,13 @@ namespace FEXCore::IR { if (Offset >= beginGpr && Offset < endGpr) { auto reg = (Offset - beginGpr) / 8; - return (uint64_t(GPRFixedClass.Val)<<32) | reg; + return PhysicalRegister(GPRFixedClass, reg); } else if (Offset >= beginFpr && Offset < endFpr) { auto reg = (Offset - beginFpr) / 16; - return (uint64_t(FPRFixedClass.Val)<<32) | reg; + return PhysicalRegister(FPRFixedClass, reg); } else { LogMan::Throw::A(false, "Unexpected Offset %d", Offset); - return ~0UL; + return INVALID_REGCLASS; } }; @@ -657,16 +647,13 @@ namespace FEXCore::IR { }; // Get a StaticMap entry from reg and class - auto GetStaticMapFromReg = [&](int64_t RegAndClass) { - uint32_t Class = RegAndClass >> 32; - uint32_t Reg = RegAndClass; - - if (Class == GPRFixedClass.Val) { - return &StaticMaps[Reg]; - } else if (Class == FPRFixedClass.Val) { - return &StaticMaps[GprSize + Reg]; + auto GetStaticMapFromReg = [&](IR::PhysicalRegister PhyReg) { + if (PhyReg.Class == GPRFixedClass.Val) { + return &StaticMaps[PhyReg.Reg]; + } else if (PhyReg.Class == FPRFixedClass.Val) { + return &StaticMaps[GprSize + PhyReg.Reg]; } else { - LogMan::Throw::A(false, "Unexpected Class %d", Class); + LogMan::Throw::A(false, "Unexpected Class %d", PhyReg.Class); return (LiveRange**)nullptr; } }; @@ -680,7 +667,7 @@ namespace FEXCore::IR { //int -1 /*vreg*/ = (int)(Op->Offset / 8) - 1; if (IsPreWritable(IROp->Size, Op->StaticClass) - && LiveRanges[Op->Value.ID()].PrefferedRegister == -1 + && LiveRanges[Op->Value.ID()].PrefferedRegister.IsInvalid() && !LiveRanges[Op->Value.ID()].Global) { //pre-write and sra-allocate in the defining node - this might be undone if a read before the actual store happens @@ -712,7 +699,7 @@ namespace FEXCore::IR { // ACCESSED after write, let's not SRA this one if (LiveRanges[ArgNode].Written) { SRA_DEBUG("Demoting ssa%d because accessed after write in ssa%d\n", ArgNode, Node); - LiveRanges[ArgNode].PrefferedRegister = -1; + LiveRanges[ArgNode].PrefferedRegister = INVALID_REGCLASS; auto ArgNodeNode = IR->GetNode(IROp->Args[i]); SetNodeClass(Graph, ArgNode, GetRegClassFromNode(IR, ArgNodeNode->Op(IR->GetData()))); } @@ -722,7 +709,7 @@ namespace FEXCore::IR { if (IROp->HasDest) { // If this is a pre-write, update the StaticMap so we track writes - if (LiveRanges[Node].PrefferedRegister != -1) { + if (!LiveRanges[Node].PrefferedRegister.IsInvalid()) { SRA_DEBUG("ssa%d is a pre-write\n", Node); auto StaticMap = GetStaticMapFromReg(LiveRanges[Node].PrefferedRegister); if ((*StaticMap)) { @@ -746,13 +733,13 @@ namespace FEXCore::IR { uint32_t ID = (*StaticMap) - &LiveRanges[0]; SRA_DEBUG("ssa%d cannot be a pre-write because ssa%d reads from sra%d before storereg", ID, Node, -1 /*vreg*/); - (*StaticMap)->PrefferedRegister = -1; + (*StaticMap)->PrefferedRegister = INVALID_REGCLASS; (*StaticMap)->PreWritten = 0; SetNodeClass(Graph, ID, Op->Class); } // if not sra-allocated and full size, sra-allocate - if (!LiveRanges[Node].Global && LiveRanges[Node].PrefferedRegister == -1) { + if (!LiveRanges[Node].Global && LiveRanges[Node].PrefferedRegister.IsInvalid()) { // only full size reads can be aliased if (IsAliasable(IROp->Size, Op->StaticClass, Op->Offset)) { @@ -895,13 +882,11 @@ namespace FEXCore::IR { // Now that we have all the live ranges calculated we need to add them to our interference graph - auto GetClass = [](uint64_t RegAndClass) { - uint32_t Class = RegAndClass >> 32; - - if (Class == IR::GPRPairClass.Val) + auto GetClass = [](PhysicalRegister PhyReg) { + if (PhyReg.Class == IR::GPRPairClass.Val) return IR::GPRClass.Val; else - return Class; + return (uint32_t)PhyReg.Class; }; SpanStart.resize(NodeCount); @@ -910,7 +895,7 @@ namespace FEXCore::IR { if (LiveRanges[i].Begin != ~0U) { LogMan::Throw::A(LiveRanges[i].Begin < LiveRanges[i].End , "Span must Begin before Ending"); - auto Class = GetClass(Graph->RegAllocMap.RegAndClass[i]); + auto Class = GetClass(Graph->AllocData->Map[i]); SpanStart[LiveRanges[i].Begin].Append(INFO_MAKE(i, Class)); SpanEnd[LiveRanges[i].End] .Append(INFO_MAKE(i, Class)); } @@ -946,14 +931,14 @@ namespace FEXCore::IR { void ConstrainedRAPass::AllocateVirtualRegisters() { for (uint32_t i = 0; i < Graph->NodeCount; ++i) { RegisterNode *CurrentNode = &Graph->Nodes[i]; - auto &CurrentRegAndClass = Graph->RegAllocMap.RegAndClass[i]; + auto &CurrentRegAndClass = Graph->AllocData->Map[i]; if (CurrentRegAndClass == INVALID_REGCLASS) continue; auto LiveRange = &LiveRanges[i]; - FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentRegAndClass >> 32)}; - uint64_t RegAndClass = INVALID_REGCLASS; + FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{CurrentRegAndClass.Class}; + auto RegAndClass = INVALID_REGCLASS; RegisterClass *RAClass = &Graph->Set.Classes[RegClass]; if (CurrentNode->Head.PhiPartner) { @@ -992,26 +977,25 @@ namespace FEXCore::IR { } else { - if (LiveRange->PrefferedRegister != -1) { + if (!LiveRange->PrefferedRegister.IsInvalid()) { RegAndClass = LiveRange->PrefferedRegister; } else { uint32_t RegisterConflicts = 0; CurrentNode->Interferences.Iterate([&](const uint32_t InterferenceNode) { - RegisterConflicts |= GetConflicts(Graph, Graph->RegAllocMap.RegAndClass[InterferenceNode], RegClass); + RegisterConflicts |= GetConflicts(Graph, Graph->AllocData->Map[InterferenceNode], {RegClass}); }); RegisterConflicts = (~RegisterConflicts) & RAClass->CountMask; int Reg = ffs(RegisterConflicts); if (Reg != 0) { - RegAndClass = (static_cast(RegClass) << 32) + Reg -1; + RegAndClass = PhysicalRegister({RegClass}, Reg-1); } } // If we failed to find a virtual register then use INVALID_REG and mark allocation as failed - if (RegAndClass == INVALID_REGCLASS) { - RegAndClass = (static_cast(RegClass.Val) << 32); - RegAndClass |= INVALID_REG; + if (RegAndClass.IsInvalid()) { + RegAndClass = IR::PhysicalRegister(RegClass, INVALID_REG); HadFullRA = false; SpillPointId = i; @@ -1362,13 +1346,13 @@ namespace FEXCore::IR { uint32_t Node = IR.GetID(CodeNode); RegisterNode *CurrentNode = &Graph->Nodes[Node]; - auto &CurrentRegAndClass = Graph->RegAllocMap.RegAndClass[Node]; + auto &CurrentRegAndClass = Graph->AllocData->Map[Node]; LiveRange *OpLiveRange = &LiveRanges[Node]; // If this node is allocated above the number of physical registers we have then we need to search the interference list and spill the one // that is cheapest - FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentRegAndClass >> 32)}; - bool NeedsToSpill = (uint32_t)CurrentRegAndClass == INVALID_REG; + FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{CurrentRegAndClass.Class}; + bool NeedsToSpill = CurrentRegAndClass.Reg == INVALID_REG; if (NeedsToSpill) { bool Spilled = false; @@ -1398,11 +1382,11 @@ namespace FEXCore::IR { if (!Spilled) { uint32_t InterferenceNode = FindNodeToSpill(IREmit, CurrentNode, Node, OpLiveRange); if (InterferenceNode != ~0U) { - FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{uint32_t(Graph->RegAllocMap.RegAndClass[InterferenceNode] >> 32)}; + FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{Graph->AllocData->Map[InterferenceNode].Class}; uint32_t SpillSlot = FindSpillSlot(InterferenceNode, InterferenceRegClass); RegisterNode *InterferenceRegisterNode = &Graph->Nodes[InterferenceNode]; LogMan::Throw::A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!"); - //LogMan::Throw::A((InterferenceRegisterNode->Head.RegAndClass & ~0U) != ~0U, "Interference node never assigned a register?"); + //LogMan::Throw::A(InterferenceRegisterNode->Head.RegAndClass.Reg != INVALID_REG, "Interference node never assigned a register?"); LogMan::Throw::A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?"); LogMan::Throw::A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently"); LogMan::Throw::A(InterferenceRegClass == RegClass, "Class doesn't match"); @@ -1476,7 +1460,7 @@ namespace FEXCore::IR { if (OptimizeSRA) OptimizeStaticRegisters(&IR); - // Linear foward scan based interference calculation is faster for smaller blocks + // Linear forward scan based interference calculation is faster for smaller blocks // Smarter block based interference calculation is faster for larger blocks /*if (SSACount >= 2048) { CalculateBlockInterferences(&IR); @@ -1540,6 +1524,8 @@ namespace FEXCore::IR { CompactionPass->Run(IREmit); } + Graph->AllocData->SpillSlotCount = Graph->SpillStack.size(); + return Changed; } diff --git a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h index 3ebabf5c7..ed1b8a6c9 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h +++ b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h @@ -6,14 +6,50 @@ namespace FEXCore::IR { template class IRListView; +union PhysicalRegister { + uint8_t Raw; + struct { + uint8_t Reg: 5; + uint8_t Class: 3; + }; + + bool operator==(const PhysicalRegister &Other) const { + return Raw == Other.Raw; + } + + PhysicalRegister(RegisterClassType Class, uint8_t Reg) : Class(Class.Val), Reg(Reg) { } + + static const PhysicalRegister Invalid() { + return PhysicalRegister(InvalidClass, InvalidReg); + } + + bool IsInvalid() { + return *this == Invalid(); + } +}; + +static_assert(sizeof(PhysicalRegister) == 1); + +class RegisterAllocationData; +struct RegisterAllocationDataDeleter { + void operator()(RegisterAllocationData* r) { + free(r); + } +}; + class RegisterAllocationData { public: - std::vector RegAndClass; uint32_t SpillSlotCount {}; - uint64_t GetNodeRegister(uint32_t Node) { - return RegAndClass[Node]; + PhysicalRegister Map[0]; + + PhysicalRegister GetNodeRegister(uint32_t Node) const { + return Map[Node]; } uint32_t SpillSlots() const { return SpillSlotCount; } + + static size_t Size(uint32_t NodeCount) { + return sizeof(RegisterAllocationData) + NodeCount * sizeof(Map[0]); + } }; class RegisterAllocationPass : public FEXCore::IR::Pass { @@ -42,10 +78,14 @@ class RegisterAllocationPass : public FEXCore::IR::Pass { * @{ */ /** - * @brief Returns the register and class encoded together - * Top 32bits is the class, lower 32bits is the register + * @brief Returns the register and class map array */ - virtual RegisterAllocationData* GetAllocationData() = 0; + virtual RegisterAllocationData *GetAllocationData() = 0; + + /** + * @brief Returns and transfers ownership of the register and class map array + */ + virtual std::unique_ptr &PullAllocationData() = 0; /** @} */ protected: From 972219043db29f0e51e61aabb37e2d8d577cc394 Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Fri, 29 Jan 2021 04:02:15 +0200 Subject: [PATCH 4/4] Core: Refactor compilation driver, lifetime of objects --- .../Source/Interface/Context/Context.h | 6 +- .../Source/Interface/Core/CompileService.cpp | 47 +- .../Source/Interface/Core/CompileService.h | 7 +- .../FEXCore/Source/Interface/Core/Core.cpp | 440 ++++++++++-------- .../Core/Interpreter/InterpreterCore.cpp | 4 +- .../Source/Interface/Core/JIT/Arm64/JIT.cpp | 2 +- .../Source/Interface/Core/JIT/x86_64/JIT.cpp | 2 +- .../Source/Interface/IR/PassManager.cpp | 2 +- .../IR/Passes/RegisterAllocationPass.cpp | 6 +- .../IR/Passes/RegisterAllocationPass.h | 49 +- .../FEXCore/Debug/InternalThreadState.h | 5 +- .../FEXCore/IR/RegisterAllocationData.h | 52 +++ 12 files changed, 328 insertions(+), 294 deletions(-) create mode 100644 External/FEXCore/include/FEXCore/IR/RegisterAllocationData.h diff --git a/External/FEXCore/Source/Interface/Context/Context.h b/External/FEXCore/Source/Interface/Context/Context.h index 682da6c45..1808ddb6a 100644 --- a/External/FEXCore/Source/Interface/Context/Context.h +++ b/External/FEXCore/Source/Interface/Context/Context.h @@ -139,7 +139,9 @@ namespace FEXCore::Context { FEXCore::Core::ThreadState *GetThreadState(); void LoadEntryList(); - std::tuple CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); + std::tuple *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t> GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); + + std::tuple *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool> CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); uintptr_t CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); uintptr_t CompileFallbackBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); @@ -158,7 +160,7 @@ namespace FEXCore::Context { #endif protected: - void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); + void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache); private: void WaitForIdleWithTimeout(); diff --git a/External/FEXCore/Source/Interface/Core/CompileService.cpp b/External/FEXCore/Source/Interface/Core/CompileService.cpp index 23f3c4b6c..568718816 100644 --- a/External/FEXCore/Source/Interface/Core/CompileService.cpp +++ b/External/FEXCore/Source/Interface/Core/CompileService.cpp @@ -33,7 +33,7 @@ namespace FEXCore { WorkerThread.join(); } - void CompileService::ClearCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { + void CompileService::ClearCache(FEXCore::Core::InternalThreadState *Thread) { // On cache clear we need to spin down the execution thread to ensure it isn't trying to give us more work items if (CompileMutex.try_lock()) { // We can only clear these things if we pulled the compile mutex @@ -57,14 +57,9 @@ namespace FEXCore { } } - if (GuestRIP == 0) { - CompileThreadData->IRLists.clear(); - } - else { - auto IR = CompileThreadData->IRLists.find(GuestRIP)->second.release(); - CompileThreadData->IRLists.clear(); - CompileThreadData->IRLists.try_emplace(GuestRIP, IR); - } + LogMan::Throw::A(CompileThreadData->IRLists.size() == 0, "Compile service must never have IRLists"); + LogMan::Throw::A(CompileThreadData->RALists.size() == 0, "Compile service must never have RALists"); + LogMan::Throw::A(CompileThreadData->DebugData.size() == 0, "Compile service must never have DebugData"); CompileMutex.unlock(); } @@ -76,11 +71,6 @@ namespace FEXCore { SelectedThread->IntBackend->ClearCache(); } - void CompileService::RemoveCodeEntry(uint64_t GuestRIP) { - CompileThreadData->IRLists.erase(GuestRIP); - CompileThreadData->DebugData.erase(GuestRIP); - CompileThreadData->LookupCache->Erase(GuestRIP); - } CompileService::WorkItem *CompileService::CompileCode(uint64_t RIP) { // Tell the worker thread to compile code for us @@ -132,29 +122,26 @@ namespace FEXCore { // If we had a work item then work on it if (Item) { - // Does the block cache already contain this RIP? - void *CompiledCode = reinterpret_cast(CompileThreadData->LookupCache->FindBlock(Item->RIP)); - FEXCore::Core::DebugData *DebugData = nullptr; + // Make sure it's not in lookup cache by accident + LogMan::Throw::A(CompileThreadData->LookupCache->FindBlock(Item->RIP) == 0, "Compile Service must never have entries in the LookupCache"); + + // Code isn't in cache, compile now + // Set our thread state's RIP + CompileThreadData->State.State.rip = Item->RIP; - if (!CompiledCode) { - // Code isn't in cache, compile now - // Set our thread state's RIP - CompileThreadData->State.State.rip = Item->RIP; - auto [Code, Data] = CTX->CompileCode(CompileThreadData.get(), Item->RIP); - CompiledCode = Code; - DebugData = Data; - } + auto [CodePtr, IRList, DebugData, RAData, Generated] = CTX->CompileCode(CompileThreadData.get(), Item->RIP); - if (!CompiledCode) { + LogMan::Throw::A(Generated == true, "Compile Service doesn't have IR Cache"); + + if (!CodePtr) { // XXX: We currently have the expectation that compile service code will be significantly smaller than regular thread's code ERROR_AND_DIE("Couldn't compile code for thread at RIP: 0x%lx", Item->RIP); } - CompileThreadData->LookupCache->AddBlockMapping(Item->RIP, CompiledCode); - - Item->CodePtr = CompiledCode; - Item->IRList = CompileThreadData->IRLists.find(Item->RIP)->second.get(); + Item->CodePtr = CodePtr; + Item->IRList = IRList; Item->DebugData = DebugData; + Item->RAData = RAData; GCArray.emplace_back(Item); Item->ServiceWorkDone.NotifyAll(); diff --git a/External/FEXCore/Source/Interface/Core/CompileService.h b/External/FEXCore/Source/Interface/Core/CompileService.h index a2ee7dbef..321ac4326 100644 --- a/External/FEXCore/Source/Interface/Core/CompileService.h +++ b/External/FEXCore/Source/Interface/Core/CompileService.h @@ -16,6 +16,9 @@ namespace Core { struct InternalThreadState; } +namespace IR { + class RegisterAllocationData; +}; class CompileService final { public: CompileService(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread); @@ -29,6 +32,7 @@ class CompileService final { // Outgoing void *CodePtr{}; FEXCore::IR::IRListView *IRList{}; + FEXCore::IR::RegisterAllocationData *RAData{}; FEXCore::Core::DebugData *DebugData{}; // Communication @@ -37,8 +41,7 @@ class CompileService final { }; WorkItem *CompileCode(uint64_t RIP); - void ClearCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP); - void RemoveCodeEntry(uint64_t GuestRIP); + void ClearCache(FEXCore::Core::InternalThreadState *Thread); private: FEXCore::Context::Context *CTX; diff --git a/External/FEXCore/Source/Interface/Core/Core.cpp b/External/FEXCore/Source/Interface/Core/Core.cpp index 2f1434648..d23410c71 100644 --- a/External/FEXCore/Source/Interface/Core/Core.cpp +++ b/External/FEXCore/Source/Interface/Core/Core.cpp @@ -378,7 +378,7 @@ namespace FEXCore::Context { // Walk the threads and tell them to clear their caches // Useful when our block size is set to a large number and we need to step a single instruction for (auto &Thread : Threads) { - ClearCodeCache(Thread, 0); + ClearCodeCache(Thread, true); } } CoreRunningMode PreviousRunningMode = this->Config.RunningMode; @@ -464,6 +464,8 @@ namespace FEXCore::Context { // Run the passmanager over the IR from the dispatcher Thread->PassManager->Run(IR); Thread->IRLists.try_emplace(Addr, IR->CreateIRCopy()); + Thread->DebugData.try_emplace(Addr, new Core::DebugData()); + Thread->RALists.try_emplace(Addr, Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->PullAllocationData() : nullptr); }; LocalLoader->AddIR(IRHandler); @@ -563,242 +565,250 @@ namespace FEXCore::Context { Thread->LookupCache->AddBlockMapping(Address, Ptr); } - void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { + void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache) { Thread->LookupCache->ClearCache(); Thread->CPUBackend->ClearCache(); Thread->IntBackend->ClearCache(); - if (Thread->CompileService) { - Thread->CompileService->ClearCache(Thread, GuestRIP); - } - - if (GuestRIP == 0) { + if (AlsoClearIRCache) { Thread->IRLists.clear(); - } - else { - auto IR = Thread->IRLists.find(GuestRIP)->second.release(); - Thread->IRLists.clear(); - Thread->IRLists.try_emplace(GuestRIP, IR); + Thread->RALists.clear(); + Thread->DebugData.clear(); } } - std::tuple Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { + std::tuple *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t> Context::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { uint8_t const *GuestCode{}; - GuestCode = reinterpret_cast(GuestRIP); + GuestCode = reinterpret_cast(GuestRIP); - // Do we already have this in the IR cache? - auto IR = Thread->IRLists.find(GuestRIP); - FEXCore::IR::IRListView *IRList {}; - FEXCore::Core::DebugData *DebugData {}; - FEXCore::IR::RegisterAllocationData *RAData {}; + bool HadDispatchError {false}; - if (IR == Thread->IRLists.end()) { - bool HadDispatchError {false}; + uint64_t TotalInstructions {0}; + uint64_t TotalInstructionsLength {0}; - uint64_t TotalInstructions {0}; - uint64_t TotalInstructionsLength {0}; + if (!Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP)) { + if (Config.BreakOnFrontendFailure) { + LogMan::Msg::E("Had Frontend decoder error"); + Stop(false /* Ignore Current Thread */); + } + return { nullptr, nullptr, 0, 0 }; + } - if (!Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP)) { - if (Config.BreakOnFrontendFailure) { - LogMan::Msg::E("Had Frontend decoder error"); - Stop(false /* Ignore Current Thread */); - } - return { nullptr, nullptr }; + auto CodeBlocks = Thread->FrontendDecoder->GetDecodedBlocks(); + + Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks); + + for (size_t j = 0; j < CodeBlocks->size(); ++j) { + FEXCore::Frontend::Decoder::DecodedBlocks const &Block = CodeBlocks->at(j); + // Set the block entry point + Thread->OpDispatcher->SetNewBlockIfChanged(Block.Entry); + + + uint64_t BlockInstructionsLength {}; + + // Reset any block-specific state + Thread->OpDispatcher->StartNewBlock(); + + uint64_t InstsInBlock = Block.NumInstructions; + + if (Block.HasInvalidInstruction) { + uint8_t GPRSize = Config.Is64BitMode ? 8 : 4; + Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry)); + break; } - auto CodeBlocks = Thread->FrontendDecoder->GetDecodedBlocks(); + for (size_t i = 0; i < InstsInBlock; ++i) { + FEXCore::X86Tables::X86InstInfo const* TableInfo {nullptr}; + FEXCore::X86Tables::DecodedInst const* DecodedInfo {nullptr}; - Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks); + TableInfo = Block.DecodedInstructions[i].TableInfo; + DecodedInfo = &Block.DecodedInstructions[i]; - for (size_t j = 0; j < CodeBlocks->size(); ++j) { - FEXCore::Frontend::Decoder::DecodedBlocks const &Block = CodeBlocks->at(j); - // Set the block entry point - Thread->OpDispatcher->SetNewBlockIfChanged(Block.Entry); + if (Config.SMCChecks) { + auto ExistingCodePtr = reinterpret_cast(Block.Entry + BlockInstructionsLength); + auto CodeChanged = Thread->OpDispatcher->_ValidateCode(ExistingCodePtr[0], ExistingCodePtr[1], (uintptr_t)ExistingCodePtr, DecodedInfo->InstSize); - uint64_t BlockInstructionsLength {}; + auto InvalidateCodeCond = Thread->OpDispatcher->_CondJump(CodeChanged); - // Reset any block-specific state - Thread->OpDispatcher->StartNewBlock(); + auto CurrentBlock = Thread->OpDispatcher->GetCurrentBlock(); + auto CodeWasChangedBlock = Thread->OpDispatcher->CreateNewCodeBlockAtEnd(); + Thread->OpDispatcher->SetTrueJumpTarget(InvalidateCodeCond, CodeWasChangedBlock); - uint64_t InstsInBlock = Block.NumInstructions; + Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock); + Thread->OpDispatcher->_RemoveCodeEntry(GuestRIP); + Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(Block.Entry + BlockInstructionsLength)); + + auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock); - if (Block.HasInvalidInstruction) { - uint8_t GPRSize = Config.Is64BitMode ? 8 : 4; - Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry)); - break; + Thread->OpDispatcher->SetFalseJumpTarget(InvalidateCodeCond, NextOpBlock); + Thread->OpDispatcher->SetCurrentCodeBlock(NextOpBlock); } - for (size_t i = 0; i < InstsInBlock; ++i) { - FEXCore::X86Tables::X86InstInfo const* TableInfo {nullptr}; - FEXCore::X86Tables::DecodedInst const* DecodedInfo {nullptr}; - - TableInfo = Block.DecodedInstructions[i].TableInfo; - DecodedInfo = &Block.DecodedInstructions[i]; - - if (Config.SMCChecks) { - auto ExistingCodePtr = reinterpret_cast(Block.Entry + BlockInstructionsLength); - - auto CodeChanged = Thread->OpDispatcher->_ValidateCode(ExistingCodePtr[0], ExistingCodePtr[1], (uintptr_t)ExistingCodePtr, DecodedInfo->InstSize); - - auto InvalidateCodeCond = Thread->OpDispatcher->_CondJump(CodeChanged); - - auto CurrentBlock = Thread->OpDispatcher->GetCurrentBlock(); - auto CodeWasChangedBlock = Thread->OpDispatcher->CreateNewCodeBlockAtEnd(); - Thread->OpDispatcher->SetTrueJumpTarget(InvalidateCodeCond, CodeWasChangedBlock); - - Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock); - Thread->OpDispatcher->_RemoveCodeEntry(GuestRIP); - Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(Block.Entry + BlockInstructionsLength)); - - auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock); - - Thread->OpDispatcher->SetFalseJumpTarget(InvalidateCodeCond, NextOpBlock); - Thread->OpDispatcher->SetCurrentCodeBlock(NextOpBlock); - } - - if (TableInfo->OpcodeDispatcher) { - auto Fn = TableInfo->OpcodeDispatcher; - std::invoke(Fn, Thread->OpDispatcher, DecodedInfo); - if (Thread->OpDispatcher->HadDecodeFailure()) { - if (Config.BreakOnFrontendFailure) { - LogMan::Msg::E("Had OpDispatcher error at 0x%lx", GuestRIP); - Stop(false /* Ignore Current Thread */); - } - HadDispatchError = true; + if (TableInfo->OpcodeDispatcher) { + auto Fn = TableInfo->OpcodeDispatcher; + std::invoke(Fn, Thread->OpDispatcher, DecodedInfo); + if (Thread->OpDispatcher->HadDecodeFailure()) { + if (Config.BreakOnFrontendFailure) { + LogMan::Msg::E("Had OpDispatcher error at 0x%lx", GuestRIP); + Stop(false /* Ignore Current Thread */); } - else { - BlockInstructionsLength += DecodedInfo->InstSize; - TotalInstructionsLength += DecodedInfo->InstSize; - ++TotalInstructions; - } - } - else { - LogMan::Msg::E("Missing OpDispatcher at 0x%lx{'%s'}", Block.Entry + BlockInstructionsLength, TableInfo->Name); HadDispatchError = true; } - - // If we had a dispatch error then leave early - if (HadDispatchError) { - if (TotalInstructions == 0) { - // Couldn't handle any instruction in op dispatcher - Thread->OpDispatcher->ResetWorkingList(); - return { nullptr, nullptr }; - } - else { - uint8_t GPRSize = Config.Is64BitMode ? 8 : 4; - - // We had some instructions. Early exit - Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry + BlockInstructionsLength)); - break; - } + else { + BlockInstructionsLength += DecodedInfo->InstSize; + TotalInstructionsLength += DecodedInfo->InstSize; + ++TotalInstructions; } + } + else { + LogMan::Msg::E("Missing OpDispatcher at 0x%lx{'%s'}", Block.Entry + BlockInstructionsLength, TableInfo->Name); + HadDispatchError = true; + } - if (Thread->OpDispatcher->FinishOp(DecodedInfo->PC + DecodedInfo->InstSize, i + 1 == InstsInBlock)) { + // If we had a dispatch error then leave early + if (HadDispatchError) { + if (TotalInstructions == 0) { + // Couldn't handle any instruction in op dispatcher + Thread->OpDispatcher->ResetWorkingList(); + return { nullptr, nullptr, 0, 0 }; + } + else { + uint8_t GPRSize = Config.Is64BitMode ? 8 : 4; + + // We had some instructions. Early exit + Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry + BlockInstructionsLength)); break; } } - } - Thread->OpDispatcher->Finalize(); - - - - auto IRDumper = [Thread, GuestRIP](IR::RegisterAllocationData* RA) { - FILE* f = nullptr; - bool CloseAfter = false; - - if (Thread->CTX->Config.DumpIR=="stderr") { - f = stderr; - } - else if (Thread->CTX->Config.DumpIR=="stdout") { - f = stdout; - } - else { - std::stringstream fileName; - fileName << Thread->CTX->Config.DumpIR << "/" << std::hex << GuestRIP << (RA ? "-post.ir" : "-pre.ir"); - - f = fopen(fileName.str().c_str(), "w"); - CloseAfter = true; - } - - if (f) { - std::stringstream out; - auto NewIR = Thread->OpDispatcher->ViewIR(); - FEXCore::IR::Dump(&out, &NewIR, RA); - fprintf(f,"IR-%s 0x%lx:\n%s\n@@@@@\n", RA ? "post" : "pre", GuestRIP, out.str().c_str()); - - if (CloseAfter) { - fclose(f); - } - } - }; - - if (Thread->CTX->Config.DumpIR != "no") { - IRDumper(nullptr); - } - - if (Thread->CTX->Config.ValidateIRarser) { - // Convert to text, Parse, Convert to text again and make sure the texts match - std::stringstream out; - static auto compaction = IR::CreateIRCompaction(); - compaction->Run(Thread->OpDispatcher.get()); - auto NewIR = Thread->OpDispatcher->ViewIR(); - Dump(&out, &NewIR, nullptr); - out.seekg(0); - auto reparsed = IR::Parse(&out); - if (reparsed == nullptr) { - LogMan::Msg::A("Failed to parse ir\n"); - } else { - std::stringstream out2; - auto NewIR2 = reparsed->ViewIR(); - Dump(&out2, &NewIR2, nullptr); - if (out.str() != out2.str()) { - printf("one:\n %s\n", out.str().c_str()); - printf("two:\n %s\n", out2.str().c_str()); - LogMan::Msg::A("Parsed ir doesn't match\n"); - } - delete reparsed; + if (Thread->OpDispatcher->FinishOp(DecodedInfo->PC + DecodedInfo->InstSize, i + 1 == InstsInBlock)) { + break; } } - // Run the passmanager over the IR from the dispatcher - Thread->PassManager->Run(Thread->OpDispatcher.get()); - - if (Thread->CTX->Config.DumpIR != "no") { - IRDumper(Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr); - } - - if (Thread->OpDispatcher->ShouldDump) { - std::stringstream out; - auto NewIR = Thread->OpDispatcher->ViewIR(); - FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr); - printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str()); - } - - RAData = Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr; - - // Create a copy of the IR and place it in this thread's IR cache - auto AddedIR = Thread->IRLists.try_emplace(GuestRIP, Thread->OpDispatcher->CreateIRCopy()); - Thread->OpDispatcher->ResetWorkingList(); - - auto Debugit = &Thread->DebugData.try_emplace(GuestRIP).first->second; - Debugit->GuestCodeSize = TotalInstructionsLength; - Debugit->GuestInstructionCount = TotalInstructions; - - IRList = AddedIR.first->second.get(); - DebugData = Debugit; - Thread->Stats.BlocksCompiled.fetch_add(1); } - else { - IRList = IR->second.get(); - auto Debugit = Thread->DebugData.find(GuestRIP); - if (Debugit != Thread->DebugData.end()) { - DebugData = &Debugit->second; + + Thread->OpDispatcher->Finalize(); + + auto IRDumper = [Thread, GuestRIP](IR::RegisterAllocationData* RA) { + FILE* f = nullptr; + bool CloseAfter = false; + + if (Thread->CTX->Config.DumpIR=="stderr") { + f = stderr; } + else if (Thread->CTX->Config.DumpIR=="stdout") { + f = stdout; + } + else { + std::stringstream fileName; + fileName << Thread->CTX->Config.DumpIR << "/" << std::hex << GuestRIP << (RA ? "-post.ir" : "-pre.ir"); + + f = fopen(fileName.str().c_str(), "w"); + CloseAfter = true; + } + + if (f) { + std::stringstream out; + auto NewIR = Thread->OpDispatcher->ViewIR(); + FEXCore::IR::Dump(&out, &NewIR, RA); + fprintf(f,"IR-%s 0x%lx:\n%s\n@@@@@\n", RA ? "post" : "pre", GuestRIP, out.str().c_str()); + + if (CloseAfter) { + fclose(f); + } + } + }; + + if (Thread->CTX->Config.DumpIR != "no") { + IRDumper(nullptr); + } + + if (Thread->CTX->Config.ValidateIRarser) { + // Convert to text, Parse, Convert to text again and make sure the texts match + std::stringstream out; + static auto compaction = IR::CreateIRCompaction(); + compaction->Run(Thread->OpDispatcher.get()); + auto NewIR = Thread->OpDispatcher->ViewIR(); + Dump(&out, &NewIR, nullptr); + out.seekg(0); + auto reparsed = IR::Parse(&out); + if (reparsed == nullptr) { + LogMan::Msg::A("Failed to parse ir\n"); + } else { + std::stringstream out2; + auto NewIR2 = reparsed->ViewIR(); + Dump(&out2, &NewIR2, nullptr); + if (out.str() != out2.str()) { + printf("one:\n %s\n", out.str().c_str()); + printf("two:\n %s\n", out2.str().c_str()); + LogMan::Msg::A("Parsed ir doesn't match\n"); + } + delete reparsed; + } + } + // Run the passmanager over the IR from the dispatcher + Thread->PassManager->Run(Thread->OpDispatcher.get()); + + if (Thread->CTX->Config.DumpIR != "no") { + IRDumper(Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr); + } + + if (Thread->OpDispatcher->ShouldDump) { + std::stringstream out; + auto NewIR = Thread->OpDispatcher->ViewIR(); + FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr); + printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str()); + } + + auto RAData = Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->PullAllocationData() : nullptr; + auto IRList = Thread->OpDispatcher->CreateIRCopy(); + + Thread->OpDispatcher->ResetWorkingList(); + + return {IRList, RAData.release(), TotalInstructions, TotalInstructionsLength}; + } + + std::tuple *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool> Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { + FEXCore::IR::IRListView *IRList {}; + FEXCore::Core::DebugData *DebugData {}; + FEXCore::IR::RegisterAllocationData *RAData {}; + bool GeneratedIR {}; + + // Do we already have this in the IR cache? + auto IR = Thread->IRLists.find(GuestRIP); + + if (IR != Thread->IRLists.end()) { + // Entry already exists + // pull in the data + IRList = IR->second.get(); + DebugData = Thread->DebugData.find(GuestRIP)->second.get(); + RAData = Thread->RALists.find(GuestRIP)->second.get(); + + GeneratedIR = false; + } else { + + // Generate IR + Meta Info + auto [IRCopy, RACopy, TotalInstructions, TotalInstructionsLength] = GenerateIR(Thread, GuestRIP); + + // Setup pointers to internal structures + IRList = IRCopy; + RAData = RACopy; + DebugData = new FEXCore::Core::DebugData(); + + // Initialize metadata + DebugData->GuestCodeSize = TotalInstructionsLength; + DebugData->GuestInstructionCount = TotalInstructions; + + // Increment stats + Thread->Stats.BlocksCompiled.fetch_add(1); + + // These blocks aren't already in the cache + GeneratedIR = true; } // Attempt to get the CPU backend to compile this code - return { Thread->CPUBackend->CompileCode(IRList, DebugData, RAData), DebugData }; + return { Thread->CPUBackend->CompileCode(IRList, DebugData, RAData), IRList, DebugData, RAData, GeneratedIR}; } uintptr_t Context::CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { @@ -809,28 +819,42 @@ namespace FEXCore::Context { return HostCode; } - void *CodePtr; - FEXCore::Core::DebugData *DebugData; + void *CodePtr {}; + FEXCore::IR::IRListView *IRList {}; + FEXCore::Core::DebugData *DebugData {}; + FEXCore::IR::RegisterAllocationData *RAData {}; + bool DecrementRefCount = false; + bool GeneratedIR {}; if (Thread->CompileBlockReentrantRefCount != 0) { if (!Thread->CompileService) { Thread->CompileService = std::make_shared(this, Thread); Thread->CompileService->Initialize(); } + auto WorkItem = Thread->CompileService->CompileCode(GuestRIP); WorkItem->ServiceWorkDone.Wait(); // Return here with the data in place - Thread->IRLists.try_emplace(GuestRIP, WorkItem->IRList->CreateCopy()); CodePtr = WorkItem->CodePtr; + IRList = WorkItem->IRList; DebugData = WorkItem->DebugData; + RAData = WorkItem->RAData; WorkItem->SafeToClear = true; + + // The compile service will always generate IR + DebugData + RAData + // Remove the entries here to make sure we don't fail to insert later on + RemoveCodeEntry(Thread, GuestRIP); + GeneratedIR = true; } else { ++Thread->CompileBlockReentrantRefCount; DecrementRefCount = true; - auto [Code, Data] = CompileCode(Thread, GuestRIP); + auto [Code, IR, Data, RA, Generated] = CompileCode(Thread, GuestRIP); CodePtr = Code; + IRList = IR; DebugData = Data; + RAData = RA; + GeneratedIR = Generated; } if (CodePtr != nullptr) { @@ -847,8 +871,17 @@ namespace FEXCore::Context { } #endif + // Insert to caches if we generated IR + if (GeneratedIR) { + Thread->IRLists.emplace(GuestRIP, IRList); + Thread->DebugData.emplace(GuestRIP, DebugData); + Thread->RALists.emplace(GuestRIP, RAData); + } + if (DecrementRefCount) --Thread->CompileBlockReentrantRefCount; + + // Insert to lookup cache AddBlockMapping(Thread, GuestRIP, CodePtr); return (uintptr_t)CodePtr; @@ -921,12 +954,9 @@ namespace FEXCore::Context { void Context::RemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) { Thread->IRLists.erase(GuestRIP); + Thread->RALists.erase(GuestRIP); Thread->DebugData.erase(GuestRIP); Thread->LookupCache->Erase(GuestRIP); - - if (Thread->CompileService) { - Thread->CompileService->RemoveCodeEntry(GuestRIP); - } } // Debug interface diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp index 1f4a97956..bc03698af 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp @@ -4719,10 +4719,12 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { } } + #ifndef NDEBUG auto DebugData = Thread->DebugData.find(Thread->State.State.rip); if (DebugData != Thread->DebugData.end()) { - Thread->Stats.InstructionsExecuted.fetch_add(DebugData->second.GuestInstructionCount); + Thread->Stats.InstructionsExecuted.fetch_add(DebugData->second->GuestInstructionCount); } + #endif } FEXCore::CPU::CPUBackend *CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) { diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index ad422b2ea..51053093a 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -798,7 +798,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const // Fairly excessive buffer range to make sure we don't overflow uint32_t BufferRange = SSACount * 16; if ((GetCursorOffset() + BufferRange) > CurrentCodeBuffer->Size) { - State->CTX->ClearCodeCache(State, HeaderOp->Entry); + State->CTX->ClearCodeCache(State, false); } // AAPCS64 diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp index 1647131ef..c07c6ab80 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp @@ -613,7 +613,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const // Fairly excessive buffer range to make sure we don't overflow uint32_t BufferRange = SSACount * 16; if ((getSize() + BufferRange) > CurrentCodeBuffer->Size) { - ThreadState->CTX->ClearCodeCache(ThreadState, HeaderOp->Entry); + ThreadState->CTX->ClearCodeCache(ThreadState, false); } void *Entry = getCurr(); diff --git a/External/FEXCore/Source/Interface/IR/PassManager.cpp b/External/FEXCore/Source/Interface/IR/PassManager.cpp index 39eed4690..992fef2c0 100644 --- a/External/FEXCore/Source/Interface/IR/PassManager.cpp +++ b/External/FEXCore/Source/Interface/IR/PassManager.cpp @@ -1,6 +1,6 @@ +#include "Interface/IR/PassManager.h" #include "Interface/IR/Passes.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" -#include "Interface/IR/PassManager.h" #include diff --git a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp index 32d59d931..762c12487 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp +++ b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp @@ -381,7 +381,7 @@ namespace FEXCore::IR { * Top 32bits is the class, lower 32bits is the register */ RegisterAllocationData* GetAllocationData() override; - std::unique_ptr &PullAllocationData() override; + std::unique_ptr PullAllocationData() override; private: bool OptimizeSRA; uint32_t SpillPointId; @@ -461,8 +461,8 @@ namespace FEXCore::IR { return Graph->AllocData.get(); } - std::unique_ptr& ConstrainedRAPass::PullAllocationData() { - return Graph->AllocData; + std::unique_ptr ConstrainedRAPass::PullAllocationData() { + return std::move(Graph->AllocData); } void ConstrainedRAPass::RecursiveLiveRangeExpansion(FEXCore::IR::IRListView *IR, uint32_t Node, uint32_t DefiningBlockID, LiveRange *LiveRange, const std::unordered_set &Predecessors, std::unordered_set &VisitedPredecessors) { diff --git a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h index ed1b8a6c9..86f21e4aa 100644 --- a/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h +++ b/External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h @@ -1,57 +1,12 @@ #pragma once #include "Interface/IR/PassManager.h" +#include #include namespace FEXCore::IR { template class IRListView; -union PhysicalRegister { - uint8_t Raw; - struct { - uint8_t Reg: 5; - uint8_t Class: 3; - }; - - bool operator==(const PhysicalRegister &Other) const { - return Raw == Other.Raw; - } - - PhysicalRegister(RegisterClassType Class, uint8_t Reg) : Class(Class.Val), Reg(Reg) { } - - static const PhysicalRegister Invalid() { - return PhysicalRegister(InvalidClass, InvalidReg); - } - - bool IsInvalid() { - return *this == Invalid(); - } -}; - -static_assert(sizeof(PhysicalRegister) == 1); - -class RegisterAllocationData; -struct RegisterAllocationDataDeleter { - void operator()(RegisterAllocationData* r) { - free(r); - } -}; - -class RegisterAllocationData { - public: - uint32_t SpillSlotCount {}; - PhysicalRegister Map[0]; - - PhysicalRegister GetNodeRegister(uint32_t Node) const { - return Map[Node]; - } - uint32_t SpillSlots() const { return SpillSlotCount; } - - static size_t Size(uint32_t NodeCount) { - return sizeof(RegisterAllocationData) + NodeCount * sizeof(Map[0]); - } -}; - class RegisterAllocationPass : public FEXCore::IR::Pass { public: bool HasFullRA() const { return HadFullRA; } @@ -85,7 +40,7 @@ class RegisterAllocationPass : public FEXCore::IR::Pass { /** * @brief Returns and transfers ownership of the register and class map array */ - virtual std::unique_ptr &PullAllocationData() = 0; + virtual std::unique_ptr PullAllocationData() = 0; /** @} */ protected: diff --git a/External/FEXCore/include/FEXCore/Debug/InternalThreadState.h b/External/FEXCore/include/FEXCore/Debug/InternalThreadState.h index 18b983af5..c58ab276e 100644 --- a/External/FEXCore/include/FEXCore/Debug/InternalThreadState.h +++ b/External/FEXCore/include/FEXCore/Debug/InternalThreadState.h @@ -3,7 +3,9 @@ #include #include #include +#include #include + #include #include @@ -78,7 +80,8 @@ namespace FEXCore::Core { std::unique_ptr LookupCache; std::unordered_map>> IRLists; - std::unordered_map DebugData; + std::unordered_map> RALists; + std::unordered_map> DebugData; std::unique_ptr FrontendDecoder; std::unique_ptr PassManager; diff --git a/External/FEXCore/include/FEXCore/IR/RegisterAllocationData.h b/External/FEXCore/include/FEXCore/IR/RegisterAllocationData.h new file mode 100644 index 000000000..112574813 --- /dev/null +++ b/External/FEXCore/include/FEXCore/IR/RegisterAllocationData.h @@ -0,0 +1,52 @@ +#pragma once +#include "IR.h" + +namespace FEXCore::IR { + +union PhysicalRegister { + uint8_t Raw; + struct { + uint8_t Reg: 5; + uint8_t Class: 3; + }; + + bool operator==(const PhysicalRegister &Other) const { + return Raw == Other.Raw; + } + + PhysicalRegister(RegisterClassType Class, uint8_t Reg) : Reg(Reg), Class(Class.Val) { } + + static const PhysicalRegister Invalid() { + return PhysicalRegister(InvalidClass, InvalidReg); + } + + bool IsInvalid() { + return *this == Invalid(); + } +}; + +static_assert(sizeof(PhysicalRegister) == 1); + +class RegisterAllocationData; +struct RegisterAllocationDataDeleter { + void operator()(RegisterAllocationData* r) { + free(r); + } +}; + +class RegisterAllocationData { + public: + uint32_t SpillSlotCount {}; + PhysicalRegister Map[0]; + + PhysicalRegister GetNodeRegister(uint32_t Node) const { + return Map[Node]; + } + uint32_t SpillSlots() const { return SpillSlotCount; } + + static size_t Size(uint32_t NodeCount) { + return sizeof(RegisterAllocationData) + NodeCount * sizeof(Map[0]); + } +}; + +}