steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 09:00:17 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity

Refactor IR and other changes that are hard to split

Browse Source 
I had to change how blocks are represented to make it easier to parse
This required a fairly substantial refactor that makes it so blocks are
represented differently and we can walk them sequentially.

This will make future analysis easier to deal with.
Had to rewrite the passes and core's parsing of the IR afterwards.

Moved RA in to a optimization pass to be shared between the JIT backends
This works because x86-64 and AArch64 RA can be identical.

Still doesn't support PHI nodes or spilling correctly, this is the first
step in the process of getting there.
This commit is contained in:
Ryan Houdek authored and Stefanos Kornilios Mitsis Poiitidis committed 2020-03-06 07:55:13 +02:00
1 parent 6ac9f93d0d
commit 199cfd76d8
28 files changed
+4353 -3770
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Scripts/json_ir_generator.py
+39 -11
View File
@@ -36,7 +36,7 @@ def print_ir_structs(ops, defines):
output_file.write("\ttemplate<typename T>\n")
output_file.write("\tT* CW() { return reinterpret_cast<T*>(Data); }\n")
output_file.write("\tNodeWrapper Args[0];\n")
output_file.write("\tOrderedNodeWrapper Args[0];\n")
output_file.write("};\n\n");
@@ -48,6 +48,7 @@ def print_ir_structs(ops, defines):
for op_key, op_vals in ops.items():
SSAArgs = 0
HasArgs = False
HasSSANames = False
if ("SSAArgs" in op_vals):
SSAArgs = int(op_vals["SSAArgs"])
@@ -55,16 +56,24 @@ def print_ir_structs(ops, defines):
if ("Args" in op_vals and len(op_vals["Args"]) != 0):
HasArgs = True
if ("SSANames" in op_vals and len(op_vals["SSANames"]) != 0):
HasSSANames = True
if (HasArgs or SSAArgs != 0):
output_file.write("struct __attribute__((packed)) IROp_%s {\n" % op_key)
output_file.write("\tIROp_Header Header;\n\n")
# SSA arguments have a hard requirement to appear after the header
if (SSAArgs != 0):
output_file.write("private:\n")
for i in range(0, SSAArgs):
output_file.write("\tuint64_t : (sizeof(NodeWrapper) * 8);\n");
output_file.write("public:\n")
if (HasSSANames):
for i in range(0, SSAArgs):
output_file.write("\tOrderedNodeWrapper %s;\n" % (op_vals["SSANames"][i]));
else:
output_file.write("private:\n")
for i in range(0, SSAArgs):
output_file.write("\tuint64_t : (sizeof(OrderedNodeWrapper) * 8);\n");
output_file.write("public:\n")
if (HasArgs):
output_file.write("\t// User defined data\n")
@@ -182,13 +191,32 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("#ifdef IROP_ALLOCATE_HELPERS\n")
output_file.write("\ttemplate <class T>\n")
output_file.write("\tusing IRPair = FEXCore::IR::Wrapper<T>;\n\n")
output_file.write("\tstruct Wrapper final {\n")
output_file.write("\t\tT *first;\n")
output_file.write("\t\tOrderedNode *Node; ///< Actual offset of this IR in ths list\n")
output_file.write("\t\t\n")
output_file.write("\t\toperator Wrapper<IROp_Header>() const { return Wrapper<IROp_Header> {reinterpret_cast<IROp_Header*>(first), Node}; }\n")
output_file.write("\t\toperator OrderedNode *() { return Node; }\n")
output_file.write("\t\toperator OpNodeWrapper () { return Node->Header.Value; }\n")
output_file.write("\t};\n")
output_file.write("\ttemplate <class T>\n")
output_file.write("\tusing IRPair = Wrapper<T>;\n\n")
output_file.write("\tIRPair<IROp_Header> AllocateRawOp(size_t HeaderSize) {\n")
output_file.write("\t\tauto Op = reinterpret_cast<IROp_Header*>(Data.Allocate(HeaderSize));\n")
output_file.write("\t\tmemset(Op, 0, HeaderSize);\n")
output_file.write("\t\tOp->Op = IROps::OP_DUMMY;\n")
output_file.write("\t\treturn FEXCore::IR::Wrapper<IROp_Header>{Op, CreateNode(Op)};\n")
output_file.write("\t\treturn IRPair<IROp_Header>{Op, CreateNode(Op)};\n")
output_file.write("\t}\n\n")
output_file.write("\ttemplate<class T, IROps T2>\n")
output_file.write("\tT *AllocateOrphanOp() {\n")
output_file.write("\t\tsize_t Size = FEXCore::IR::GetSize(T2);\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(Data.Allocate(Size));\n")
output_file.write("\t\tmemset(Op, 0, Size);\n")
output_file.write("\t\tOp->Header.Op = T2;\n")
output_file.write("\t\treturn Op;\n")
output_file.write("\t}\n\n")
output_file.write("\ttemplate<class T, IROps T2>\n")
@@ -197,23 +225,23 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("\t\tauto Op = reinterpret_cast<T*>(Data.Allocate(Size));\n")
output_file.write("\t\tmemset(Op, 0, Size);\n")
output_file.write("\t\tOp->Header.Op = T2;\n")
output_file.write("\t\treturn FEXCore::IR::Wrapper<T>{Op, CreateNode(&Op->Header)};\n")
output_file.write("\t\treturn IRPair<T>{Op, CreateNode(&Op->Header)};\n")
output_file.write("\t}\n\n")
output_file.write("\tuint8_t GetOpSize(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = reinterpret_cast<IROp_Header const*>(Op->Header.Value.GetPtr(Data.Begin()));\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\tLogMan::Throw::A(HeaderOp->HasDest, \"Op %s has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\treturn HeaderOp->Size;\n")
output_file.write("\t}\n\n")
output_file.write("\tuint8_t GetOpElements(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = reinterpret_cast<IROp_Header const*>(Op->Header.Value.GetPtr(Data.Begin()));\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\tLogMan::Throw::A(HeaderOp->HasDest, \"Op %s has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\treturn HeaderOp->Elements;\n")
output_file.write("\t}\n\n")
output_file.write("\tbool OpHasDest(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = reinterpret_cast<IROp_Header const*>(Op->Header.Value.GetPtr(Data.Begin()));\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\treturn HeaderOp->HasDest;\n")
output_file.write("\t}\n\n")
Source/CMakeLists.txt
+2 -2
View File
@@ -54,7 +54,6 @@ set (SRCS
Interface/Core/CPUID.cpp
Interface/Core/Frontend.cpp
Interface/Core/OpcodeDispatcher.cpp
Interface/Core/RegisterAllocation.cpp
Interface/Core/X86Tables.cpp
Interface/Core/X86DebugInfo.cpp
Interface/Core/Interpreter/InterpreterCore.cpp
@@ -71,6 +70,7 @@ set (SRCS
Interface/IR/Passes/IRCompaction.cpp
Interface/IR/Passes/IRValidation.cpp
Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
Interface/IR/Passes/RegisterAllocationPass.cpp
Interface/IR/Passes/SyscallOptimization.cpp
)
@@ -108,7 +108,7 @@ add_custom_target(IR_INC
add_library(${PROJECT_NAME} STATIC ${SRCS})
add_dependencies(${PROJECT_NAME} IR_INC)
target_link_libraries(${PROJECT_NAME} LLVM pthread rt ${JIT_LIBS})
target_link_libraries(${PROJECT_NAME} LLVM pthread rt ${JIT_LIBS} dl)
target_include_directories(${PROJECT_NAME} PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
Source/Interface/Context/Context.h
+17 -1
View File
@@ -14,6 +14,13 @@
namespace FEXCore {
class SyscallHandler;
namespace CPU {
class JITCore;
}
}
namespace FEXCore::IR {
class RegisterAllocationPass;
}
namespace FEXCore::Context {
@@ -24,6 +31,8 @@ namespace FEXCore::Context {
struct Context {
friend class FEXCore::SyscallHandler;
friend class FEXCore::CPU::JITCore;
struct {
bool Multiblock {false};
bool BreakOnFrontendFailure {true};
@@ -78,6 +87,11 @@ namespace FEXCore::Context {
FEXCore::Core::ThreadState *GetThreadState();
void LoadEntryList();
uintptr_t CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
uintptr_t CompileFallbackBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
protected:
IR::RegisterAllocationPass *GetRegisterAllocatorPass();
private:
void WaitForIdle();
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID, uint64_t ChildTID);
@@ -89,7 +103,6 @@ namespace FEXCore::Context {
void ExecutionThread(FEXCore::Core::InternalThreadState *Thread);
void RunThread(FEXCore::Core::InternalThreadState *Thread);
uintptr_t CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
uintptr_t AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr);
FEXCore::CodeLoader *LocalLoader{};
@@ -99,5 +112,8 @@ namespace FEXCore::Context {
void AddThreadRIPsToEntryList(FEXCore::Core::InternalThreadState *Thread);
void SaveEntryList();
std::set<uint64_t> EntryList;
std::vector<uint64_t> InitLocations;
uint64_t StartingRIP;
IR::RegisterAllocationPass *RAPass {};
};
}
Source/Interface/Core/Core.cpp
+163 -116
View File
@@ -8,6 +8,7 @@
#include "Interface/Core/Interpreter/InterpreterCore.h"
#include "Interface/Core/JIT/JITCore.h"
#include "Interface/Core/LLVMJIT/LLVMCore.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/CodeLoader.h>
@@ -20,7 +21,7 @@
constexpr uint64_t STACK_OFFSET = 0xc000'0000;
constexpr uint64_t FS_OFFSET = 0xb000'0000;
constexpr uint64_t FS_SIZE = 0x1000;
constexpr uint64_t FS_SIZE = 0x1000'0000;
namespace FEXCore::CPU {
bool CreateCPUCore(FEXCore::Context::Context *CTX) {
@@ -212,7 +213,7 @@ namespace FEXCore::Context {
}
memset(NewThreadState.flags, 0, 32);
NewThreadState.gs = 0;
NewThreadState.fs = FS_OFFSET + FS_SIZE / 2;
NewThreadState.fs = FS_OFFSET;
NewThreadState.flags[1] = 1;
FEXCore::Core::InternalThreadState *Thread = CreateThread(&NewThreadState, 0, 0);
@@ -220,17 +221,11 @@ namespace FEXCore::Context {
// We are the parent thread
ParentThread = Thread;
auto MemLayout = Loader->GetLayout();
uint64_t BasePtr = AlignDown(std::get<0>(MemLayout), PAGE_SIZE);
uint64_t BaseSize = AlignUp(std::get<2>(MemLayout), PAGE_SIZE);
Thread->BlockCache->HintUsedRange(BasePtr, BaseSize);
uintptr_t BaseRegion = reinterpret_cast<uintptr_t>(MapRegion(Thread, BasePtr, BaseSize, true));
uintptr_t MemoryBase = MemoryMapper.GetBaseOffset<uintptr_t>(0);
auto MemoryMapperFunction = [&](uint64_t Base, uint64_t Size) -> void* {
return MapRegion(Thread, Base, Size);
Thread->BlockCache->HintUsedRange(Base, Base);
return MapRegion(Thread, Base, Size, true);
};
Loader->MapMemoryRegion(MemoryMapperFunction);
@@ -244,15 +239,19 @@ namespace FEXCore::Context {
// Now let the code loader setup memory
auto MemoryWriterFunction = [&](void const *Data, uint64_t Addr, uint64_t Size) -> void {
// Writes the machine code to be emulated in to memory
memcpy(reinterpret_cast<void*>(BaseRegion + Addr), Data, Size);
memcpy(reinterpret_cast<void*>(MemoryBase + Addr), Data, Size);
};
Loader->LoadMemory(MemoryWriterFunction);
Loader->GetInitLocations(&InitLocations);
// Set the RIP to what the code loader wants
Thread->State.State.rip = Loader->DefaultRIP();
auto TLSSlotWriter = [&](void const *Data, uint64_t Size) -> void {
memcpy(reinterpret_cast<void*>(MemoryBase + FS_OFFSET), Data, Size);
};
LogMan::Msg::D("Memory Base: 0x%016lx", MemoryMapper.GetBaseOffset<uint64_t>(0));
// Offset next thread's FS_OFFSET by slot size
uint64_t SlotSize = Loader->InitializeThreadSlot(TLSSlotWriter);
StartingRIP = Loader->DefaultRIP();
InitializeThread(Thread);
@@ -275,7 +274,7 @@ namespace FEXCore::Context {
if (AllPaused)
break;
PauseWait.WaitFor(std::chrono::seconds(1));
PauseWait.WaitFor(std::chrono::milliseconds(10));
} while (true);
}
@@ -325,10 +324,11 @@ namespace FEXCore::Context {
Thread->FallbackBackend->Initialize();
// Compile all of our cached entries
LogMan::Msg::D("Precompiling: %ld blocks", EntryList.size());
LogMan::Msg::D("Precompiling: %ld blocks...", EntryList.size());
for (auto Entry : EntryList) {
CompileRIP(Thread, Entry);
}
LogMan::Msg::D("Done", EntryList.size());
// This will create the execution thread but it won't actually start executing
Thread->ExecutionThread = std::thread(&Context::ExecutionThread, this, Thread);
@@ -379,6 +379,15 @@ namespace FEXCore::Context {
return Thread;
}
IR::RegisterAllocationPass *Context::GetRegisterAllocatorPass() {
if (!RAPass) {
RAPass = new IR::RegisterAllocationPass();
PassManager.InsertPass(RAPass);
}
return RAPass;
}
uintptr_t Context::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr) {
auto BlockMapPtr = Thread->BlockCache->AddBlockMapping(Address, Ptr);
if (BlockMapPtr == 0) {
@@ -406,7 +415,6 @@ namespace FEXCore::Context {
bool HadDispatchError {false};
[[maybe_unused]] bool HadRIPSetter {false};
Thread->OpDispatcher->BeginBlock();
if (!FrontendDecoder.DecodeInstructionsInBlock(&GuestCode[TotalInstructionsLength], GuestRIP + TotalInstructionsLength)) {
if (Config.BreakOnFrontendFailure) {
LogMan::Msg::E("Had Frontend decoder error");
@@ -415,6 +423,7 @@ namespace FEXCore::Context {
return 0;
}
Thread->OpDispatcher->BeginFunction(GuestRIP);
auto DecodedOps = FrontendDecoder.GetDecodedInsts();
for (size_t i = 0; i < DecodedOps.second; ++i) {
FEXCore::X86Tables::X86InstInfo const* TableInfo {nullptr};
@@ -515,15 +524,17 @@ namespace FEXCore::Context {
if (!Thread->OpDispatcher->Information.HadUnconditionalExit)
{
Thread->OpDispatcher->EndBlock(TotalInstructionsLength);
Thread->OpDispatcher->CreateNewEndBlock(TotalInstructionsLength);
Thread->OpDispatcher->CreateNewBeginBlock();
Thread->OpDispatcher->ExitFunction();
Thread->OpDispatcher->CreateNewEndBlock(0);
}
Thread->OpDispatcher->Finalize();
// Run the passmanager over the IR from the dispatcher
PassManager.Run(Thread->OpDispatcher.get());
if (Thread->OpDispatcher->ShouldDump)
// if (GuestRIP == 0x48b680)
// if (Thread->OpDispatcher->ShouldDump)
{
std::stringstream out;
auto NewIR = Thread->OpDispatcher->ViewIR();
@@ -531,17 +542,15 @@ namespace FEXCore::Context {
printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str());
}
// Do RA on the IR right now?
// Create a copy of the IR and place it in this thread's IR cache
auto IR = Thread->IRLists.try_emplace(GuestRIP, Thread->OpDispatcher->CreateIRCopy());
auto AddedIR = Thread->IRLists.try_emplace(GuestRIP, Thread->OpDispatcher->CreateIRCopy());
Thread->OpDispatcher->ResetWorkingList();
auto Debugit = Thread->DebugData.try_emplace(GuestRIP, FEXCore::Core::DebugData{});
auto Debugit = Thread->DebugData.try_emplace(GuestRIP);
Debugit.first->second.GuestCodeSize = TotalInstructionsLength;
Debugit.first->second.GuestInstructionCount = TotalInstructions;
IRList = IR.first->second.get();
IRList = AddedIR.first->second.get();
DebugData = &Debugit.first->second;
Thread->Stats.BlocksCompiled.fetch_add(1);
}
@@ -560,6 +569,20 @@ namespace FEXCore::Context {
return 0;
}
using BlockFn = void (*)(FEXCore::Core::InternalThreadState *Thread);
uintptr_t Context::CompileFallbackBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
// We have ONE more chance to try and fallback to the fallback CPU backend
// This will most likely fail since regular code use won't be using a fallback core.
// It's mainly for testing new instruction encodings
void *CodePtr = Thread->FallbackBackend->CompileCode(nullptr, nullptr);
if (CodePtr) {
uintptr_t Ptr = reinterpret_cast<uintptr_t >(AddBlockMapping(Thread, GuestRIP, CodePtr));
return Ptr;
}
return 0;
}
void Context::ExecutionThread(FEXCore::Core::InternalThreadState *Thread) {
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_WAITING;
@@ -579,119 +602,143 @@ namespace FEXCore::Context {
Thread->State.RunningEvents.Running = true;
Thread->State.RunningEvents.ShouldPause = false;
constexpr uint32_t CoreDebugLevel = 0;
while (!ShouldStop.load() && !Thread->State.RunningEvents.ShouldStop.load()) {
uint64_t GuestRIP = Thread->State.State.rip;
if (CoreDebugLevel >= 1) {
char const *Name = LocalLoader->FindSymbolNameInRange(GuestRIP);
LogMan::Msg::D(">>>>RIP: 0x%lx: '%s'", GuestRIP, Name ? Name : "<Unknown>");
}
bool Initializing = false;
using BlockFn = void (*)(FEXCore::Core::InternalThreadState *Thread);
uint64_t InitializationStep = 0;
if (Initializing) {
Thread->State.State.rip = ~0ULL;
}
else {
Thread->State.State.rip = StartingRIP;
}
if (!Thread->CPUBackend->NeedsOpDispatch()) {
BlockFn Ptr = reinterpret_cast<BlockFn>(Thread->CPUBackend->CompileCode(nullptr, nullptr));
Ptr(Thread);
}
else {
// Do have have this block compiled?
auto it = Thread->BlockCache->FindBlock(GuestRIP);
if (it == 0) {
// If not compile it
it = CompileBlock(Thread, GuestRIP);
if (Thread->CPUBackend->HasCustomDispatch()) {
Thread->CPUBackend->ExecuteCustomDispatch(&Thread->State);
}
else {
while (!ShouldStop.load() && !Thread->State.RunningEvents.ShouldStop.load()) {
if (Initializing) {
if (Thread->State.State.rip == ~0ULL) {
if (InitializationStep < InitLocations.size()) {
Thread->State.State.gregs[X86State::REG_RSP] -= 8;
*MemoryMapper.GetPointer<uint64_t*>(Thread->State.State.gregs[X86State::REG_RSP]) = ~0ULL;
LogMan::Msg::D("Going down init path: 0x%lx", InitLocations[InitializationStep]);
Thread->State.State.rip = InitLocations[InitializationStep++];
}
else {
Initializing = false;
Thread->State.State.rip = StartingRIP;
}
}
}
uint64_t GuestRIP = Thread->State.State.rip;
if (CoreDebugLevel >= 1) {
char const *Name = LocalLoader->FindSymbolNameInRange(GuestRIP);
LogMan::Msg::D(">>>>RIP: 0x%lx: '%s'", GuestRIP, Name ? Name : "<Unknown>");
}
// Did we successfully compile this block?
if (it != 0) {
// Block is compiled, run it
BlockFn Ptr = reinterpret_cast<BlockFn>(it);
if (!Thread->CPUBackend->NeedsOpDispatch()) {
BlockFn Ptr = reinterpret_cast<BlockFn>(Thread->CPUBackend->CompileCode(nullptr, nullptr));
Ptr(Thread);
}
else {
// We have ONE more chance to try and fallback to the fallback CPU backend
// This will most likely fail since regular code use won't be using a fallback core.
// It's mainly for testing new instruction encodings
void *CodePtr = Thread->FallbackBackend->CompileCode(nullptr, nullptr);
if (CodePtr) {
BlockFn Ptr = reinterpret_cast<BlockFn>(AddBlockMapping(Thread, GuestRIP, CodePtr));
Ptr(Thread);
// Do have have this block compiled?
auto it = Thread->BlockCache->FindBlock(GuestRIP);
if (it == 0) {
// If not compile it
it = CompileBlock(Thread, GuestRIP);
}
// Did we successfully compile this block?
if (it != 0) {
// Block is compiled, run it
BlockFn Ptr = reinterpret_cast<BlockFn>(it);
Ptr(Thread);
}
else {
// Let the frontend know that something has happened that is unhandled
Thread->State.RunningEvents.ShouldPause = true;
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_UNKNOWNERROR;
// We have ONE more chance to try and fallback to the fallback CPU backend
// This will most likely fail since regular code use won't be using a fallback core.
// It's mainly for testing new instruction encodings
uintptr_t CodePtr = CompileFallbackBlock(Thread, GuestRIP);
if (CodePtr) {
BlockFn Ptr = reinterpret_cast<BlockFn>(CodePtr);
Ptr(Thread);
}
else {
// Let the frontend know that something has happened that is unhandled
Thread->State.RunningEvents.ShouldPause = true;
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_UNKNOWNERROR;
}
}
}
}
// if (GuestRIP == 0x48c8dd) {
// fflush(stdout);
// __builtin_trap();
// }
if (CoreDebugLevel >= 2) {
int i = 0;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
i += 4;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
i += 4;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
i += 4;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
uint64_t PackedFlags{};
for (unsigned i = 0; i < 32; ++i) {
PackedFlags |= static_cast<uint64_t>(Thread->State.State.flags[i]) << i;
if (CoreDebugLevel >= 2) {
int i = 0;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
i += 4;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
i += 4;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
i += 4;
LogMan::Msg::D("\tGPR[%d]: %016lx %016lx %016lx %016lx", i, Thread->State.State.gregs[i + 0], Thread->State.State.gregs[i + 1], Thread->State.State.gregs[i + 2], Thread->State.State.gregs[i + 3]);
uint64_t PackedFlags{};
for (i = 0; i < 32; ++i) {
PackedFlags |= static_cast<uint64_t>(Thread->State.State.flags[i]) << i;
}
LogMan::Msg::D("\tFlags: %016lx", PackedFlags);
}
LogMan::Msg::D("\tFlags: %016lx", PackedFlags);
}
if (CoreDebugLevel >= 3) {
int i = 0;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
if (CoreDebugLevel >= 3) {
int i = 0;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
i += 4;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
i += 4;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
i += 4;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
uint64_t PackedFlags{};
for (unsigned i = 0; i < 32; ++i) {
PackedFlags |= static_cast<uint64_t>(Thread->State.State.flags[i]) << i;
i += 4;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
i += 4;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
i += 4;
LogMan::Msg::D("\tXMM[%d][0]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][0], Thread->State.State.xmm[i + 1][0], Thread->State.State.xmm[i + 2][0], Thread->State.State.xmm[i + 3][0]);
LogMan::Msg::D("\tXMM[%d][1]: %016lx %016lx %016lx %016lx", i, Thread->State.State.xmm[i + 0][1], Thread->State.State.xmm[i + 1][1], Thread->State.State.xmm[i + 2][1], Thread->State.State.xmm[i + 3][1]);
uint64_t PackedFlags{};
for (i = 0; i < 32; ++i) {
PackedFlags |= static_cast<uint64_t>(Thread->State.State.flags[i]) << i;
}
LogMan::Msg::D("\tFlags: %016lx", PackedFlags);
}
LogMan::Msg::D("\tFlags: %016lx", PackedFlags);
}
if (Thread->State.RunningEvents.ShouldStop.load()) {
// If it is the parent thread that died then just leave
// XXX: This doesn't make sense when the parent thread doesn't outlive its children
if (Thread->State.ThreadManager.GetTID() == 1) {
ShouldStop = true;
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_SHUTDOWN;
if (Thread->State.RunningEvents.ShouldStop.load()) {
// If it is the parent thread that died then just leave
// XXX: This doesn't make sense when the parent thread doesn't outlive its children
if (Thread->State.ThreadManager.GetTID() == 1) {
ShouldStop = true;
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_SHUTDOWN;
}
break;
}
break;
}
if (RunningMode == FEXCore::Context::CoreRunningMode::MODE_SINGLESTEP || Thread->State.RunningEvents.ShouldPause) {
Thread->State.RunningEvents.Running = false;
Thread->State.RunningEvents.WaitingToStart = false;
if (RunningMode == FEXCore::Context::CoreRunningMode::MODE_SINGLESTEP || Thread->State.RunningEvents.ShouldPause) {
Thread->State.RunningEvents.Running = false;
Thread->State.RunningEvents.WaitingToStart = false;
// If something previously hasn't set the exit state then set it now
if (Thread->ExitReason == FEXCore::Context::ExitReason::EXIT_NONE)
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_DEBUG;
// If something previously hasn't set the exit state then set it now
if (Thread->ExitReason == FEXCore::Context::ExitReason::EXIT_NONE)
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_DEBUG;
PauseWait.NotifyAll();
Thread->StartRunning.Wait();
PauseWait.NotifyAll();
Thread->StartRunning.Wait();
// If we set it to debug then set it back to none after this
// We want to retain the state if the frontend decides to leave
if (Thread->ExitReason == FEXCore::Context::ExitReason::EXIT_DEBUG)
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_NONE;
// If we set it to debug then set it back to none after this
// We want to retain the state if the frontend decides to leave
if (Thread->ExitReason == FEXCore::Context::ExitReason::EXIT_DEBUG)
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_NONE;
Thread->State.RunningEvents.Running = true;
Thread->State.RunningEvents.Running = true;
}
}
}
@@ -731,7 +778,7 @@ namespace FEXCore::Context {
return MemoryMapper.GetMemoryBase();
}
void Context::CopyMemoryMapping([[maybe_unused]] FEXCore::Core::InternalThreadState *ParentThread, FEXCore::Core::InternalThreadState *ChildThread) {
void Context::CopyMemoryMapping([[maybe_unused]] FEXCore::Core::InternalThreadState*, FEXCore::Core::InternalThreadState *ChildThread) {
auto Regions = MemoryMapper.MappedRegions;
for (auto const& Region : Regions) {
ChildThread->CPUBackend->MapRegion(Region.Ptr, Region.Offset, Region.Size);
Source/Interface/Core/Interpreter/InterpreterCore.cpp
+21 -10
View File
@@ -39,10 +39,10 @@ private:
void *AllocateTmpSpace(size_t Size);
template<typename Res>
Res GetDest(IR::NodeWrapper Op);
Res GetDest(IR::OrderedNodeWrapper Op);
template<typename Res>
Res GetSrc(IR::NodeWrapper Src);
Res GetSrc(IR::OrderedNodeWrapper Src);
std::vector<uint8_t> TmpSpace;
DestMapType DestMap;
@@ -86,13 +86,13 @@ void *InterpreterCore::AllocateTmpSpace(size_t Size) {
}
template<typename Res>
Res InterpreterCore::GetDest(IR::NodeWrapper Op) {
Res InterpreterCore::GetDest(IR::OrderedNodeWrapper Op) {
auto DstPtr = DestMap[Op.NodeOffset];
return reinterpret_cast<Res>(DstPtr);
}
template<typename Res>
Res InterpreterCore::GetSrc(IR::NodeWrapper Src) {
Res InterpreterCore::GetSrc(IR::OrderedNodeWrapper Src) {
#if DESTMAP_AS_MAP
LogMan::Throw::A(DestMap.find(Src.NodeOffset) != DestMap.end(), "Op had source but it wasn't in the destination map");
#endif
@@ -138,8 +138,8 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
using namespace FEXCore::IR;
using namespace FEXCore::IR;
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
OrderedNodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
uint8_t OpSize = IROp->Size;
@@ -287,15 +287,16 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
case IR::OP_STOREMEM: {
#define STORE_DATA(x, y) \
case x: { \
y *Data = Thread->CTX->MemoryMapper.GetBaseOffset<y *>(*GetSrc<uint64_t*>(Op->Header.Args[0])); \
uint64_t SrcPtr = *GetSrc<uint64_t*>(Op->Header.Args[0]); \
y *Data = Thread->CTX->MemoryMapper.GetPointer<y *>(SrcPtr); \
LogMan::Throw::A(Data != nullptr, "Couldn't Map pointer to 0x%lx for size %d store\n", *GetSrc<uint64_t*>(Op->Header.Args[0]), x);\
*Data = *GetSrc<y*>(Op->Header.Args[1]); \
memcpy(Data, GetSrc<y*>(Op->Header.Args[1]), sizeof(y)); \
} \
break
auto Op = IROp->C<IR::IROp_StoreMem>();
//LogMan::Msg::D("Storing guestmem: 0x%lx (%d)", *GetSrc<uint64_t*>(Op->Header.Args[0]), Op->Size);
//LogMan::Msg::D("\tStoring: 0x%016lx", (uint64_t)*GetSrc<uint64_t*>(Op->Header.Args[1]));
// LogMan::Msg::D("Storing guestmem: 0x%lx (%d)", *GetSrc<uint64_t*>(Op->Header.Args[0]), Op->Size);
// LogMan::Msg::D("\tStoring: 0x%016lx", (uint64_t)*GetSrc<uint64_t*>(Op->Header.Args[1]));
switch (Op->Size) {
STORE_DATA(1, uint8_t);
@@ -1449,6 +1450,16 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
default: LogMan::Msg::A("Unknown LREM Size: %d", Size); break;
}
break;
}
case IR::OP_VEXTR: {
auto Op = IROp->C<IR::IROp_VExtr>();
__uint128_t Src1 = *GetSrc<__uint128_t*>(Op->Header.Args[0]);
__uint128_t Src2 = *GetSrc<__uint128_t*>(Op->Header.Args[1]);
uint8_t Offset = Op->Index * 8;
__uint128_t Dst = (Src1 << (sizeof(__uint128_t) - Offset)) | (Src2 >> Offset);
memcpy(GDP, &Dst, 16);
break;
}
default:
Source/Interface/Core/JIT/Arm64/JIT.cpp
+347 -194
View File
@@ -1,10 +1,12 @@
#include "Interface/Context/Context.h"
#include "Interface/Core/RegisterAllocation.h"
#include "Interface/Core/BlockCache.h"
#include "Interface/Core/InternalThreadState.h"
#include "Interface/Core/JIT/x86_64/JIT.h"
#include "Interface/HLE/Syscalls.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#if _M_X86_64
#define VIXL_INCLUDE_SIMULATOR_AARCH64
#include "aarch64/simulator-aarch64.h"
@@ -22,17 +24,17 @@
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
#define STATE x0
#define MEM_BASE x1
#define TMP1 x2
#define TMP2 x3
#define MEM_BASE x28
#define STATE x27
#define TMP1 x1
#define TMP2 x2
#define VTMP1 v1
#define VTMP2 v2
#define VTMP3 v3
static uint64_t SyscallThunk(FEXCore::Core::InternalThreadState *Thread, FEXCore::SyscallHandler *Handler, FEXCore::HLE::SyscallArguments *Args)
{
static uint64_t SyscallThunk(FEXCore::SyscallHandler *Handler, FEXCore::Core::InternalThreadState *Thread, FEXCore::HLE::SyscallArguments *Args) {
return Handler->HandleSyscall(Thread, Args);
}
@@ -41,6 +43,16 @@ static void CPUIDThunk(FEXCore::CPUIDEmu *CPUID, uint64_t Function, FEXCore::CPU
memcpy(Results, &Res, sizeof(FEXCore::CPUIDEmu::FunctionResults));
}
static uint64_t CompileBlockThunk(FEXCore::Context::Context* CTX, FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) {
uint64_t Result = CTX->CompileBlock(Thread, RIP);
return Result;
}
static uint64_t CompileFallbackBlockThunk(FEXCore::Context::Context* CTX, FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) {
uint64_t Result = CTX->CompileFallbackBlock(Thread, RIP);
return Result;
}
// XXX: Switch from MacroAssembler to Assembler once we drop the simulator
class JITCore final : public CPUBackend, public vixl::aarch64::MacroAssembler {
public:
@@ -57,12 +69,22 @@ public:
void SimulationExecution(FEXCore::Core::InternalThreadState *Thread);
#endif
bool HasCustomDispatch() const override { return CustomDispatchGenerated; }
#if _M_X86_64
void ExecuteCustomDispatch(FEXCore::Core::ThreadState *Thread) override;
#else
void ExecuteCustomDispatch(FEXCore::Core::ThreadState *Thread) override {
DispatchPtr(reinterpret_cast<FEXCore::Core::InternalThreadState*>(Thread->InternalState));
}
#endif
private:
FEXCore::Context::Context *CTX;
FEXCore::Core::InternalThreadState *State;
FEXCore::IR::IRListView<true> const *CurrentIR;
std::unordered_map<IR::NodeWrapper::NodeOffsetType, aarch64::Label*> JumpTargets;
std::map<IR::OrderedNodeWrapper::NodeOffsetType, aarch64::Label> JumpTargets;
/**
* @name Register Allocation
@@ -73,21 +95,19 @@ private:
constexpr static uint32_t RegisterClasses = 2;
constexpr static uint32_t GPRBase = 0;
constexpr static uint32_t GPRClass = 0;
constexpr static uint32_t GPRClass = IR::RegisterAllocationPass::GPRClass;
constexpr static uint32_t FPRBase = NumGPRs;
constexpr static uint32_t FPRClass = 1;
constexpr static uint32_t FPRClass = IR::RegisterAllocationPass::FPRClass;
RA::RegisterSet *RASet;
IR::RegisterAllocationPass::RegisterSet *RASet;
/** @} */
void FindNodeClasses();
bool CalculateLiveRange(uint32_t Nodes);
constexpr static uint8_t RA_32 = 0;
constexpr static uint8_t RA_64 = 1;
constexpr static uint8_t RA_FPR = 2;
bool HasRA = false;
RA::RegisterGraph *Graph;
IR::RegisterAllocationPass::RegisterGraph *Graph;
uint32_t GetPhys(uint32_t Node);
template<uint8_t RAType>
@@ -118,9 +138,27 @@ private:
std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> HostToGuest;
#endif
void LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant);
void CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread);
bool CustomDispatchGenerated {false};
using CustomDispatch = void(*)(FEXCore::Core::InternalThreadState *Thread);
CustomDispatch DispatchPtr{};
IR::RegisterAllocationPass *RAPass;
#if _M_X86_64
uint64_t CustomDispatchEnd;
#endif
};
#if _M_X86_64
void JITCore::ExecuteCustomDispatch(FEXCore::Core::ThreadState *Thread) {
PrintDisassembler PrintDisasm(stdout);
PrintDisasm.DisassembleBuffer(vixl::aarch64::Instruction::Cast(DispatchPtr), vixl::aarch64::Instruction::Cast(CustomDispatchEnd));
Sim.WriteXRegister(0, reinterpret_cast<uint64_t>(Thread));
Sim.RunFrom(vixl::aarch64::Instruction::Cast(DispatchPtr));
}
static void SimulatorExecution(FEXCore::Core::InternalThreadState *Thread) {
JITCore *Core = reinterpret_cast<JITCore*>(Thread->CPUBackend.get());
Core->SimulationExecution(Thread);
@@ -129,8 +167,8 @@ static void SimulatorExecution(FEXCore::Core::InternalThreadState *Thread) {
void JITCore::SimulationExecution(FEXCore::Core::InternalThreadState *Thread) {
using namespace vixl::aarch64;
auto SimulatorAddress = HostToGuest[Thread->State.State.rip];
//PrintDisassembler PrintDisasm(stdout);
//PrintDisasm.DisassembleBuffer(vixl::aarch64::Instruction::Cast(SimulatorAddress.first), vixl::aarch64::Instruction::Cast(SimulatorAddress.second));
// PrintDisassembler PrintDisasm(stdout);
// PrintDisasm.DisassembleBuffer(vixl::aarch64::Instruction::Cast(SimulatorAddress.first), vixl::aarch64::Instruction::Cast(SimulatorAddress.second));
Sim.WriteXRegister(0, reinterpret_cast<uint64_t>(Thread));
Sim.RunFrom(vixl::aarch64::Instruction::Cast(SimulatorAddress.first));
@@ -149,11 +187,14 @@ JITCore::JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadSt
// XXX: Set this to a real minimum feature set in the future
SetCPUFeatures(vixl::CPUFeatures::All());
RASet = RA::AllocateRegisterSet(RegisterCount, RegisterClasses);
RA::AddRegisters(RASet, GPRClass, GPRBase, NumGPRs);
RA::AddRegisters(RASet, FPRClass, FPRBase, NumFPRs);
RAPass = CTX->GetRegisterAllocatorPass();
RAPass->SetSupportsSpills(false);
Graph = RA::AllocateRegisterGraph(RASet, 9000);
RASet = RAPass->AllocateRegisterSet(RegisterCount, RegisterClasses);
RAPass->AddRegisters(RASet, GPRClass, GPRBase, NumGPRs);
RAPass->AddRegisters(RASet, FPRClass, FPRBase, NumFPRs);
Graph = RAPass->AllocateRegisterGraph(RASet, 9000);
LiveRanges.resize(9000);
// Just set the entire range as executable
@@ -165,11 +206,13 @@ JITCore::JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadSt
#if _M_X86_64
Sim.SetCPUFeatures(vixl::CPUFeatures::All());
#endif
SetAllowAssembler(true);
CreateCustomDispatch(Thread);
}
JITCore::~JITCore() {
FreeRegisterGraph(Graph);
FreeRegisterSet(RASet);
RAPass->FreeRegisterGraph();
RAPass->FreeRegisterSet(RASet);
}
void JITCore::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant) {
@@ -202,7 +245,7 @@ const std::array<aarch64::VRegister, 22> RAFPR = {
v29, v30, v31};
uint32_t JITCore::GetPhys(uint32_t Node) {
uint32_t Reg = RA::GetNodeRegister(Graph, Node);
uint32_t Reg = RAPass->GetNodeRegister(Node);
if (Reg < FPRBase)
return Reg;
@@ -242,114 +285,6 @@ aarch64::VRegister JITCore::GetDst(uint32_t Node) {
return RAFPR[Reg];
}
void JITCore::FindNodeClasses() {
uintptr_t ListBegin = CurrentIR->GetListData();
uintptr_t DataBegin = CurrentIR->GetData();
IR::NodeWrapperIterator Begin = CurrentIR->begin();
IR::NodeWrapperIterator End = CurrentIR->end();
while (Begin != End) {
using namespace FEXCore::IR;
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
if (IROp->HasDest) {
// XXX: This needs to be better
switch (IROp->Op) {
case OP_LOADCONTEXT: {
auto Op = IROp->C<IR::IROp_LoadContext>();
if (Op->Size == 16)
RA::SetNodeClass(Graph, WrapperOp->ID(), FPRClass);
else
RA::SetNodeClass(Graph, WrapperOp->ID(), GPRClass);
break;
}
case IR::OP_LOADMEM: {
auto Op = IROp->C<IR::IROp_LoadMem>();
if (Op->Size == 16)
RA::SetNodeClass(Graph, WrapperOp->ID(), FPRClass);
else
RA::SetNodeClass(Graph, WrapperOp->ID(), GPRClass);
break;
}
case OP_ZEXT: {
auto Op = IROp->C<IR::IROp_Zext>();
LogMan::Throw::A(Op->SrcSize <= 64, "Can't support Zext of size: %ld", Op->SrcSize);
if (Op->SrcSize == 64) {
RA::SetNodeClass(Graph, WrapperOp->ID(), FPRClass);
}
else {
RA::SetNodeClass(Graph, WrapperOp->ID(), GPRClass);
}
break;
}
case OP_CPUID: RA::SetNodeClass(Graph, WrapperOp->ID(), FPRClass); break;
default:
if (IROp->Op >= IR::OP_VOR)
RA::SetNodeClass(Graph, WrapperOp->ID(), FPRClass);
else
RA::SetNodeClass(Graph, WrapperOp->ID(), GPRClass);
break;
}
}
++Begin;
}
}
bool JITCore::CalculateLiveRange(uint32_t Nodes) {
if (Nodes > LiveRanges.size()) {
LiveRanges.resize(Nodes);
}
memset(&LiveRanges.at(0), 0xFF, Nodes * sizeof(LiveRange));
uintptr_t ListBegin = CurrentIR->GetListData();
uintptr_t DataBegin = CurrentIR->GetData();
IR::NodeWrapperIterator Begin = CurrentIR->begin();
IR::NodeWrapperIterator End = CurrentIR->end();
while (Begin != End) {
using namespace FEXCore::IR;
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
uint32_t Node = WrapperOp->ID();
// If the destination hasn't yet been set then set it now
if (IROp->HasDest && LiveRanges[Node].Begin == ~0U) {
LiveRanges[Node].Begin = Node;
// Default to ending right where it starts
LiveRanges[Node].End = Node;
}
for (uint8_t i = 0; i < IROp->NumArgs; ++i) {
uint32_t ArgNode = IROp->Args[i].ID();
// Set the node end to be at least here
LiveRanges[ArgNode].End = Node;
}
++Begin;
}
// Now that we have all the live ranges calculated we need to add them to our interference graph
for (uint32_t i = 0; i < Nodes; ++i) {
for (uint32_t j = i + 1; j < Nodes; ++j) {
if (!(LiveRanges[i].Begin >= LiveRanges[j].End ||
LiveRanges[j].Begin >= LiveRanges[i].End)) {
RA::AddNodeInterference(Graph, i, j);
}
}
}
return RA::AllocateRegisters(Graph);
}
void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData) {
using namespace aarch64;
JumpTargets.clear();
@@ -362,11 +297,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
IR::NodeWrapperIterator End = CurrentIR->end();
uintptr_t ListSize = CurrentIR->GetListSize();
uint32_t SSACount = ListSize / sizeof(IR::OrderedNode);
ResetRegisterGraph(Graph, SSACount);
FindNodeClasses();
HasRA = CalculateLiveRange(SSACount);
HasRA = RAPass->HasFullRA();
LogMan::Throw::A(HasRA, "Arm64 JIT only works with RA");
@@ -393,14 +324,17 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
auto Buffer = GetBuffer();
auto Entry = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
void *Memory = CTX->MemoryMapper.GetMemoryBase();
LoadConstant(MEM_BASE, (uint64_t)Memory);
if (!CustomDispatchGenerated) {
void *Memory = CTX->MemoryMapper.GetMemoryBase();
LoadConstant(MEM_BASE, (uint64_t)Memory);
mov(STATE, x0);
}
while (Begin != End) {
using namespace FEXCore::IR;
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
OrderedNodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
uint8_t OpSize = IROp->Size;
uint32_t Node = WrapperOp->ID();
@@ -411,7 +345,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
auto Name = FEXCore::IR::GetName(IROp->Op);
if (IROp->HasDest) {
uint32_t PhysReg = RA::GetNodeRegister(Graph, Node);
uint32_t PhysReg = RAPass->GetNodeRegister(Node);
if (PhysReg >= FPRBase)
Inst << "\tFPR" << GetPhys(Node) << " = " << Name << " ";
else
@@ -423,14 +357,12 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
for (uint8_t i = 0; i < IROp->NumArgs; ++i) {
uint32_t ArgNode = IROp->Args[i].ID();
uint32_t PhysReg = RA::GetNodeRegister(Graph, ArgNode);
uint32_t PhysReg = RAPass->GetNodeRegister(ArgNode);
if (PhysReg >= FPRBase)
Inst << "FPR" << GetPhys(ArgNode) << (i + 1 == IROp->NumArgs ? "" : ", ");
else
Inst << "Reg" << GetPhys(ArgNode) << (i + 1 == IROp->NumArgs ? "" : ", ");
}
LogMan::Msg::D("%s", Inst.str().c_str());
}
}
@@ -439,10 +371,10 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
auto IsTarget = JumpTargets.find(WrapperOp->ID());
if (IsTarget == JumpTargets.end()) {
// XXX: This is a memory leak
JumpTargets.try_emplace(WrapperOp->ID(), new aarch64::Label);
JumpTargets.try_emplace(WrapperOp->ID());
}
else {
bind(IsTarget->second);
bind(&IsTarget->second);
}
break;
}
@@ -467,7 +399,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
// X1: ThreadState
// X2: Pointer to SyscallArguments
uint64_t SPOffset = AlignUp((2 + RA64.size() + 7 + 2) * 8, 16);
uint64_t SPOffset = AlignUp((RA64.size() + 7 + 1) * 8, 16);
sub(sp, sp, SPOffset);
for (uint32_t i = 0; i < 7; ++i)
@@ -478,14 +410,26 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
str(RA, MemOperand(sp, 7 * 8 + i * 8));
i++;
}
str(STATE, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 1 * 8));
str(MEM_BASE, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 2 * 8));
str(lr, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 3 * 8));
str(lr, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 0 * 8));
// x0 = threadstate already
LoadConstant(x1, reinterpret_cast<uint64_t>(&CTX->SyscallHandler));
mov (x2, sp);
LoadConstant(x0, reinterpret_cast<uint64_t>(&CTX->SyscallHandler));
mov(x1, STATE);
mov(x2, sp);
#if _M_X86_64
CallRuntime(SyscallThunk);
#else
using ClassPtrType = uint64_t (FEXCore::SyscallHandler::*)(FEXCore::Core::InternalThreadState *, FEXCore::HLE::SyscallArguments *);
union PtrCast {
ClassPtrType ClassPtr;
uintptr_t Data;
};
PtrCast Ptr;
Ptr.ClassPtr = &FEXCore::SyscallHandler::HandleSyscall;
LoadConstant(x3, Ptr.Data);
blr(x3);
#endif
// Result is now in x0
// Fix the stack and any values that were stepped on
@@ -498,9 +442,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
// Move result to its destination register
mov(GetDst<RA_64>(Node), x0);
ldr(STATE, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 1 * 8));
ldr(MEM_BASE, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 2 * 8));
ldr(lr, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 3 * 8));
ldr(lr, MemOperand(sp, 7 * 8 + RA64.size() * 8 + 0 * 8));
add(sp, sp, SPOffset);
break;
@@ -517,9 +459,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
i++;
}
str(STATE, MemOperand(sp, RA64.size() * 8 + 0 * 8));
str(MEM_BASE, MemOperand(sp, RA64.size() * 8 + 1 * 8));
str(lr, MemOperand(sp, RA64.size() * 8 + 2 * 8));
str(lr, MemOperand(sp, RA64.size() * 8 + 0 * 8));
// x0 = CPUID Handler
// x1 = CPUID Function
@@ -540,9 +480,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
auto Dst = GetDst(Node);
ldr(Dst, MemOperand(sp, RA64.size() * 8 + 3 * 8));
ldr(STATE, MemOperand(sp, RA64.size() * 8 + 0 * 8));
ldr(MEM_BASE, MemOperand(sp, RA64.size() * 8 + 1 * 8));
ldr(lr, MemOperand(sp, RA64.size() * 8 + 2 * 8));
ldr(lr, MemOperand(sp, RA64.size() * 8 + 0 * 8));
add(sp, sp, SPOffset);
@@ -551,7 +489,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
case IR::OP_EXTRACTELEMENT: {
auto Op = IROp->C<IR::IROp_ExtractElement>();
uint32_t PhysReg = RA::GetNodeRegister(Graph, Op->Header.Args[0].ID());
uint32_t PhysReg = RAPass->GetNodeRegister(Op->Header.Args[0].ID());
if (PhysReg >= FPRBase) {
switch (OpSize) {
case 4:
@@ -574,16 +512,15 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
Label *TargetLabel;
auto IsTarget = JumpTargets.find(Op->Header.Args[0].ID());
if (IsTarget == JumpTargets.end()) {
TargetLabel = JumpTargets.try_emplace(Op->Header.Args[0].ID(), new aarch64::Label).first->second;
TargetLabel = &JumpTargets.try_emplace(Op->Header.Args[0].ID()).first->second;
}
else {
TargetLabel = IsTarget->second;
TargetLabel = &IsTarget->second;
}
b(TargetLabel);
break;
}
case IR::OP_CONDJUMP: {
auto Op = IROp->C<IR::IROp_CondJump>();
@@ -591,10 +528,10 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
auto IsTarget = JumpTargets.find(Op->Header.Args[1].ID());
if (IsTarget == JumpTargets.end()) {
// XXX: This is a memory leak
TargetLabel = JumpTargets.try_emplace(Op->Header.Args[1].ID(), new aarch64::Label).first->second;
TargetLabel = &JumpTargets.try_emplace(Op->Header.Args[1].ID()).first->second;
}
else {
TargetLabel = IsTarget->second;
TargetLabel = &IsTarget->second;
}
cbnz(GetSrc<RA_64>(Op->Header.Args[0].ID()), TargetLabel);
@@ -709,7 +646,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
lsrv(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()), GetSrc<RA_64>(Op->Header.Args[1].ID()));
else
lsrv(GetDst<RA_32>(Node), GetSrc<RA_32>(Op->Header.Args[0].ID()), GetSrc<RA_32>(Op->Header.Args[1].ID()));
break;
break;
}
case IR::OP_ASHR: {
auto Op = IROp->C<IR::IROp_Ashr>();
@@ -717,7 +654,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
asrv(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()), GetSrc<RA_64>(Op->Header.Args[1].ID()));
else
asrv(GetDst<RA_32>(Node), GetSrc<RA_32>(Op->Header.Args[0].ID()), GetSrc<RA_32>(Op->Header.Args[1].ID()));
break;
break;
}
case IR::OP_LSHL: {
auto Op = IROp->C<IR::IROp_Lshl>();
@@ -725,7 +662,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
lslv(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()), GetSrc<RA_64>(Op->Header.Args[1].ID()));
else
lslv(GetDst<RA_32>(Node), GetSrc<RA_32>(Op->Header.Args[0].ID()), GetSrc<RA_32>(Op->Header.Args[1].ID()));
break;
break;
}
case IR::OP_ROR: {
auto Op = IROp->C<IR::IROp_Ror>();
@@ -745,7 +682,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_ROL: {
auto Op = IROp->C<IR::IROp_Rol>();
uint8_t Mask = OpSize * 8 - 1;
@@ -768,7 +704,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_SEXT: {
auto Op = IROp->C<IR::IROp_Sext>();
LogMan::Throw::A(Op->SrcSize <= 64, "Can't support Zext of size: %ld", Op->SrcSize);
@@ -794,7 +729,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
case IR::OP_ZEXT: {
auto Op = IROp->C<IR::IROp_Zext>();
LogMan::Throw::A(Op->SrcSize <= 64, "Can't support Zext of size: %ld", Op->SrcSize);
uint32_t PhysReg = RA::GetNodeRegister(Graph, Op->Header.Args[0].ID());
uint32_t PhysReg = RAPass->GetNodeRegister(Op->Header.Args[0].ID());
if (PhysReg >= FPRBase) {
// FPR -> GPR transfer with free truncation
switch (Op->SrcSize) {
@@ -886,19 +821,20 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_BFE: {
auto Op = IROp->C<IR::IROp_Bfe>();
LogMan::Throw::A(OpSize <= 16, "OpSize is too large for BFE: %d", OpSize);
LogMan::Throw::A(Op->Width != 0, "Invalid BFE width of 0");
auto Dst = GetDst<RA_64>(Node);
if (OpSize == 16) {
LogMan::Throw::A(!(Op->lsb < 64 && (Op->lsb + Op->Width > 64)), "Trying to BFE an XMM across the 64bit split: Beginning at %d, ending at %d", Op->lsb, Op->lsb + Op->Width);
uint8_t Offset = Op->lsb;
if (Offset < 64) {
mov(Dst, GetSrc(Op->Header.Args[0].ID()), 0);
mov(Dst, GetSrc(Op->Header.Args[0].ID()).D(), 0);
}
else {
mov(Dst, GetSrc(Op->Header.Args[0].ID()), 1);
mov(Dst, GetSrc(Op->Header.Args[0].ID()).D(), 1);
Offset -= 64;
}
@@ -912,18 +848,20 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
else {
lsr(Dst, GetSrc<RA_64>(Op->Header.Args[0].ID()), Op->lsb);
and_(Dst, Dst, ((1ULL << Op->Width) - 1));
if (Op->Width != 64) {
and_(Dst, Dst, ((1ULL << Op->Width) - 1));
}
}
break;
}
case IR::OP_POPCOUNT: {
auto Op = IROp->C<IR::IROp_Popcount>();
auto Dst = GetDst<RA_64>(Node);
fmov(VTMP1, GetSrc<RA_64>(Op->Header.Args[0].ID()));
fmov(VTMP1.V1D(), GetSrc<RA_64>(Op->Header.Args[0].ID()));
cnt(VTMP1.V8B(), VTMP1.V8B());
addv(VTMP1.B(), VTMP1.V8B());
umov(Dst, VTMP1.B(), 0);
break;
umov(Dst.W(), VTMP1.B(), 0);
break;
}
case IR::OP_FINDLSB: {
auto Op = IROp->C<IR::IROp_FindLSB>();
@@ -980,7 +918,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
mov(GetDst<RA_64>(Node), TMP2);
break;
}
case IR::OP_SELECT: {
auto Op = IROp->C<IR::IROp_Select>();
@@ -1162,7 +1099,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_LUREM: {
auto Op = IROp->C<IR::IROp_LURem>();
// Each source is OpSize in size
@@ -1219,7 +1155,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VINSELEMENT: {
auto Op = IROp->C<IR::IROp_VInsElement>();
mov(VTMP1, GetSrc(Op->Header.Args[0].ID()));
@@ -1397,17 +1332,28 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
eor(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case IR::OP_VEXTR: {
auto Op = IROp->C<IR::IROp_VExtr>();
// AArch64 ext op has bit arrangement as [Vm:Vn] so arguments need to be swapped
ext(GetDst(Node).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), Op->Index);
break;
}
case IR::OP_VUSHLS: {
auto Op = IROp->C<IR::IROp_VUShlS>();
switch (Op->ElementSize) {
case 1: {
dup(VTMP1.V16B(), GetSrc<RA_32>(Op->Header.Args[1].ID()));
ushl(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), VTMP1.V16B());
break;
}
case 2: {
dup(VTMP1.V8H(), GetSrc<RA_64>(Op->Header.Args[1].ID()));
dup(VTMP1.V8H(), GetSrc<RA_32>(Op->Header.Args[1].ID()));
ushl(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), VTMP1.V8H());
break;
}
case 4: {
dup(VTMP1.V4S(), GetSrc<RA_64>(Op->Header.Args[1].ID()));
dup(VTMP1.V4S(), GetSrc<RA_32>(Op->Header.Args[1].ID()));
ushl(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), VTMP1.V4S());
break;
}
@@ -1420,11 +1366,58 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VUMIN: {
auto Op = IROp->C<IR::IROp_VUMin>();
switch (Op->ElementSize) {
case 1: {
umin(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
umin(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
umin(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
umin(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VSMIN: {
auto Op = IROp->C<IR::IROp_VSMin>();
switch (Op->ElementSize) {
case 1: {
smin(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
smin(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
smin(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
smin(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_CYCLECOUNTER: {
if (0)
#ifdef DEBUG_CYCLES
movz(GetDst<RA_64>(Node), 0);
else
#else
mrs(GetDst<RA_64>(Node), CNTVCT_EL0);
#endif
break;
}
default:
@@ -1437,14 +1430,174 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
FinalizeCode();
#if _M_X86_64
auto CodeEnd = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
HostToGuest[State->State.State.rip] = std::make_pair(Entry, CodeEnd);
return (void*)SimulatorExecution;
#else
return reinterpret_cast<void*>(Entry);
if (!CustomDispatchGenerated) {
auto CodeEnd = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
HostToGuest[State->State.State.rip] = std::make_pair(Entry, CodeEnd);
return (void*)SimulatorExecution;
}
#endif
return reinterpret_cast<void*>(Entry);
}
void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) {
auto Buffer = GetBuffer();
DispatchPtr = Buffer->GetOffsetAddress<CustomDispatch>(GetCursorOffset());
EmissionCheckScope(this, 0);
// while (!Thread->State.RunningEvents.ShouldStop.load()) {
// Ptr = FindBlock(RIP)
// if (!Ptr)
// Ptr = CTX->CompileBlock(RIP);
//
// if (Ptr)
// Ptr();
// else
// {
// Ptr = FallbackCore->CompileBlock()
// if (Ptr)
// Ptr()
// else {
// ShouldStop = true;
// }
// }
// }
// Push all the register we need to save
PushCalleeSavedRegisters();
// Push our memory base to the correct register
void *Memory = CTX->MemoryMapper.GetMemoryBase();
LoadConstant(MEM_BASE, (uint64_t)Memory);
// Move our thread pointer to the correct register
// This is passed in to parameter 0 (x0)
mov(STATE, x0);
aarch64::Label LoopTop;
bind(&LoopTop);
// Load in our RIP
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::ThreadState, State.rip)));
LoadConstant(x0, Thread->BlockCache->GetPagePointer());
// Steal the page offset
and_(x1, x2, 0x0FFF);
// Offset the address and add to our page pointer
add(x3, x0, Operand(x2, LSR, 12));
// Load the pointer from the offset
ldr(x3, MemOperand(x3));
aarch64::Label NoBlock;
// If page pointer is zero then we have no block
cbz(x3, &NoBlock);
// Now load from that pointer offset by the page offset to get our real block
ldr(x3, MemOperand(x3, x1));
cbz(x3, &NoBlock);
// If we've made it here then we have a real compiled block
{
blr(x3);
}
aarch64::Label ExitCheck;
bind(&ExitCheck);
constexpr uint64_t ShouldStopOffset = offsetof(FEXCore::Core::ThreadState, RunningEvents.ShouldStop);
// If we don't need to stop then keep going
add(x1, STATE, ShouldStopOffset);
ldarb(x0, MemOperand(x1));
cbz(x0, &LoopTop);
PopCalleeSavedRegisters();
// Return from the function
// LR is set to the correct return location now
ret();
aarch64::Label FallbackCore;
// Need to create the block
{
bind(&NoBlock);
LoadConstant(x0, reinterpret_cast<uintptr_t>(CTX));
mov(x1, STATE);
#if _M_X86_64
CallRuntime(CompileBlockThunk);
#else
using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::InternalThreadState *, uint64_t);
union PtrCast {
ClassPtrType ClassPtr;
uintptr_t Data;
};
PtrCast Ptr;
Ptr.ClassPtr = &FEXCore::Context::Context::CompileBlock;
LoadConstant(x3, Ptr.Data);
// X2 contains our guest RIP
blr(x3); // { ThreadState, RIP}
#endif
// X0 now contains either nullptr or block pointer
cbz(x0, &FallbackCore);
blr(x0);
b(&ExitCheck);
}
aarch64::Label ExitError;
// We need to fallback to our fallback core
{
bind(&FallbackCore);
#if _M_X86_64
// XXX: Fallback core doesn't work on x86-64
// We can't tell the difference between simulator entry points and not
b(&ExitError);
#else
LoadConstant(x0, reinterpret_cast<uintptr_t>(CTX));
mov(x1, STATE);
using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::InternalThreadState *, uint64_t);
union PtrCast {
ClassPtrType ClassPtr;
uintptr_t Data;
};
PtrCast Ptr;
Ptr.ClassPtr = &FEXCore::Context::Context::CompileFallbackBlock;
LoadConstant(x3, Ptr.Data);
// X2 contains our guest RIP
blr(x3); // {ThreadState, RIP}
#endif
// X0 now contains either nullptr or block pointer
cbz(x0, &ExitError);
blr(x0);
b(&ExitCheck);
}
// Exit error
{
bind(&ExitError);
LoadConstant(x0, 1);
add(x1, STATE, ShouldStopOffset);
stlrb(x0, MemOperand(x1));
b(&ExitCheck);
}
#if _M_X86_64
CustomDispatchEnd = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
#endif
FinalizeCode();
// XXX: Crashes currently.
// Disabling will be useful for debugging ThreadState
// CustomDispatchGenerated = true;
}
FEXCore::CPU::CPUBackend *CreateJITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
return new JITCore(ctx, Thread);
Source/Interface/Core/JIT/x86_64/JIT.cpp
+2718 -2606
View File
File diff suppressed because it is too large. Load diff
Source/Interface/Core/LLVMJIT/LLVMCore.cpp
+69 -40
View File
@@ -21,7 +21,7 @@
#include <llvm/Transforms/Vectorize.h>
#include <vector>
#define DESTMAP_AS_MAP 0
#define DESTMAP_AS_MAP 1
#if DESTMAP_AS_MAP
using DestMapType = std::unordered_map<uint64_t, llvm::Value*>;
#else
@@ -185,18 +185,18 @@ private:
llvm::Value *CastVectorToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize);
llvm::Value *CastToOpaqueStructure(llvm::Value *Arg, llvm::Type *DstType);
void SetDest(IR::NodeWrapper Op, llvm::Value *Val);
llvm::Value *GetSrc(IR::NodeWrapper Src);
void SetDest(IR::OrderedNodeWrapper Op, llvm::Value *Val);
llvm::Value *GetSrc(IR::OrderedNodeWrapper Src);
DestMapType DestMap;
FEXCore::IR::IRListView<true> const *CurrentIR;
std::unordered_map<IR::NodeWrapper::NodeOffsetType, llvm::BasicBlock*> JumpTargets;
std::unordered_map<IR::NodeWrapper::NodeOffsetType, llvm::BasicBlock*> ForwardJumpTargets;
std::unordered_map<IR::OrderedNodeWrapper::NodeOffsetType, llvm::BasicBlock*> JumpTargets;
std::unordered_map<IR::OrderedNodeWrapper::NodeOffsetType, llvm::BasicBlock*> ForwardJumpTargets;
// Target Machines
const std::string arch = "x86-64";
const std::string cpu = "znver2";
const std::string cpu = "skylake";
const llvm::Triple TargetTriple{"x86_64", "unknown", "linux", "gnu"};
const llvm::SmallVector<std::string, 0> Attrs;
llvm::TargetMachine *LLVMTarget;
@@ -754,16 +754,16 @@ llvm::Value *LLVMJITCore::CastToOpaqueStructure(llvm::Value *Arg, llvm::Type *Ds
return JITState.IRBuilder->CreateZExtOrTrunc(Arg, DstType->getPointerElementType());
}
void LLVMJITCore::SetDest(IR::NodeWrapper Op, llvm::Value *Val) {
DestMap[Op.NodeOffset] = Val;
void LLVMJITCore::SetDest(IR::OrderedNodeWrapper Op, llvm::Value *Val) {
DestMap[Op.ID()] = Val;
}
llvm::Value *LLVMJITCore::GetSrc(IR::NodeWrapper Src) {
llvm::Value *LLVMJITCore::GetSrc(IR::OrderedNodeWrapper Src) {
#if DESTMAP_AS_MAP
LogMan::Throw::A(DestMap.find(Src.NodeOffset) != DestMap.end(), "Op had Src but wasn't added to the dest map");
LogMan::Throw::A(DestMap.find(Src.ID()) != DestMap.end(), "Op had Src but wasn't added to the dest map");
#endif
auto DstPtr = DestMap[Src.NodeOffset];
auto DstPtr = DestMap[Src.ID()];
LogMan::Throw::A(DstPtr != nullptr, "Destmap had slot but wasn't allocated memory");
return DstPtr;
}
@@ -771,11 +771,11 @@ llvm::Value *LLVMJITCore::GetSrc(IR::NodeWrapper Src) {
void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrapperIterator *Node) {
using namespace llvm;
uintptr_t ListBegin = CurrentIR->GetListData();
uintptr_t DataBegin = CurrentIR->GetData();
uintptr_t ListBegin = CurrentIR->GetListData();
uintptr_t DataBegin = CurrentIR->GetData();
IR::NodeWrapper *WrapperOp = (*Node)();
IR::OrderedNode *RealNode = reinterpret_cast<IR::OrderedNode*>(WrapperOp->GetPtr(ListBegin));
IR::OrderedNodeWrapper *WrapperOp = (*Node)();
IR::OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
uint8_t OpSize = IROp->Size;
@@ -1812,7 +1812,6 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
LogMan::Msg::A("Unknown IR Op: %d(%s)", IROp->Op, FEXCore::IR::GetName(IROp->Op).data());
break;
}
++*Node;
}
void* FEXCore::CPU::LLVMJITCore::CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) {
@@ -1855,35 +1854,66 @@ void* FEXCore::CPU::LLVMJITCore::CompileCode(FEXCore::IR::IRListView<true> const
FunctionModule);
Func->setCallingConv(CallingConv::C);
{
auto Entry = BasicBlock::Create(*Con, "Entry", Func);
JITCurrentState.Blocks.emplace_back(Entry);
JITState.IRBuilder->SetInsertPoint(Entry);
JITCurrentState.CurrentBlock = Entry;
{
auto Entry = BasicBlock::Create(*Con, "Entry", Func);
JITCurrentState.Blocks.emplace_back(Entry);
JITState.IRBuilder->SetInsertPoint(Entry);
JITCurrentState.CurrentBlock = Entry;
CreateGlobalVariables(Engine, FunctionModule);
CreateGlobalVariables(Engine, FunctionModule);
auto Builder = JITState.IRBuilder;
auto Builder = JITState.IRBuilder;
// Let's create the exit block quick
JITCurrentState.ExitBlock = BasicBlock::Create(*Con, "ExitBlock", Func);
JITCurrentState.Blocks.emplace_back(JITCurrentState.ExitBlock);
// Let's create the exit block quick
JITCurrentState.ExitBlock = BasicBlock::Create(*Con, "ExitBlock", Func);
JITCurrentState.Blocks.emplace_back(JITCurrentState.ExitBlock);
JITState.IRBuilder->SetInsertPoint(JITCurrentState.ExitBlock);
Builder->CreateRetVoid();
JITState.IRBuilder->SetInsertPoint(JITCurrentState.ExitBlock);
Builder->CreateRetVoid();
JITState.IRBuilder->SetInsertPoint(Entry);
JITCurrentState.CurrentBlock = Entry;
JITCurrentState.Blocks.emplace_back(JITCurrentState.CurrentBlock);
JITCurrentState.CurrentBlockHasTerm = false;
JITState.IRBuilder->SetInsertPoint(Entry);
JITCurrentState.CurrentBlock = Entry;
JITCurrentState.Blocks.emplace_back(JITCurrentState.CurrentBlock);
JITCurrentState.CurrentBlockHasTerm = false;
IR::NodeWrapperIterator Begin = CurrentIR->begin();
IR::NodeWrapperIterator End = CurrentIR->end();
uintptr_t ListBegin = CurrentIR->GetListData();
uintptr_t DataBegin = CurrentIR->GetData();
while (Begin != End) {
HandleIR(CurrentIR, &Begin);
}
}
auto HeaderIterator = CurrentIR->begin();
IR::OrderedNodeWrapper *HeaderNodeWrapper = HeaderIterator();
IR::OrderedNode *HeaderNode = HeaderNodeWrapper->GetNode(ListBegin);
auto HeaderOp = HeaderNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
LogMan::Throw::A(HeaderOp->Header.Op == IR::OP_IRHEADER, "First op wasn't IRHeader");
IR::OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
while (1) {
using namespace FEXCore::IR;
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
// We grab these nodes this way so we can iterate easily
auto CodeBegin = CurrentIR->at(BlockIROp->Begin);
auto CodeLast = CurrentIR->at(BlockIROp->Last);
while (1) {
HandleIR(CurrentIR, &CodeBegin);
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
break;
}
++CodeBegin;
}
if (BlockIROp->Next.ID() == 0) {
break;
} else {
BlockNode = BlockIROp->Next.GetNode(ListBegin);
}
}
}
for (auto &Block : JITCurrentState.Blocks) {
// If the block is empty then let is just jump to the exit block
@@ -1904,8 +1934,7 @@ void* FEXCore::CPU::LLVMJITCore::CompileCode(FEXCore::IR::IRListView<true> const
raw_ostream &Out = outs();
//if (CTX->Config.LLVM_PrinterPass)
if (ThreadState->State.State.rip == 0x4021b0)
// if (CTX->Config.LLVM_PrinterPass)
{
FPM.addPass(PrintModulePass(Out));
}
Source/Interface/Core/OpcodeDispatcher.cpp
+167 -143
View File
@@ -94,6 +94,7 @@ void OpDispatchBuilder::RETOp(OpcodeArgs) {
// Store the new RIP
_StoreContext(8, offsetof(FEXCore::Core::CPUState, rip), NewRIP);
_EndFunction();
CreateNewEndBlock(0);
Information.HadUnconditionalExit = true;
}
@@ -279,11 +280,7 @@ void OpDispatchBuilder::CALLOp(OpcodeArgs) {
_StoreContext(8, offsetof(FEXCore::Core::CPUState, rip), NewRIP);
_ExitFunction(); // If we get here then leave the function now
// Fracking RIPSetter check ending the block causes issues
// Split the block and leave early to work around the bug
_EndBlock(0);
// Make sure to start a new block after ending this one
_BeginBlock();
CreateNewEndBlock(0);
Information.HadUnconditionalExit = true;
}
@@ -306,13 +303,8 @@ void OpDispatchBuilder::CALLAbsoluteOp(OpcodeArgs) {
_StoreContext(8, offsetof(FEXCore::Core::CPUState, rip), JMPPCOffset);
_ExitFunction(); // If we get here then leave the function now
// Fracking RIPSetter check ending the block causes issues
// Split the block and leave early to work around the bug
_EndBlock(0);
// Make sure to start a new block after ending this one
_BeginBlock();
CreateNewEndBlock(0);
Information.HadUnconditionalExit = true;
}
void OpDispatchBuilder::CondJUMPOp(OpcodeArgs) {
@@ -514,9 +506,6 @@ void OpDispatchBuilder::CondJUMPOp(OpcodeArgs) {
#endif
// Fallback
{
// XXX: Test
GetPackedRFLAG(false);
auto CondJump = _CondJump(SrcCond);
auto RIPOffset = LoadSource(Op, Op->Src1, Op->Flags);
@@ -528,10 +517,10 @@ void OpDispatchBuilder::CondJUMPOp(OpcodeArgs) {
_StoreContext(8, offsetof(FEXCore::Core::CPUState, rip), NewRIP);
_ExitFunction();
_EndBlock(0);
CreateNewEndBlock(0);
// Make sure to start a new block after ending this one
auto JumpTarget = _BeginBlock();
auto JumpTarget = CreateNewBeginBlock();
// This very explicitly avoids the isDest path for Ops. We want the actual destination here
SetJumpTarget(CondJump, JumpTarget);
}
@@ -580,10 +569,7 @@ void OpDispatchBuilder::JUMPOp(OpcodeArgs) {
_StoreContext(8, offsetof(FEXCore::Core::CPUState, rip), NewRIP);
_ExitFunction();
_EndBlock(0);
// Make sure to start a new block after ending this one
_BeginBlock();
CreateNewEndBlock(0);
Information.HadUnconditionalExit = true;
}
}
@@ -598,12 +584,8 @@ void OpDispatchBuilder::JUMPAbsoluteOp(OpcodeArgs) {
_StoreContext(8, offsetof(FEXCore::Core::CPUState, rip), RIPOffset);
_ExitFunction();
_EndBlock(0);
// Make sure to start a new block after ending this one
_BeginBlock();
CreateNewEndBlock(0);
Information.HadUnconditionalExit = true;
}
void OpDispatchBuilder::SETccOp(OpcodeArgs) {
@@ -1159,19 +1141,10 @@ void OpDispatchBuilder::SHLOp(OpcodeArgs) {
GenerateFlags_Shift(Op, _Bfe(Size, 0, ALUOp), _Bfe(Size, 0, Dest), _Bfe(Size, 0, Src));
}
template<bool SHR1Bit>
void OpDispatchBuilder::SHROp(OpcodeArgs) {
bool SHR1Bit = false;
#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))
switch (Op->OP) {
case OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 5):
case OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 5):
SHR1Bit = true;
break;
}
#undef OPD
OrderedNode *Src;
OrderedNode *Dest = LoadSource(Op, Op->Dest, Op->Flags);
auto Dest = LoadSource(Op, Op->Dest, Op->Flags);
if (SHR1Bit) {
Src = _Constant(1);
@@ -1496,9 +1469,7 @@ void OpDispatchBuilder::RDTSCOp(OpcodeArgs) {
}
void OpDispatchBuilder::INCOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) {
LogMan::Msg::A("Can't handle REP on this\n");
}
LogMan::Throw::A(!(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX), "Can't handle REP on this\n");
OrderedNode *Dest = LoadSource(Op, Op->Dest, Op->Flags);
auto OneConst = _Constant(1);
@@ -1511,9 +1482,7 @@ void OpDispatchBuilder::INCOp(OpcodeArgs) {
}
void OpDispatchBuilder::DECOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) {
LogMan::Msg::A("Can't handle REP on this\n");
}
LogMan::Throw::A(!(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX), "Can't handle REP on this\n");
OrderedNode *Dest = LoadSource(Op, Op->Dest, Op->Flags);
auto OneConst = _Constant(1);
@@ -1526,9 +1495,8 @@ void OpDispatchBuilder::DECOp(OpcodeArgs) {
}
void OpDispatchBuilder::STOSOp(OpcodeArgs) {
if (!(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX)) {
LogMan::Msg::A("Can't handle REP not existing on STOS\n");
}
LogMan::Throw::A(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX, "Can't handle REP not existing on STOS\n");
auto Size = GetSrcSize(Op);
auto ZeroConst = _Constant(0);
@@ -1545,10 +1513,10 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) {
OrderedNode *Src = LoadSource(Op, Op->Src1, Op->Flags);
auto JumpStart = _Jump();
_EndBlock(0);
CreateNewEndBlock(0);
// Make sure to start a new block after ending this one
auto LoopStart = _BeginBlock();
auto LoopStart = CreateNewBeginBlock();
SetJumpTarget(JumpStart, LoopStart);
OrderedNode *Counter = _LoadContext(8, offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RCX]));
@@ -1576,23 +1544,18 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) {
// Jump back to the start, we have more work to do
_Jump(LoopStart);
_EndBlock(0);
CreateNewEndBlock(0);
// Make sure to start a new block after ending this one
auto LoopEnd = _BeginBlock();
auto LoopEnd = CreateNewBeginBlock();
SetJumpTarget(CondJump, LoopEnd);
}
void OpDispatchBuilder::MOVSOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) {
LogMan::Msg::A("Can't handle REP\n");
}
LogMan::Throw::A(!(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX), "Can't handle REP on this\n");
_Break(0, 0);
}
void OpDispatchBuilder::CMPSOp(OpcodeArgs) {
if (!(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX)) {
LogMan::Msg::A("Can't only handle REP\n");
}
LogMan::Throw::A(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX, "Can't only handle REP\n");
auto Size = GetSrcSize(Op);
@@ -1608,9 +1571,9 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) {
SizeConst, NegSizeConst);
auto JumpStart = _Jump();
_EndBlock(0);
CreateNewEndBlock(0);
// Make sure to start a new block after ending this one
auto LoopStart = _BeginBlock();
auto LoopStart = CreateNewBeginBlock();
SetJumpTarget(JumpStart, LoopStart);
OrderedNode *Counter = _LoadContext(8, offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RCX]));
@@ -1645,9 +1608,9 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) {
// Jump back to the start, we have more work to do
_Jump(LoopStart);
_EndBlock(0);
CreateNewEndBlock(0);
// Make sure to start a new block after ending this one
auto LoopEnd = _BeginBlock();
auto LoopEnd = CreateNewBeginBlock();
SetJumpTarget(CondJump, LoopEnd);
}
@@ -2178,12 +2141,43 @@ void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) {
}
}
void OpDispatchBuilder::BeginBlock() {
_BeginBlock();
OpDispatchBuilder::IRPair<IROp_BeginBlock> OpDispatchBuilder::CreateNewBeginBlock() {
auto CodeNode = CreateCodeNode();
auto BeginBlock = _BeginBlock();
SetCodeNodeBegin(CodeNode, BeginBlock);
CurrentCodeBlock = CodeNode;
return BeginBlock;
}
void OpDispatchBuilder::EndBlock(uint64_t RIPIncrement) {
_EndBlock(RIPIncrement);
OpDispatchBuilder::IRPair<IROp_EndBlock> OpDispatchBuilder::CreateNewEndBlock(uint64_t RIPIncrement) {
auto EndBlock = _EndBlock(RIPIncrement);
SetCodeNodeLast(CurrentCodeBlock, EndBlock);
return EndBlock;
}
void OpDispatchBuilder::BeginFunction(uint64_t RIP) {
_IRHeader(RIP, InvalidNode->Wrapped(ListData.Begin()), 0);
CreateNewBeginBlock();
}
void OpDispatchBuilder::Finalize() {
// Node 0 is invalid node
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(GetNode(1));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(Data.Begin());
LogMan::Throw::A(IROp->Op == OP_IRHEADER, "First op in function must be our header");
FEXCore::IR::IROp_IRHeader *Op = IROp->CW<FEXCore::IR::IROp_IRHeader>();
Op->BlockCount = CodeBlocks.size();
OrderedNode *PrevCodeBlock{};
for (auto &CodeBlock : CodeBlocks) {
if (PrevCodeBlock) {
LinkCodeBlocks(PrevCodeBlock, CodeBlock);
}
PrevCodeBlock = CodeBlock;
}
Op->Blocks = CodeBlocks[0]->Wrapped(ListData.Begin());
CodeBlocks.clear();
}
void OpDispatchBuilder::ExitFunction() {
@@ -2437,7 +2431,7 @@ void OpDispatchBuilder::StoreResult(FEXCore::X86Tables::DecodedOp Op, OrderedNod
void OpDispatchBuilder::TestFunction() {
printf("Doing Test Function\n");
_BeginBlock();
CreateNewBeginBlock();
auto Load1 = _LoadContext(8, 0);
auto Load2 = _LoadContext(8, 0);
//auto Res = Load1 <Add> Load2;
@@ -2461,12 +2455,14 @@ OpDispatchBuilder::OpDispatchBuilder()
void OpDispatchBuilder::ResetWorkingList() {
Data.Reset();
ListData.Reset();
CodeBlocks.clear();
CurrentWriteCursor = nullptr;
// This is necessary since we do "null" pointer checks
InvalidNode = reinterpret_cast<OrderedNode*>(ListData.Allocate(sizeof(OrderedNode)));
DecodeFailure = false;
Information.HadUnconditionalExit = false;
ShouldDump = false;
CurrentCodeBlock = nullptr;
}
template<unsigned BitOffset>
@@ -2529,9 +2525,6 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(bool Lower8) {
}
void OpDispatchBuilder::GenerateFlags_ADC(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2, OrderedNode *CF) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
auto Size = GetSrcSize(Op) * 8;
// AF
{
@@ -2542,9 +2535,9 @@ void OpDispatchBuilder::GenerateFlags_ADC(FEXCore::X86Tables::DecodedOp Op, Orde
// SF
{
auto ThirtyOneConst = _Constant(Size - 1);
auto SignBitConst = _Constant(GetSrcSize(Op) * 8 - 1);
auto LshrOp = _Lshr(Res, ThirtyOneConst);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
@@ -2552,7 +2545,7 @@ void OpDispatchBuilder::GenerateFlags_ADC(FEXCore::X86Tables::DecodedOp Op, Orde
{
auto PopCountOp = _Popcount(_And(Res, _Constant(0xFF)));
auto XorOp = _Xor(PopCountOp, OneConst);
auto XorOp = _Xor(PopCountOp, _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(XorOp);
}
@@ -2561,7 +2554,7 @@ void OpDispatchBuilder::GenerateFlags_ADC(FEXCore::X86Tables::DecodedOp Op, Orde
auto Dst8 = _Bfe(Size, 0, Res);
auto SelectOp = _Select(FEXCore::IR::COND_EQ,
Dst8, ZeroConst, OneConst, ZeroConst);
Dst8, _Constant(0), _Constant(1), _Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
}
@@ -2571,9 +2564,9 @@ void OpDispatchBuilder::GenerateFlags_ADC(FEXCore::X86Tables::DecodedOp Op, Orde
auto Dst8 = _Bfe(Size, 0, Res);
auto Src8 = _Bfe(Size, 0, Src2);
auto SelectOpLT = _Select(FEXCore::IR::COND_LT, Dst8, Src8, OneConst, ZeroConst);
auto SelectOpLE = _Select(FEXCore::IR::COND_LE, Dst8, Src8, OneConst, ZeroConst);
auto SelectCF = _Select(FEXCore::IR::COND_EQ, CF, OneConst, SelectOpLE, SelectOpLT);
auto SelectOpLT = _Select(FEXCore::IR::COND_LT, Dst8, Src8, _Constant(1), _Constant(0));
auto SelectOpLE = _Select(FEXCore::IR::COND_LE, Dst8, Src8, _Constant(1), _Constant(0));
auto SelectCF = _Select(FEXCore::IR::COND_EQ, CF, _Constant(1), SelectOpLE, SelectOpLT);
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectCF);
}
@@ -2605,9 +2598,6 @@ void OpDispatchBuilder::GenerateFlags_ADC(FEXCore::X86Tables::DecodedOp Op, Orde
}
void OpDispatchBuilder::GenerateFlags_SBB(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2, OrderedNode *CF) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
// AF
{
OrderedNode *AFRes = _Xor(_Xor(Src1, Src2), Res);
@@ -2617,9 +2607,9 @@ void OpDispatchBuilder::GenerateFlags_SBB(FEXCore::X86Tables::DecodedOp Op, Orde
// SF
{
auto ThirtyOneConst = _Constant(GetSrcSize(Op) * 8 - 1);
auto SignBitConst = _Constant(GetSrcSize(Op) * 8 - 1);
auto LshrOp = _Lshr(Res, ThirtyOneConst);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
@@ -2627,14 +2617,14 @@ void OpDispatchBuilder::GenerateFlags_SBB(FEXCore::X86Tables::DecodedOp Op, Orde
{
auto PopCountOp = _Popcount(_And(Res, _Constant(0xFF)));
auto XorOp = _Xor(PopCountOp, OneConst);
auto XorOp = _Xor(PopCountOp, _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(XorOp);
}
// ZF
{
auto SelectOp = _Select(FEXCore::IR::COND_EQ,
Res, ZeroConst, OneConst, ZeroConst);
Res, _Constant(0), _Constant(1), _Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
}
@@ -2644,9 +2634,9 @@ void OpDispatchBuilder::GenerateFlags_SBB(FEXCore::X86Tables::DecodedOp Op, Orde
auto Dst8 = _Bfe(GetSrcSize(Op) * 8, 0, Res);
auto Src8_1 = _Bfe(GetSrcSize(Op) * 8, 0, Src1);
auto SelectOpLT = _Select(FEXCore::IR::COND_GT, Dst8, Src8_1, OneConst, ZeroConst);
auto SelectOpLE = _Select(FEXCore::IR::COND_GE, Dst8, Src8_1, OneConst, ZeroConst);
auto SelectCF = _Select(FEXCore::IR::COND_EQ, CF, OneConst, SelectOpLE, SelectOpLT);
auto SelectOpLT = _Select(FEXCore::IR::COND_GT, Dst8, Src8_1, _Constant(1), _Constant(0));
auto SelectOpLE = _Select(FEXCore::IR::COND_GE, Dst8, Src8_1, _Constant(1), _Constant(0));
auto SelectCF = _Select(FEXCore::IR::COND_EQ, CF, _Constant(1), SelectOpLE, SelectOpLT);
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectCF);
}
@@ -2677,8 +2667,6 @@ void OpDispatchBuilder::GenerateFlags_SBB(FEXCore::X86Tables::DecodedOp Op, Orde
}
void OpDispatchBuilder::GenerateFlags_SUB(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
// AF
{
OrderedNode *AFRes = _Xor(_Xor(Src1, Src2), Res);
@@ -2698,12 +2686,14 @@ void OpDispatchBuilder::GenerateFlags_SUB(FEXCore::X86Tables::DecodedOp Op, Orde
{
auto EightBitMask = _Constant(0xFF);
auto PopCountOp = _Popcount(_And(Res, EightBitMask));
auto XorOp = _Xor(PopCountOp, OneConst);
auto XorOp = _Xor(PopCountOp, _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(XorOp);
}
// ZF
{
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
auto Bfe8 = _Bfe(GetSrcSize(Op) * 8, 0, Res);
auto SelectOp = _Select(FEXCore::IR::COND_EQ,
Bfe8, ZeroConst, OneConst, ZeroConst);
@@ -2712,6 +2702,9 @@ void OpDispatchBuilder::GenerateFlags_SUB(FEXCore::X86Tables::DecodedOp Op, Orde
// CF
{
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
auto SelectOp = _Select(FEXCore::IR::COND_LT,
Src1, Src2, OneConst, ZeroConst);
@@ -2743,9 +2736,6 @@ void OpDispatchBuilder::GenerateFlags_SUB(FEXCore::X86Tables::DecodedOp Op, Orde
}
void OpDispatchBuilder::GenerateFlags_ADD(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
// AF
{
OrderedNode *AFRes = _Xor(_Xor(Src1, Src2), Res);
@@ -2765,14 +2755,14 @@ void OpDispatchBuilder::GenerateFlags_ADD(FEXCore::X86Tables::DecodedOp Op, Orde
{
auto EightBitMask = _Constant(0xFF);
auto PopCountOp = _Popcount(_And(Res, EightBitMask));
auto XorOp = _Xor(PopCountOp, OneConst);
auto XorOp = _Xor(PopCountOp, _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(XorOp);
}
// ZF
{
auto SelectOp = _Select(FEXCore::IR::COND_EQ,
Res, ZeroConst, OneConst, ZeroConst);
Res, _Constant(0), _Constant(1), _Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
}
// CF
@@ -2780,7 +2770,7 @@ void OpDispatchBuilder::GenerateFlags_ADD(FEXCore::X86Tables::DecodedOp Op, Orde
auto Dst8 = _Bfe(GetSrcSize(Op) * 8, 0, Res);
auto Src8 = _Bfe(GetSrcSize(Op) * 8, 0, Src2);
auto SelectOp = _Select(FEXCore::IR::COND_LT, Dst8, Src8, OneConst, ZeroConst);
auto SelectOp = _Select(FEXCore::IR::COND_LT, Dst8, Src8, _Constant(1), _Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectOp);
}
@@ -2813,17 +2803,15 @@ void OpDispatchBuilder::GenerateFlags_ADD(FEXCore::X86Tables::DecodedOp Op, Orde
}
void OpDispatchBuilder::GenerateFlags_MUL(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *High) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
auto SignBitConst = _Constant(GetSrcSize(Op) * 8 - 1);
// PF/AF/ZF/SF
// Undefined
{
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(_Constant(0));
}
// CF/OF
@@ -2833,7 +2821,7 @@ void OpDispatchBuilder::GenerateFlags_MUL(FEXCore::X86Tables::DecodedOp Op, Orde
auto SignBit = _Ashr(Res, SignBitConst);
auto SelectOp = _Select(FEXCore::IR::COND_EQ, High, SignBit, ZeroConst, OneConst);
auto SelectOp = _Select(FEXCore::IR::COND_EQ, High, SignBit, _Constant(0), _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectOp);
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(SelectOp);
@@ -2841,16 +2829,13 @@ void OpDispatchBuilder::GenerateFlags_MUL(FEXCore::X86Tables::DecodedOp Op, Orde
}
void OpDispatchBuilder::GenerateFlags_UMUL(FEXCore::X86Tables::DecodedOp Op, OrderedNode *High) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
// AF/SF/PF/ZF
// Undefined
{
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(_Constant(0));
}
// CF/OF
@@ -2858,7 +2843,7 @@ void OpDispatchBuilder::GenerateFlags_UMUL(FEXCore::X86Tables::DecodedOp Op, Ord
// CF and OF are set if the result of the operation can't be fit in to the destination register
// The result register will be all zero if it can't fit due to how multiplication behaves
auto SelectOp = _Select(FEXCore::IR::COND_EQ, High, ZeroConst, ZeroConst, OneConst);
auto SelectOp = _Select(FEXCore::IR::COND_EQ, High, _Constant(0), _Constant(0), _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectOp);
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(SelectOp);
@@ -2866,13 +2851,11 @@ void OpDispatchBuilder::GenerateFlags_UMUL(FEXCore::X86Tables::DecodedOp Op, Ord
}
void OpDispatchBuilder::GenerateFlags_Logical(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
// AF
{
// Undefined
// Set to zero anyway
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(0));
}
// SF
@@ -2887,35 +2870,32 @@ void OpDispatchBuilder::GenerateFlags_Logical(FEXCore::X86Tables::DecodedOp Op,
{
auto EightBitMask = _Constant(0xFF);
auto PopCountOp = _Popcount(_And(Res, EightBitMask));
auto XorOp = _Xor(PopCountOp, OneConst);
auto XorOp = _Xor(PopCountOp, _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(XorOp);
}
// ZF
{
auto SelectOp = _Select(FEXCore::IR::COND_EQ,
Res, ZeroConst, OneConst, ZeroConst);
Res, _Constant(0), _Constant(1), _Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
}
// CF/OF
{
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(_Constant(0));
}
}
void OpDispatchBuilder::GenerateFlags_Shift(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
auto CmpResult = _Select(FEXCore::IR::COND_EQ, Src2, ZeroConst, OneConst, ZeroConst);
auto CmpResult = _Select(FEXCore::IR::COND_EQ, Src2, _Constant(0), _Constant(1), _Constant(0));
auto CondJump = _CondJump(CmpResult);
// AF
{
// Undefined
// Set to zero anyway
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(0));
}
// SF
@@ -2930,36 +2910,35 @@ void OpDispatchBuilder::GenerateFlags_Shift(FEXCore::X86Tables::DecodedOp Op, Or
{
auto EightBitMask = _Constant(0xFF);
auto PopCountOp = _Popcount(_And(Res, EightBitMask));
auto XorOp = _Xor(PopCountOp, OneConst);
auto XorOp = _Xor(PopCountOp, _Constant(1));
SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(XorOp);
}
// ZF
{
auto SelectOp = _Select(FEXCore::IR::COND_EQ,
Res, ZeroConst, OneConst, ZeroConst);
Res, _Constant(0), _Constant(1), _Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
}
// CF/OF
{
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(_Constant(0));
}
_EndBlock(0);
auto NewBlock = _BeginBlock();
CreateNewEndBlock(0);
auto NewBlock = CreateNewBeginBlock();
SetJumpTarget(CondJump, NewBlock);
}
void OpDispatchBuilder::GenerateFlags_Rotate(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
auto ZeroConst = _Constant(0);
// CF/OF
// XXX: These are wrong
{
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(ZeroConst);
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(_Constant(0));
}
}
@@ -3091,10 +3070,10 @@ void OpDispatchBuilder::INTOp(OpcodeArgs) {
// If condition doesn't hold then keep going
auto CondJump = _CondJump(_Xor(Flag, _Constant(1)));
_Break(Reason, Literal);
_EndBlock(0);
CreateNewEndBlock(0);
// Make sure to start a new block after ending this one
auto JumpTarget = _BeginBlock();
auto JumpTarget = CreateNewBeginBlock();
SetJumpTarget(CondJump, JumpTarget);
}
else {
@@ -3240,8 +3219,40 @@ void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
}
}
void OpDispatchBuilder::PAlignrOp(OpcodeArgs) {
OrderedNode *Src1 = LoadSource(Op, Op->Dest, Op->Flags);
OrderedNode *Src2 = LoadSource(Op, Op->Src1, Op->Flags);
uint8_t Index = Op->Src2.TypeLiteral.Literal;
OrderedNode *Res = _VExtr(GetDstSize(Op), 1, Src1, Src2, Index);
StoreResult(Op, Res);
}
#undef OpcodeArgs
void OpDispatchBuilder::ReplaceAllUsesWithInclusive(OrderedNode *Node, OrderedNode *NewNode, IR::NodeWrapperIterator After, IR::NodeWrapperIterator End) {
uintptr_t ListBegin = ListData.Begin();
uintptr_t DataBegin = Data.Begin();
while (After != End) {
OrderedNodeWrapper *WrapperOp = After();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
for (uint8_t i = 0; i < IROp->NumArgs; ++i) {
if (IROp->Args[i].ID() == Node->Wrapped(ListBegin).ID()) {
LogMan::Msg::D("\tAt %%ssa%d: Replacing ID %%ssa%d with %%ssa%d", WrapperOp->ID(), IROp->Args[i].ID(), NewNode->Wrapped(ListBegin).ID());
Node->RemoveUse();
NewNode->AddUse();
IROp->Args[i].NodeOffset = NewNode->Wrapped(ListBegin).NodeOffset;
}
}
++After;
}
}
void InstallOpcodeHandlers() {
const std::vector<std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr>> BaseOpTable = {
// Instructions
@@ -3279,6 +3290,7 @@ void InstallOpcodeHandlers() {
{0x9F, 1, &OpDispatchBuilder::LAHFOp},
{0xA0, 4, &OpDispatchBuilder::MOVOffsetOp},
{0xA4, 2, &OpDispatchBuilder::MOVSOp},
// XXX: Causes issues with ld.so
{0xA6, 2, &OpDispatchBuilder::CMPSOp},
{0xA8, 2, &OpDispatchBuilder::TESTOp},
{0xAA, 2, &OpDispatchBuilder::STOSOp},
@@ -3396,37 +3408,37 @@ void InstallOpcodeHandlers() {
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 0), 1, &OpDispatchBuilder::ROLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 1), 1, &OpDispatchBuilder::ROROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 5), 1, &OpDispatchBuilder::SHROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 5), 1, &OpDispatchBuilder::SHROp<false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 7), 1, &OpDispatchBuilder::ASHROp}, // SAR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 0), 1, &OpDispatchBuilder::ROLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 1), 1, &OpDispatchBuilder::ROROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 5), 1, &OpDispatchBuilder::SHROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 5), 1, &OpDispatchBuilder::SHROp<false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 7), 1, &OpDispatchBuilder::ASHROp}, // SAR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 0), 1, &OpDispatchBuilder::ROLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 1), 1, &OpDispatchBuilder::ROROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 5), 1, &OpDispatchBuilder::SHROp}, // 1Bit SHR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 5), 1, &OpDispatchBuilder::SHROp<true>}, // 1Bit SHR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD0), 7), 1, &OpDispatchBuilder::ASHROp}, // SAR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 0), 1, &OpDispatchBuilder::ROLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 1), 1, &OpDispatchBuilder::ROROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 5), 1, &OpDispatchBuilder::SHROp}, // 1Bit SHR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 5), 1, &OpDispatchBuilder::SHROp<true>}, // 1Bit SHR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD1), 7), 1, &OpDispatchBuilder::ASHROp}, // SAR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, &OpDispatchBuilder::ROLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, &OpDispatchBuilder::ROROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, &OpDispatchBuilder::SHROp<false>}, // SHR by CL
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 7), 1, &OpDispatchBuilder::ASHROp}, // SAR
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, &OpDispatchBuilder::ROLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, &OpDispatchBuilder::ROROp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, &OpDispatchBuilder::SHLOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, &OpDispatchBuilder::SHROp<false>}, // SHR by CL
{OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, &OpDispatchBuilder::ASHROp}, // SAR
// GROUP 3
@@ -3459,7 +3471,6 @@ void InstallOpcodeHandlers() {
{OPD(FEXCore::X86Tables::TYPE_GROUP_5, OpToIndex(0xFF), 6), 1, &OpDispatchBuilder::PUSHOp},
// GROUP 11
// XXX: LLVM hangs when commented out?
{OPD(FEXCore::X86Tables::TYPE_GROUP_11, OpToIndex(0xC6), 0), 1, &OpDispatchBuilder::MOVOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_11, OpToIndex(0xC7), 0), 1, &OpDispatchBuilder::MOVOp},
#undef OPD
@@ -3467,9 +3478,7 @@ void InstallOpcodeHandlers() {
const std::vector<std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr>> RepModOpTable = {
{0x19, 7, &OpDispatchBuilder::NOPOp},
{0x6F, 1, &OpDispatchBuilder::MOVUPSOp},
// XXX: Causes LLVM to crash if commented out?
{0x7E, 1, &OpDispatchBuilder::MOVQOp},
{0x7F, 1, &OpDispatchBuilder::MOVUPSOp},
};
@@ -3545,7 +3554,8 @@ constexpr uint16_t PF_F2 = 3;
{OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 2), 1, &OpDispatchBuilder::PSRLD<4>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 6), 1, &OpDispatchBuilder::PSLL<8, true>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 3), 1, &OpDispatchBuilder::PSRLDQ},
{OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 7), 1, &OpDispatchBuilder::PSLL<16, true>},
// XXX: Causes issues with ld.so
// {OPD(FEXCore::X86Tables::TYPE_GROUP_14, PF_66, 7), 1, &OpDispatchBuilder::PSLL<16, true>},
// GROUP 15
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 0), 1, &OpDispatchBuilder::FXSaveOp},
@@ -3564,6 +3574,18 @@ constexpr uint16_t PF_F2 = 3;
const std::vector<std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr>> X87OpTable = {
};
#define OPD(REX, prefix, opcode) ((REX << 9) | (prefix << 8) | opcode)
#define PF_3A_NONE 0
#define PF_3A_66 1
const std::vector<std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr>> H0F3ATable = {
{OPD(0, PF_3A_66, 0x0F), 1, &OpDispatchBuilder::PAlignrOp},
};
#undef PF_3A_NONE
#undef PF_3A_66
#undef OPD
uint64_t NumInsts{};
auto InstallToTable = [&NumInsts](auto& FinalTable, auto& LocalTable) {
for (auto Op : LocalTable) {
@@ -3600,6 +3622,8 @@ constexpr uint16_t PF_F2 = 3;
InstallToTable(FEXCore::X86Tables::X87Ops, X87OpTable);
InstallToTable(FEXCore::X86Tables::H0F3ATableOps, H0F3ATable);
// Useful for debugging
// CheckTable(FEXCore::X86Tables::BaseOps);
printf("We installed %ld instructions to the tables\n", NumInsts);
Source/Interface/Core/OpcodeDispatcher.h
+70 -9
View File
@@ -28,9 +28,9 @@ public:
void ResetWorkingList();
bool HadDecodeFailure() { return DecodeFailure; }
void BeginBlock();
void EndBlock(uint64_t RIPIncrement);
void BeginFunction(uint64_t RIP);
void ExitFunction();
void Finalize();
// Dispatch builder functions
#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
@@ -70,6 +70,7 @@ public:
void CMOVOp(OpcodeArgs);
void CPUIDOp(OpcodeArgs);
void SHLOp(OpcodeArgs);
template<bool SHR1Bit>
void SHROp(OpcodeArgs);
void ASHROp(OpcodeArgs);
void ROROp(OpcodeArgs);
@@ -128,6 +129,8 @@ public:
void FXSaveOp(OpcodeArgs);
void FXRStoreOp(OpcodeArgs);
void PAlignrOp(OpcodeArgs);
#undef OpcodeArgs
/**
@@ -215,7 +218,9 @@ public:
IRPair<IROp_VUShr> _VUShr(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) {
return _VUShr(ssa0, ssa1, RegisterSize, ElementSize);
}
IRPair<IROp_VExtr> _VExtr(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1, uint8_t Index) {
return _VExtr(ssa0, ssa1, RegisterSize, ElementSize, Index);
}
IRPair<IROp_Jump> _Jump() {
return _Jump(InvalidNode);
}
@@ -232,8 +237,8 @@ public:
/** @} */
bool IsValueConstant(NodeWrapper ssa, uint64_t *Constant) {
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(ssa.GetPtr(ListData.Begin()));
bool IsValueConstant(OrderedNodeWrapper ssa, uint64_t *Constant) {
OrderedNode *RealNode = ssa.GetNode(ListData.Begin());
FEXCore::IR::IROp_Header *IROp = RealNode->Op(Data.Begin());
if (IROp->Op == OP_CONSTANT) {
auto Op = IROp->C<IR::IROp_Constant>();
@@ -259,6 +264,8 @@ public:
return Node;
}
void ReplaceAllUsesWithInclusive(OrderedNode *Node, OrderedNode *NewNode, IR::NodeWrapperIterator After, IR::NodeWrapperIterator End);
void Unlink(OrderedNode *Node) {
Node->Unlink(ListData.Begin());
}
@@ -271,8 +278,55 @@ public:
LogMan::Throw::A(rhs.ListData.BackingSize() <= ListData.BackingSize(), "Trying to take ownership of data that is too large");
Data.CopyData(rhs.Data);
ListData.CopyData(rhs.ListData);
InvalidNode = rhs.InvalidNode;
CurrentWriteCursor = rhs.CurrentWriteCursor;
CodeBlocks = rhs.CodeBlocks;
}
void SetWriteCursor(OrderedNode *Node) {
CurrentWriteCursor = Node;
}
OrderedNode *GetWriteCursor() {
return CurrentWriteCursor;
}
/**
* @brief This creates an orphaned code node
* The IROp backing is in the correct list but the OrderedNode lives outside of the list
*
* XXX: This is because we don't want code blocks to interleave with current instruction IR ops currently
* We can change this behaviour once we remove the old BeginBlock/EndBlock types
*
* @return OrderedNode
*/
OrderedNode *CreateCodeNode() {
OrderedNode *CodeNode = _CodeBlock(InvalidNode, InvalidNode, InvalidNode);
CodeBlocks.emplace_back(CodeNode);
return CodeNode;
}
void SetCodeNodeBegin(OrderedNode *CodeNode, OrderedNode *Begin) {
FEXCore::IR::IROp_CodeBlock *IROp = CodeNode->Op(Data.Begin())->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(IROp->Header.Op == IROps::OP_CODEBLOCK, "Invalid");
IROp->Begin = Begin->Wrapped(ListData.Begin());
}
void SetCodeNodeLast(OrderedNode *CodeNode, OrderedNode *Last) {
FEXCore::IR::IROp_CodeBlock *IROp = CodeNode->Op(Data.Begin())->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(IROp->Header.Op == IROps::OP_CODEBLOCK, "Invalid");
IROp->Last = Last->Wrapped(ListData.Begin());
}
void LinkCodeBlocks(OrderedNode *CodeNode, OrderedNode *Next) {
FEXCore::IR::IROp_CodeBlock *IROp = CodeNode->Op(Data.Begin())->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(IROp->Header.Op == IROps::OP_CODEBLOCK, "Invalid");
IROp->Next = Next->Wrapped(ListData.Begin());
}
IRPair<IROp_BeginBlock> CreateNewBeginBlock();
IRPair<IROp_EndBlock> CreateNewEndBlock(uint64_t RIPIncrement);
private:
void TestFunction();
bool DecodeFailure{false};
@@ -314,12 +368,17 @@ private:
return Node;
}
void SetWriteCursor(OrderedNode *Node) {
CurrentWriteCursor = Node;
OrderedNode *GetNode(uint32_t SSANode) {
uintptr_t ListBegin = ListData.Begin();
OrderedNode *Node = reinterpret_cast<OrderedNode *>(ListBegin + SSANode * sizeof(OrderedNode));
return Node;
}
OrderedNode *GetWriteCursor() {
return CurrentWriteCursor;
OrderedNode *EmplaceOrphanedNode(OrderedNode *OldNode) {
size_t Size = sizeof(OrderedNode);
OrderedNode *Ptr = reinterpret_cast<OrderedNode*>(ListData.Allocate(Size));
memcpy(Ptr, OldNode, Size);
return Ptr;
}
OrderedNode *CurrentWriteCursor = nullptr;
@@ -329,6 +388,8 @@ private:
IntrusiveAllocator ListData;
OrderedNode *InvalidNode;
OrderedNode *CurrentCodeBlock{};
std::vector<OrderedNode*> CodeBlocks;
};
void InstallOpcodeHandlers();
Source/Interface/Core/RegisterAllocation.cpp
-230
View File
@@ -1,230 +0,0 @@
#include "Common/BitSet.h"
#include "Interface/Core/RegisterAllocation.h"
#include <LogManager.h>
#include <vector>
constexpr uint32_t INVALID_REG = ~0U;
constexpr uint32_t INVALID_CLASS = ~0U;
namespace FEXCore::RA {
struct Register {
};
struct RegisterClass {
uint32_t RegisterBase;
uint32_t NumberOfRegisters{0};
BitSet<uint32_t> Registers;
};
struct RegisterNode {
uint32_t RegisterClass;
uint32_t Register;
uint32_t InterferenceCount;
uint32_t InterferenceListSize;
uint32_t *InterferenceList;
BitSet<uint32_t> Interference;
};
static_assert(std::is_pod<RegisterNode>::value, "We want this to be POD");
struct RegisterSet {
Register *Registers;
RegisterClass *RegisterClasses;
uint32_t RegisterCount;
uint32_t ClassCount;
};
struct SpillStackUnit {
uint32_t Node;
uint32_t Class;
};
struct RegisterGraph {
RegisterSet *Set;
RegisterNode *Nodes;
uint32_t NodeCount;
uint32_t MaxNodeCount;
std::vector<SpillStackUnit> SpillStack;
};
RegisterSet *AllocateRegisterSet(uint32_t RegisterCount, uint32_t ClassCount) {
RegisterSet *Set = new RegisterSet;
Set->RegisterCount = RegisterCount;
Set->ClassCount = ClassCount;
Set->Registers = static_cast<Register*>(calloc(RegisterCount, sizeof(Register)));
Set->RegisterClasses = static_cast<RegisterClass*>(calloc(ClassCount, sizeof(RegisterClass)));
for (uint32_t i = 0; i < ClassCount; ++i) {
Set->RegisterClasses[i].Registers.Allocate(RegisterCount);
}
return Set;
}
void FreeRegisterSet(RegisterSet *Set) {
for (uint32_t i = 0; i < Set->ClassCount; ++i) {
Set->RegisterClasses[i].Registers.Free();
}
free(Set->RegisterClasses);
free(Set->Registers);
delete Set;
}
void AddRegisters(RegisterSet *Set, uint32_t Class, uint32_t RegistersBase, uint32_t RegisterCount) {
for (uint32_t i = 0; i < RegisterCount; ++i) {
Set->RegisterClasses[Class].Registers.Set(RegistersBase + i);
}
Set->RegisterClasses[Class].RegisterBase = RegistersBase;
Set->RegisterClasses[Class].NumberOfRegisters += RegisterCount;
}
RegisterGraph *AllocateRegisterGraph(RegisterSet *Set, uint32_t NodeCount) {
RegisterGraph *Graph = new RegisterGraph;
Graph->Set = Set;
Graph->NodeCount = NodeCount;
Graph->MaxNodeCount = NodeCount;
Graph->Nodes = static_cast<RegisterNode*>(calloc(NodeCount, sizeof(RegisterNode)));
// Initialize nodes
for (uint32_t i = 0; i < NodeCount; ++i) {
Graph->Nodes[i].Register = INVALID_REG;
Graph->Nodes[i].RegisterClass = INVALID_CLASS;
Graph->Nodes[i].InterferenceListSize = 32;
Graph->Nodes[i].InterferenceList = reinterpret_cast<uint32_t*>(calloc(Graph->Nodes[i].InterferenceListSize, sizeof(uint32_t)));
Graph->Nodes[i].InterferenceCount = 0;
Graph->Nodes[i].Interference.Allocate(NodeCount);
Graph->Nodes[i].Interference.Clear(NodeCount);
}
return Graph;
}
void ResetRegisterGraph(RegisterGraph *Graph, uint32_t NodeCount) {
if (NodeCount > Graph->MaxNodeCount) {
uint32_t OldNodeCount = Graph->MaxNodeCount;
Graph->NodeCount = NodeCount;
Graph->MaxNodeCount = NodeCount;
Graph->Nodes = static_cast<RegisterNode*>(realloc(Graph->Nodes, NodeCount * sizeof(RegisterNode)));
// Initialize nodes
for (uint32_t i = 0; i < OldNodeCount; ++i) {
Graph->Nodes[i].Register = INVALID_REG;
Graph->Nodes[i].RegisterClass = INVALID_CLASS;
Graph->Nodes[i].InterferenceCount = 0;
Graph->Nodes[i].Interference.Realloc(NodeCount);
Graph->Nodes[i].Interference.Clear(NodeCount);
}
for (uint32_t i = OldNodeCount; i < NodeCount; ++i) {
Graph->Nodes[i].Register = INVALID_REG;
Graph->Nodes[i].RegisterClass = INVALID_CLASS;
Graph->Nodes[i].InterferenceListSize = 32;
Graph->Nodes[i].InterferenceList = reinterpret_cast<uint32_t*>(calloc(Graph->Nodes[i].InterferenceListSize, sizeof(uint32_t)));
Graph->Nodes[i].InterferenceCount = 0;
Graph->Nodes[i].Interference.Allocate(NodeCount);
Graph->Nodes[i].Interference.Clear(NodeCount);
}
}
else {
// We are only handling a node count of this size right now
Graph->NodeCount = NodeCount;
// Initialize nodes
for (uint32_t i = 0; i < NodeCount; ++i) {
Graph->Nodes[i].Register = INVALID_REG;
Graph->Nodes[i].RegisterClass = INVALID_CLASS;
Graph->Nodes[i].InterferenceCount = 0;
Graph->Nodes[i].Interference.Clear(NodeCount);
}
}
}
void FreeRegisterGraph(RegisterGraph *Graph) {
for (uint32_t i = 0; i < Graph->MaxNodeCount; ++i) {
RegisterNode *Node = &Graph->Nodes[i];
Node->InterferenceCount = 0;
Node->InterferenceListSize = 0;
free(Node->InterferenceList);
Node->Interference.Free();
}
free(Graph->Nodes);
Graph->NodeCount = 0;
Graph->MaxNodeCount = 0;
delete Graph;
}
void SetNodeClass(RegisterGraph *Graph, uint32_t Node, uint32_t Class) {
Graph->Nodes[Node].RegisterClass = Class;
}
void AddNodeInterference(RegisterGraph *Graph, uint32_t Node1, uint32_t Node2) {
auto AddInterference = [&Graph](uint32_t Node1, uint32_t Node2) {
RegisterNode *Node = &Graph->Nodes[Node1];
Node->Interference.Set(Node2);
if (Node->InterferenceListSize <= Node->InterferenceCount) {
Node->InterferenceListSize *= 2;
Node->InterferenceList = reinterpret_cast<uint32_t*>(realloc(Node->InterferenceList, Node->InterferenceListSize * sizeof(uint32_t)));
}
Node->InterferenceList[Node->InterferenceCount] = Node2;
++Node->InterferenceCount;
};
AddInterference(Node1, Node2);
AddInterference(Node2, Node1);
}
uint32_t GetNodeRegister(RegisterGraph *Graph, uint32_t Node) {
return Graph->Nodes[Node].Register;
}
static bool HasInterference(RegisterGraph *Graph, RegisterNode *Node, uint32_t Register) {
for (uint32_t i = 0; i < Node->InterferenceCount; ++i) {
RegisterNode *IntNode = &Graph->Nodes[Node->InterferenceList[i]];
if (IntNode->Register == Register) {
return true;
}
}
return false;
}
bool AllocateRegisters(RegisterGraph *Graph) {
Graph->SpillStack.clear();
for (uint32_t i = 0; i < Graph->NodeCount; ++i) {
RegisterNode *CurrentNode = &Graph->Nodes[i];
if (CurrentNode->RegisterClass == INVALID_CLASS)
continue;
uint32_t Reg = ~0U;
RegisterClass *RAClass = &Graph->Set->RegisterClasses[CurrentNode->RegisterClass];
for (uint32_t ri = 0; ri < RAClass->NumberOfRegisters; ++ri) {
if (!HasInterference(Graph, CurrentNode, RAClass->RegisterBase + ri)) {
Reg = ri;
break;
}
}
if (Reg == ~0U) {
Graph->SpillStack.emplace_back(SpillStackUnit{i, CurrentNode->RegisterClass});
}
else {
CurrentNode->Register = RAClass->RegisterBase + Reg;
}
}
if (!Graph->SpillStack.empty()) {
printf("Couldn't allocate %ld registers\n", Graph->SpillStack.size());
return false;
}
return true;
}
}
Source/Interface/Core/RegisterAllocation.h
-30
View File
@@ -1,30 +0,0 @@
#pragma once
#include <stdint.h>
#include <vector>
namespace FEXCore::RA {
struct RegisterSet;
struct RegisterGraph;
using CrappyBitset = std::vector<bool>;
RegisterSet *AllocateRegisterSet(uint32_t RegisterCount, uint32_t ClassCount);
void FreeRegisterSet(RegisterSet *Set);
void AddRegisters(RegisterSet *Set, uint32_t Class, uint32_t RegistersBase, uint32_t RegisterCount);
/**
* @name Inference graph handling
* @{ */
RegisterGraph *AllocateRegisterGraph(RegisterSet *Set, uint32_t NodeCount);
void FreeRegisterGraph(RegisterGraph *Graph);
void ResetRegisterGraph(RegisterGraph *Graph, uint32_t NodeCount);
void SetNodeClass(RegisterGraph *Graph, uint32_t Node, uint32_t Class);
void AddNodeInterference(RegisterGraph *Graph, uint32_t Node1, uint32_t Node2);
uint32_t GetNodeRegister(RegisterGraph *Graph, uint32_t Node);
bool AllocateRegisters(RegisterGraph *Graph);
/** @} */
}
Source/Interface/HLE/Syscalls.cpp
+55 -31
View File
@@ -8,6 +8,7 @@
#include <FEXCore/Core/X86Enums.h>
#include <sys/mman.h>
#include <unistd.h>
constexpr uint64_t PAGE_SIZE = 4096;
@@ -128,7 +129,7 @@ void SyscallHandler::DefaultProgramBreak(FEXCore::Core::InternalThreadState *Thr
DefaultProgramBreakAddress = Addr;
// Just allocate 1GB of data memory past the default program break location at this point
CTX->MapRegion(Thread, Addr, 0x1000'0000);
CTX->MapRegion(Thread, Addr, 0x1000'0000, true);
}
uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Thread, FEXCore::HLE::SyscallArguments *Args) {
@@ -154,8 +155,11 @@ uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Threa
}
// Memory management
case SYSCALL_BRK: {
LogMan::Msg::D("\tBRK: 0x%lx - 0x%lx", Args->Argument[1], DataSpace);
if (Args->Argument[1] == 0) { // Just wants to get the location of the program break atm
if (DataSpace == 0) {
// XXX: We need to setup our default BRK space first
DefaultProgramBreak(Thread, 0xe000'0000);
}
Result = DataSpace + DataSpaceSize;
}
else {
@@ -175,12 +179,8 @@ uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Threa
break;
}
case SYSCALL_MMAP: {
LogMan::Msg::D("\tMMAP(<addr> %p, <length> 0x%lx, <prot>%d, <flags>0x%x, <fd>%d, <offset>0x%lx)",
Args->Argument[1], Args->Argument[2],
Args->Argument[3], Args->Argument[4],
Args->Argument[5], Args->Argument[6]);
int Flags = Args->Argument[4];
int GuestFD = Args->Argument[5];
int GuestFD = static_cast<int32_t>(Args->Argument[5]);
int HostFD = -1;
if (GuestFD != -1) {
@@ -193,13 +193,13 @@ uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Threa
uint64_t Prot = Args->Argument[3];
#ifdef DEBUG_MMAP
FileSizeToUse = Size;
Prot = PROT_READ | PROT_WRITE | PROT_EXEC;
// FileSizeToUse = Size;
#endif
Prot = PROT_READ | PROT_WRITE | PROT_EXEC;
if (Flags & MAP_FIXED) {
Base = Args->Argument[1];
void *HostPtr = CTX->MemoryMapper.GetPointer<void*>(Base);
void *HostPtr = CTX->MemoryMapper.GetPointerSizeCheck(Base, FileSizeToUse);
if (!HostPtr) {
HostPtr = CTX->MapRegion(Thread, Base, Size, true);
}
@@ -232,7 +232,13 @@ uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Threa
}
else {
// XXX: MMAP should map memory regions for all threads
void *HostPtr = CTX->MapRegion(Thread, Base, Size, true);
void *HostPtr = CTX->MemoryMapper.GetPointerSizeCheck(Base, FileSizeToUse);
if (!HostPtr) {
HostPtr = CTX->MapRegion(Thread, Base, Size, true);
}
else {
LogMan::Msg::D("\tMapping Fixed pointer in already mapped space: 0x%lx -> %p", Base, HostPtr);
}
if (HostFD != -1) {
#ifdef DEBUG_MMAP
@@ -258,14 +264,12 @@ uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Threa
break;
}
case SYSCALL_MPROTECT: {
LogMan::Msg::D("\tMPROTECT: 0x%x, 0x%lx, 0x%lx", Args->Argument[1], Args->Argument[2], Args->Argument[3]);
void *HostPtr = CTX->MemoryMapper.GetPointer<void*>(Args->Argument[1]);
Result = mprotect(HostPtr, Args->Argument[2], Args->Argument[3]);
// Result = mprotect(HostPtr, Args->Argument[2], Args->Argument[3]);
break;
}
case SYSCALL_ARCH_PRCTL: {
LogMan::Msg::D("\tPRTCL: 0x%x: 0x%lx", Args->Argument[1], Args->Argument[2]);
switch (Args->Argument[1]) {
case 0x1001: // ARCH_SET_GS
Thread->State.State.gs = Args->Argument[2];
@@ -508,7 +512,21 @@ uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Threa
Args->Argument[3],
Args->Argument[4]);
break;
case SYSCALL_IOCTL:
Result = FM.Ioctl(
Args->Argument[1],
Args->Argument[2],
CTX->MemoryMapper.GetPointer<void*>(Args->Argument[3]));
break;
case SYSCALL_TIME: {
time_t *ClockResult = CTX->MemoryMapper.GetPointer<time_t*>(Args->Argument[2]);
Result = time(ClockResult);
// XXX: Debug
// memset(ClockResult, 0, sizeof(time_t));
// Result = 0;
}
break;
case SYSCALL_CLOCK_GETTIME: {
timespec *ClockResult = CTX->MemoryMapper.GetPointer<timespec*>(Args->Argument[2]);
Result = clock_gettime(Args->Argument[1], ClockResult);
@@ -533,25 +551,31 @@ uint64_t SyscallHandler::HandleSyscall(FEXCore::Core::InternalThreadState *Threa
break;
}
case SYSCALL_PRLIMIT64: {
LogMan::Throw::A(Args->Argument[3] == 0, "Guest trying to set limit for %d", Args->Argument[2]);
struct rlimit {
uint64_t rlim_cur;
uint64_t rlim_max;
};
switch (Args->Argument[2]) {
case 3: { // Stack limits
rlimit *old_limit = CTX->MemoryMapper.GetPointer<rlimit*>(Args->Argument[3]);
// Default size
old_limit->rlim_cur = 8 * 1024;
old_limit->rlim_max = ~0ULL;
break;
}
default: LogMan::Msg::A("Unknown PRLimit: %d", Args->Argument[2]);
}
Result = 0;
LogMan::Throw::A(Args->Argument[3] == 0, "Guest trying to set limit for %d", Args->Argument[2]);
struct rlimit {
uint64_t rlim_cur;
uint64_t rlim_max;
};
switch (Args->Argument[2]) {
case 3: { // Stack limits
rlimit *old_limit = CTX->MemoryMapper.GetPointer<rlimit*>(Args->Argument[3]);
// Default size
old_limit->rlim_cur = 8 * 1024;
old_limit->rlim_max = ~0ULL;
break;
}
default: LogMan::Msg::A("Unknown PRLimit: %d", Args->Argument[2]);
}
Result = 0;
break;
}
case SYSCALL_UMASK:
// Just say that the mask has always matched what was passed in
Result = Args->Argument[1];
break;
case SYSCALL_CHDIR:
Result = chdir(CTX->MemoryMapper.GetPointer<char const*>(Args->Argument[1]));
break;
// Currently unhandled
// Return fake result
case SYSCALL_RT_SIGACTION:
Source/Interface/IR/IR.cpp
+88 -29
View File
@@ -8,12 +8,20 @@ namespace FEXCore::IR {
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView<false> const* IR, uint64_t Arg) {
*out << "0x" << std::hex << Arg;
}
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView<false> const* IR, RegisterClassType Arg) {
if (Arg == 0)
*out << "GPR";
else if (Arg == 1)
*out << "FPR";
else
*out << "Unknown Registerclass " << Arg;
}
static void PrintArg(std::stringstream *out, IRListView<false> const* IR, NodeWrapper Arg) {
static void PrintArg(std::stringstream *out, IRListView<false> const* IR, OrderedNodeWrapper Arg) {
uintptr_t Data = IR->GetData();
uintptr_t ListBegin = IR->GetListData();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(Arg.GetPtr(ListBegin));
OrderedNode *RealNode = Arg.GetNode(ListBegin);
auto IROp = RealNode->Op(Data);
*out << "%ssa" << std::to_string(Arg.ID()) << " i" << std::dec << (IROp->Size * 8);
@@ -23,44 +31,95 @@ static void PrintArg(std::stringstream *out, IRListView<false> const* IR, NodeWr
}
void Dump(std::stringstream *out, IRListView<false> const* IR) {
uintptr_t Data = IR->GetData();
uintptr_t ListBegin = IR->GetListData();
uintptr_t DataBegin = IR->GetData();
auto Begin = IR->begin();
auto End = IR->end();
while (Begin != End) {
auto Op = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(Op->GetPtr(ListBegin));
auto IROp = RealNode->Op(Data);
auto Op = Begin();
auto Name = FEXCore::IR::GetName(IROp->Op);
OrderedNode *RealNode = Op->GetNode(ListBegin);
auto HeaderOp = RealNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
if (IROp->HasDest) {
*out << "%ssa" << std::to_string(Op->ID()) << " i" << std::dec << (IROp->Size * 8);
if (IROp->Elements > 1) {
*out << "v" << std::dec << IROp->Elements;
OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
uint8_t CurrentIndent = 0;
auto AddIndent = [&out, &CurrentIndent]() {
for (uint8_t i = 0; i < CurrentIndent; ++i) {
*out << "\t";
}
};
*out << "(%%ssa" << std::to_string(RealNode->Wrapped(ListBegin).ID()) << ") " << "IRHeader ";
*out << "0x" << std::hex << HeaderOp->Entry << ", ";
*out << "%%ssa" << HeaderOp->Blocks.ID() << ", ";
*out << std::dec << HeaderOp->BlockCount << std::endl;
while (1) {
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
// We grab these nodes this way so we can iterate easily
auto CodeBegin = IR->at(BlockIROp->Begin);
auto CodeLast = IR->at(BlockIROp->Last);
*out << "(%%ssa" << std::to_string(BlockNode->Wrapped(ListBegin).ID()) << ") " << "CodeBlock ";
*out << "%%ssa" << std::to_string(BlockIROp->Begin.ID()) << ", ";
*out << "%%ssa" << std::to_string(BlockIROp->Last.ID()) << ", ";
*out << "%%ssa" << std::to_string(BlockIROp->Next.ID()) << std::endl;
while (1) {
OrderedNodeWrapper *CodeOp = CodeBegin();
OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
auto IROp = CodeNode->Op(DataBegin);
auto Name = FEXCore::IR::GetName(IROp->Op);
AddIndent();
if (IROp->HasDest) {
*out << "%ssa" << std::to_string(CodeOp->ID()) << " i" << std::dec << (IROp->Size * 8);
if (IROp->Elements > 1) {
*out << "v" << std::dec << IROp->Elements;
}
*out << " = ";
}
*out << " = ";
else {
*out << "(%%ssa" << std::to_string(CodeOp->ID()) << ") ";
}
*out << Name;
switch (IROp->Op) {
case IR::OP_BEGINBLOCK:
*out << " %ssa" << std::to_string(CodeOp->ID());
++CurrentIndent;
break;
case IR::OP_ENDBLOCK:
--CurrentIndent;
break;
default: break;
}
#define IROP_ARGPRINTER_HELPER
#include "IRDefines.inc"
default: *out << "<Unknown Args>"; break;
}
*out << "\n";
printf("%s", out->str().c_str());
*out = std::stringstream{};
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
break;
}
++CodeBegin;
}
*out << Name;
switch (IROp->Op) {
case IR::OP_BEGINBLOCK:
*out << " %ssa" << std::to_string(Op->ID());
if (BlockIROp->Next.ID() == 0) {
break;
default: break;
} else {
BlockNode = BlockIROp->Next.GetNode(ListBegin);
}
#define IROP_ARGPRINTER_HELPER
#include "IRDefines.inc"
default: *out << "<Unknown Args>"; break;
}
*out << "\n";
++Begin;
}
}
}
Source/Interface/IR/IR.json
+42 -4
View File
@@ -19,6 +19,21 @@
"Ops": {
"Dummy": {
},
"IRHeader": {
"Args": [
"uint64_t", "Entry",
"OrderedNodeWrapper", "Blocks",
"uint32_t", "BlockCount"
]
},
"CodeBlock": {
"SSAArgs": "3",
"SSANames": [
"Begin",
"Last",
"Next"
]
},
"Constant": {
"HasDest": true,
"FixedDestSize": "8",
@@ -79,6 +94,20 @@
"uint32_t", "Offset"
]
},
"SpillRegister": {
"SSAArgs": "1",
"Args": [
"uint32_t", "Slot",
"RegisterClassType", "Class"
]
},
"FillRegister": {
"HasDest": true,
"Args": [
"uint32_t", "Slot",
"RegisterClassType", "Class"
]
},
"LoadFlag": {
"HasDest": true,
@@ -343,6 +372,11 @@
"SSAArgs": "3"
},
"Print": {
"DispatcherUnary": true,
"SSAArgs": "1"
},
"CreateVector2": {
"HasDest": true,
"DestSize": "GetOpSize(ssa0) * 2",
@@ -513,11 +547,15 @@
]
},
"Print": {
"DispatcherUnary": true,
"SSAArgs": "1"
"VExtr": {
"HasDest": true,
"SSAArgs": "2",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize",
"uint8_t", "Index"
]
},
"Last": {
"Last": true,
"Args": []
Source/Interface/IR/PassManager.cpp
+6 -1
View File
@@ -1,14 +1,19 @@
#include "Interface/IR/Passes.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include "Interface/IR/PassManager.h"
namespace FEXCore::IR {
void PassManager::AddDefaultPasses() {
Passes.emplace_back(std::unique_ptr<FEXCore::IR::Pass>(CreateConstProp()));
Passes.emplace_back(std::unique_ptr<FEXCore::IR::Pass>(CreateRedundantContextLoadElimination()));
// XXX: Causes corrupted output in test app
// Passes.emplace_back(std::unique_ptr<FEXCore::IR::Pass>(CreateRedundantContextLoadElimination()));
Passes.emplace_back(std::unique_ptr<FEXCore::IR::Pass>(CreateRedundantFlagCalculationEliminination()));
Passes.emplace_back(std::unique_ptr<FEXCore::IR::Pass>(CreateSyscallOptimization()));
Passes.emplace_back(std::unique_ptr<FEXCore::IR::Pass>(CreatePassDeadContextStoreElimination()));
// If the IR is compacted post-RA then the node indexing gets messed up and the backend isn't able to find the register assigned to a node
// Compact before IR, don't worry about RA generating spills/fills
Passes.emplace_back(std::unique_ptr<FEXCore::IR::Pass>(CreateIRCompaction()));
}
Source/Interface/IR/PassManager.h
+3
View File
@@ -18,6 +18,9 @@ class PassManager final {
public:
void AddDefaultPasses();
void AddDefaultValidationPasses();
void InsertPass(Pass *Pass) {
Passes.emplace_back(Pass);
}
bool Run(OpDispatchBuilder *Disp);
private:
Source/Interface/IR/Passes/ConstProp.cpp
+46 -19
View File
@@ -14,33 +14,60 @@ bool ConstProp::Run(OpDispatchBuilder *Disp) {
uintptr_t ListBegin = CurrentIR.GetListData();
uintptr_t DataBegin = CurrentIR.GetData();
IR::NodeWrapperIterator Begin = CurrentIR.begin();
IR::NodeWrapperIterator End = CurrentIR.end();
auto Begin = CurrentIR.begin();
auto Op = Begin();
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
OrderedNode *RealNode = Op->GetNode(ListBegin);
auto HeaderOp = RealNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
switch (IROp->Op) {
case OP_ZEXT: {
auto Op = IROp->C<IR::IROp_Zext>();
uint64_t Constant;
if (Disp->IsValueConstant(Op->Header.Args[0], &Constant)) {
uint64_t NewConstant = Constant & ((1ULL << Op->SrcSize) - 1);
auto ConstantVal = Disp->_Constant(NewConstant);
Disp->ReplaceAllUsesWith(RealNode, ConstantVal);
Changed = true;
OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
auto OriginalWriteCursor = Disp->GetWriteCursor();
while (1) {
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
// We grab these nodes this way so we can iterate easily
auto CodeBegin = CurrentIR.at(BlockIROp->Begin);
auto CodeLast = CurrentIR.at(BlockIROp->Last);
while (1) {
auto CodeOp = CodeBegin();
OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
auto IROp = CodeNode->Op(DataBegin);
switch (IROp->Op) {
case OP_ZEXT: {
auto Op = IROp->C<IR::IROp_Zext>();
uint64_t Constant;
if (Disp->IsValueConstant(Op->Header.Args[0], &Constant)) {
uint64_t NewConstant = Constant & ((1ULL << Op->SrcSize) - 1);
Disp->SetWriteCursor(CodeNode);
auto ConstantVal = Disp->_Constant(NewConstant);
Disp->ReplaceAllUsesWith(CodeNode, ConstantVal);
Changed = true;
}
break;
}
default: break;
}
break;
}
default: break;
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
break;
}
++CodeBegin;
}
++Begin;
if (BlockIROp->Next.ID() == 0) {
break;
} else {
BlockNode = BlockIROp->Next.GetNode(ListBegin);
}
}
Disp->SetWriteCursor(OriginalWriteCursor);
return Changed;
}
Source/Interface/IR/Passes/DeadContextStoreElimination.cpp
+69 -49
View File
@@ -42,68 +42,88 @@ static bool IsGPR(uint32_t Offset, uint8_t *greg) {
bool RCLE::Run(OpDispatchBuilder *Disp) {
bool Changed = false;
auto CurrentIR = Disp->ViewIR();
std::array<OrderedNodeWrapper::NodeOffsetType, 16> LastValidGPRStores{};
auto OriginalWriteCursor = Disp->GetWriteCursor();
uintptr_t ListBegin = CurrentIR.GetListData();
uintptr_t DataBegin = CurrentIR.GetData();
IR::NodeWrapperIterator Begin = CurrentIR.begin();
IR::NodeWrapperIterator End = CurrentIR.end();
auto Begin = CurrentIR.begin();
auto Op = Begin();
std::array<NodeWrapper*, 16> LastValidGPRStores{};
OrderedNode *RealNode = Op->GetNode(ListBegin);
auto HeaderOp = RealNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
if (IROp->Op == OP_BEGINBLOCK ||
IROp->Op == OP_ENDBLOCK ||
IROp->Op == OP_JUMP ||
IROp->Op == OP_CONDJUMP ||
IROp->Op == OP_EXITFUNCTION) {
// We don't track across block boundaries
LastValidGPRStores.fill(nullptr);
}
while (1) {
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
if (IROp->Op == OP_STORECONTEXT) {
auto Op = IROp->CW<IR::IROp_StoreContext>();
// Make sure we are within GREG state
uint8_t greg = ~0;
if (IsAlignedGPR(Op->Size, Op->Offset, &greg)) {
FEXCore::IR::IROp_Header *ArgOp = reinterpret_cast<OrderedNode*>(Op->Header.Args[0].GetPtr(ListBegin))->Op(DataBegin);
// Ensure we aren't doing a mismatched store
// XXX: We should really catch this in IR validation
if (ArgOp->Size == 8) {
LastValidGPRStores[greg] = &Op->Header.Args[0];
}
else {
LastValidGPRStores[greg] = nullptr;
}
} else if (IsGPR(Op->Offset, &greg)) {
// If we aren't overwriting the whole state then we don't want to track this value
LastValidGPRStores[greg] = nullptr;
}
}
// We grab these nodes this way so we can iterate easily
auto CodeBegin = CurrentIR.at(BlockIROp->Begin);
auto CodeLast = CurrentIR.at(BlockIROp->Last);
while (1) {
auto CodeOp = CodeBegin();
OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
auto IROp = CodeNode->Op(DataBegin);
if (IROp->Op == OP_LOADCONTEXT) {
auto Op = IROp->C<IR::IROp_LoadContext>();
// Make sure we are within GREG state
uint8_t greg = ~0;
if (IsAlignedGPR(Op->Size, Op->Offset, &greg)) {
if (LastValidGPRStores[greg] != nullptr) {
// If the last store matches this load value then we can replace the loaded value with the previous valid one
auto MovVal = Disp->_Mov(reinterpret_cast<OrderedNode*>(LastValidGPRStores[greg]->GetPtr(ListBegin)));
Disp->ReplaceAllUsesWith(RealNode, MovVal);
Changed = true;
}
} else if (IsGPR(Op->Offset, &greg)) {
if (IROp->Op == OP_STORECONTEXT) {
auto Op = IROp->CW<IR::IROp_StoreContext>();
// Make sure we are within GREG state
uint8_t greg = ~0;
if (IsAlignedGPR(Op->Size, Op->Offset, &greg)) {
LastValidGPRStores[greg] = Op->Header.Args[0].NodeOffset;
} else if (IsGPR(Op->Offset, &greg)) {
// If we aren't overwriting the whole state then we don't want to track this value
LastValidGPRStores[greg] = nullptr; // 0 is invalid
LastValidGPRStores[greg] = 0;
}
}
if (IROp->Op == OP_LOADCONTEXT) {
auto Op = IROp->C<IR::IROp_LoadContext>();
// Make sure we are within GREG state
uint8_t greg = ~0;
if (IsAlignedGPR(Op->Size, Op->Offset, &greg)) {
if (LastValidGPRStores[greg] != 0) {
// If the last store matches this load value then we can replace the loaded value with the previous valid one
if (1) {
Disp->SetWriteCursor(CodeNode);
auto MovVal = Disp->_Mov(OrderedNodeWrapper::WrapOffset(LastValidGPRStores[greg]).GetNode(ListBegin));
Disp->ReplaceAllUsesWith(CodeNode, MovVal);
}
else {
Disp->ReplaceAllUsesWith(CodeNode, OrderedNodeWrapper::WrapOffset(LastValidGPRStores[greg]).GetNode(ListBegin));
}
Changed = true;
}
} else if (IsGPR(Op->Offset, &greg)) {
// If we aren't overwriting the whole state then we don't want to track this value
LastValidGPRStores[greg] = 0; // 0 is invalid
}
}
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
break;
}
++CodeBegin;
}
++Begin;
if (BlockIROp->Next.ID() == 0) {
break;
} else {
BlockNode = BlockIROp->Next.GetNode(ListBegin);
}
// We don't track across block boundaries
LastValidGPRStores.fill(0);
}
Disp->SetWriteCursor(OriginalWriteCursor);
return Changed;
}
Source/Interface/IR/Passes/IRCompaction.cpp
+179 -66
View File
@@ -12,7 +12,7 @@ public:
private:
OpDispatchBuilder LocalBuilder;
std::vector<IR::NodeWrapper::NodeOffsetType> NodeLocationRemapper;
std::vector<IR::OrderedNodeWrapper::NodeOffsetType> NodeLocationRemapper;
};
IRCompaction::IRCompaction() {
@@ -21,15 +21,16 @@ IRCompaction::IRCompaction() {
bool IRCompaction::Run(OpDispatchBuilder *Disp) {
auto CurrentIR = Disp->ViewIR();
auto LocalIR = LocalBuilder.ViewIR();
uint32_t NodeCount = LocalIR.GetListSize() / sizeof(OrderedNode);
uint32_t NodeCount = CurrentIR.GetSSACount();
// Reset our local working list
LocalBuilder.ResetWorkingList();
if (NodeLocationRemapper.size() < NodeCount) {
NodeLocationRemapper.resize(NodeCount);
}
memset(&NodeLocationRemapper.at(0), 0xFF, NodeCount * sizeof(IR::NodeWrapper::NodeOffsetType));
memset(&NodeLocationRemapper.at(0), 0xFF, NodeCount * sizeof(IR::OrderedNodeWrapper::NodeOffsetType));
// Reset our local working list
LocalBuilder.ResetWorkingList();
auto LocalIR = LocalBuilder.ViewIR();
uintptr_t LocalListBegin = LocalIR.GetListData();
uintptr_t LocalDataBegin = LocalIR.GetData();
@@ -37,89 +38,201 @@ bool IRCompaction::Run(OpDispatchBuilder *Disp) {
uintptr_t ListBegin = CurrentIR.GetListData();
uintptr_t DataBegin = CurrentIR.GetData();
IR::NodeWrapperIterator Begin = CurrentIR.begin();
IR::NodeWrapperIterator End = CurrentIR.end();
auto HeaderIterator = CurrentIR.begin();
OrderedNodeWrapper *HeaderNodeWrapper = HeaderIterator();
OrderedNode *HeaderNode = HeaderNodeWrapper->GetNode(ListBegin);
auto HeaderOp = HeaderNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
// This compaction pass is something that we need to ensure correct ordering and distances between IROps\
// This compaction pass is something that we need to ensure correct ordering and distances between IROps
// Later on we assume that an IROp's SSA value live range is its Node locations
//
// RA distance calculation is calculated purely on the Node locations
// So we just need to reorder those
// So we need to reorder those
//
// Additionally there may be some dead ops hanging out in the IR list that are orphaned.
// These can also be dropped during this pass
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
// First thing is first, we need to do some housekeeping
// Create the IRHeader op
// Create the codeblocks
// Then create all the ops inside the code blocks
if (IROp->HasDest && RealNode->GetUses() == 0) {
// Should this be in a dedicated DCE pass?
++Begin;
continue;
auto LocalHeaderOp = LocalBuilder._IRHeader(HeaderOp->Entry, OrderedNodeWrapper::WrapOffset(0), HeaderOp->BlockCount);
NodeLocationRemapper[HeaderNode->Wrapped(ListBegin).ID()] = LocalHeaderOp.Node->Wrapped(LocalListBegin).ID();
struct CodeBlockData {
OrderedNode *OldNode;
OrderedNode *NewNode;
};
std::vector<CodeBlockData> GeneratedCodeBlocks{};
{
// Generate our codeblocks and link them together
OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
OrderedNode* PrevCodeBlock{};
while (1) {
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
auto LocalBlockIRNode = LocalBuilder.CreateCodeNode();
NodeLocationRemapper[BlockNode->Wrapped(ListBegin).ID()] = LocalBlockIRNode->Wrapped(LocalListBegin).ID();
GeneratedCodeBlocks.emplace_back(CodeBlockData{BlockNode, LocalBlockIRNode});
if (PrevCodeBlock) {
auto PrevLocalBlockIROp = PrevCodeBlock->Op(LocalDataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
PrevLocalBlockIROp->Next = LocalBlockIRNode->Wrapped(LocalListBegin);
}
if (BlockIROp->Next.ID() == 0) {
break;
} else {
BlockNode = BlockIROp->Next.GetNode(ListBegin);
PrevCodeBlock = LocalBlockIRNode;
}
}
size_t OpSize = FEXCore::IR::GetSize(IROp->Op);
// Allocate the ops locally for our local dispatch
auto LocalPair = LocalBuilder.AllocateRawOp(OpSize);
IR::NodeWrapper LocalNodeWrapper = LocalPair.Node->Wrapped(LocalListBegin);
// Copy over the op
memcpy(LocalPair.first, IROp, OpSize);
// Set our map remapper to map the new location
// Even nodes that don't have a destination need to be in this map
// Need to be able to remap branch targets any other bits
NodeLocationRemapper[WrapperOp->ID()] = LocalNodeWrapper.ID();
++Begin;
// Link the IRHeader to the first code block
LocalHeaderOp.first->Blocks = GeneratedCodeBlocks[0].NewNode->Wrapped(LocalListBegin);
}
Begin = CurrentIR.begin();
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
{
// Copy all of our IR ops over to the new location
for (auto &Block : GeneratedCodeBlocks) {
auto BlockIROp = Block.OldNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
if (IROp->HasDest && RealNode->GetUses() == 0) {
// Should this be in a dedicated DCE pass?
++Begin;
continue;
// We grab these nodes this way so we can iterate easily
auto CodeBegin = CurrentIR.at(BlockIROp->Begin);
auto CodeLast = CurrentIR.at(BlockIROp->Last);
CodeBlockData FirstNode{};
CodeBlockData LastNode{};
uint32_t i {};
while (1) {
auto CodeOp = CodeBegin();
OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
auto IROp = CodeNode->Op(DataBegin);
LogMan::Throw::A(IROp->Op != OP_IRHEADER, "%%ssa%d ended up being IRHeader. Shouldn't have hit this", CodeOp->ID());
size_t OpSize = FEXCore::IR::GetSize(IROp->Op);
// Allocate the ops locally for our local dispatch
auto LocalPair = LocalBuilder.AllocateRawOp(OpSize);
IR::OrderedNodeWrapper LocalNodeWrapper = LocalPair.Node->Wrapped(LocalListBegin);
// Copy over the op
memcpy(LocalPair.first, IROp, OpSize);
LogMan::Throw::A(LocalPair.first->Op == IROp->Op, "What. How did this fail");
// Set our map remapper to map the new location
// Even nodes that don't have a destination need to be in this map
// Need to be able to remap branch targets any other bits
NodeLocationRemapper[CodeOp->ID()] = LocalNodeWrapper.ID();
if (i == 0) {
FirstNode.OldNode = CodeNode;
FirstNode.NewNode = LocalPair.Node;
}
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
LastNode.OldNode = CodeNode;
LastNode.NewNode = LocalPair.Node;
break;
}
++CodeBegin;
++i;
}
// Set the code block's begin and end correctly
auto NewBlockIROp = Block.NewNode->Op(LocalDataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
NewBlockIROp->Begin = FirstNode.NewNode->Wrapped(LocalListBegin);
NewBlockIROp->Last = LastNode.NewNode->Wrapped(LocalListBegin);
}
NodeWrapper LocalNodeWrapper = NodeWrapper::WrapOffset(NodeLocationRemapper[WrapperOp->ID()] * sizeof(OrderedNode));
OrderedNode *LocalNode = reinterpret_cast<OrderedNode*>(LocalNodeWrapper.GetPtr(LocalListBegin));
FEXCore::IR::IROp_Header *LocalIROp = LocalNode->Op(LocalDataBegin);
// Now that we have the op copied over, we need to modify SSA values to point to the new correct locations
for (uint8_t i = 0; i < IROp->NumArgs; ++i) {
NodeWrapper OldArg = IROp->Args[i];
LogMan::Throw::A(NodeLocationRemapper[OldArg.ID()] != ~0U, "Tried remapping unfound node");
LocalIROp->Args[i].NodeOffset = NodeLocationRemapper[OldArg.ID()] * sizeof(OrderedNode);
}
++Begin;
}
// uintptr_t OldListSize = CurrentIR.GetListSize();
// uintptr_t OldDataSize = CurrentIR.GetDataSize();
{
// Fixup the arguments of all the IROps
for (auto &Block : GeneratedCodeBlocks) {
auto BlockIROp = Block.OldNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
// We grab these nodes this way so we can iterate easily
auto CodeBegin = CurrentIR.at(BlockIROp->Begin);
auto CodeLast = CurrentIR.at(BlockIROp->Last);
while (1) {
auto CodeOp = CodeBegin();
OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
auto IROp = CodeNode->Op(DataBegin);
OrderedNodeWrapper LocalNodeWrapper = OrderedNodeWrapper::WrapOffset(NodeLocationRemapper[CodeOp->ID()] * sizeof(OrderedNode));
OrderedNode *LocalNode = LocalNodeWrapper.GetNode(LocalListBegin);
FEXCore::IR::IROp_Header *LocalIROp = LocalNode->Op(LocalDataBegin);
// Now that we have the op copied over, we need to modify SSA values to point to the new correct locations
for (uint8_t i = 0; i < IROp->NumArgs; ++i) {
uint32_t OldArg = IROp->Args[i].ID();
LogMan::Throw::A(NodeLocationRemapper[OldArg] != ~0U, "Tried remapping unfound node %%ssa%d", OldArg);
LocalIROp->Args[i].NodeOffset = NodeLocationRemapper[OldArg] * sizeof(OrderedNode);
}
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
break;
}
++CodeBegin;
}
}
}
// XXX: Example for iterating blocks
// OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
// while (1) {
// auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
// LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
//
// uintptr_t NewListSize = LocalIR.GetListSize();
// uintptr_t NewDataSize = LocalIR.GetDataSize();
// // We grab these nodes this way so we can iterate easily
// auto CodeBegin = CurrentIR.at(BlockIROp->Begin);
// auto CodeLast = CurrentIR.at(BlockIROp->Last);
// while (1) {
// auto CodeOp = CodeBegin();
// OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
// auto IROp = CodeNode->Op(DataBegin);
//
// if (NewListSize < OldListSize ||
// NewDataSize < OldDataSize) {
// if (NewListSize < OldListSize) {
// LogMan::Msg::D("Shaved %ld bytes off the list size", OldListSize - NewListSize);
// // CodeLast is inclusive. So we still need to dump the CodeLast op as well
// if (CodeBegin == CodeLast) {
// break;
// }
// ++CodeBegin;
// }
// if (NewDataSize < OldDataSize) {
// LogMan::Msg::D("Shaved %ld bytes off the data size", OldDataSize - NewDataSize);
//
// if (BlockIROp->Next.ID() == 0) {
// break;
// } else {
// BlockNode = BlockIROp->Next.GetNode(ListBegin);
// }
// }
// if (NewListSize > OldListSize ||
// NewDataSize > OldDataSize) {
// LogMan::Msg::A("Whoa. Compaction made the IR a different size when it shouldn't have. 0x%lx > 0x%lx or 0x%lx > 0x%lx",NewListSize, OldListSize, NewDataSize, OldDataSize);
// }
// uintptr_t OldListSize = CurrentIR.GetListSize();
// uintptr_t OldDataSize = CurrentIR.GetDataSize();
// uintptr_t NewListSize = LocalIR.GetListSize();
// uintptr_t NewDataSize = LocalIR.GetDataSize();
// if (NewListSize < OldListSize ||
// NewDataSize < OldDataSize) {
// if (NewListSize < OldListSize) {
// LogMan::Msg::D("Shaved %ld bytes off the list size", OldListSize - NewListSize);
// }
// if (NewDataSize < OldDataSize) {
// LogMan::Msg::D("Shaved %ld bytes off the data size", OldDataSize - NewDataSize);
// }
// }
// if (NewListSize > OldListSize ||
// NewDataSize > OldDataSize) {
// LogMan::Msg::A("Whoa. Compaction made the IR a different size when it shouldn't have. 0x%lx > 0x%lx or 0x%lx > 0x%lx",NewListSize, OldListSize, NewDataSize, OldDataSize);
// }
Disp->CopyData(LocalBuilder);
Source/Interface/IR/Passes/IRValidation.cpp
+13 -13
View File
@@ -6,13 +6,13 @@
namespace FEXCore::IR::Validation {
struct BlockInfo {
IR::NodeWrapper *Begin;
IR::NodeWrapper *End;
IR::OrderedNodeWrapper *Begin;
IR::OrderedNodeWrapper *End;
bool HasExit;
std::vector<IR::NodeWrapper*> Predecessors;
std::vector<IR::NodeWrapper*> Successors;
std::vector<IR::OrderedNodeWrapper*> Predecessors;
std::vector<IR::OrderedNodeWrapper*> Successors;
};
class IRValidation final : public FEXCore::IR::Pass {
@@ -20,7 +20,7 @@ public:
bool Run(OpDispatchBuilder *Disp) override;
private:
std::unordered_map<IR::NodeWrapper::NodeOffsetType, BlockInfo> OffsetToBlockMap;
std::unordered_map<IR::OrderedNodeWrapper::NodeOffsetType, BlockInfo> OffsetToBlockMap;
};
bool IRValidation::Run(OpDispatchBuilder *Disp) {
@@ -37,8 +37,8 @@ bool IRValidation::Run(OpDispatchBuilder *Disp) {
std::ostringstream Errors;
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
OrderedNodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
uint8_t OpSize = IROp->Size;
@@ -56,7 +56,7 @@ bool IRValidation::Run(OpDispatchBuilder *Disp) {
}
for (uint8_t i = 0; i < IROp->NumArgs; ++i) {
NodeWrapper Arg = IROp->Args[i];
OrderedNodeWrapper Arg = IROp->Args[i];
if (Arg.ID() == 0) {
HadError |= true;
Errors << "Op" << WrapperOp->ID() <<": Arg[" << i << "] has invalid target of %ssa0" << std::endl;
@@ -70,7 +70,7 @@ bool IRValidation::Run(OpDispatchBuilder *Disp) {
Errors << "BasicBlock " << WrapperOp->ID() << ": Begin in middle of block" << std::endl;
}
auto Block = OffsetToBlockMap.try_emplace(WrapperOp->ID(), BlockInfo{}).first;
auto Block = OffsetToBlockMap.try_emplace(WrapperOp->ID()).first;
CurrentBlock = &Block->second;
CurrentBlock->Begin = WrapperOp;
InBlock = true;
@@ -109,14 +109,14 @@ bool IRValidation::Run(OpDispatchBuilder *Disp) {
CurrentBlock->Successors.emplace_back(IterLocation());
}
OrderedNode *TargetNode = reinterpret_cast<OrderedNode*>(IterLocation()->GetPtr(ListBegin));
OrderedNode *TargetNode = IterLocation()->GetNode(ListBegin);
FEXCore::IR::IROp_Header *TargetOp = TargetNode->Op(DataBegin);
HadError |= TargetOp->Op != OP_BEGINBLOCK;
if (TargetOp->Op != OP_BEGINBLOCK) {
Errors << "CondJump " << WrapperOp->ID() << ": CondJump to Op that isn't the begining of a block" << std::endl;
}
else {
auto Block = OffsetToBlockMap.try_emplace(IterLocation()->NodeOffset, BlockInfo{}).first;
auto Block = OffsetToBlockMap.try_emplace(IterLocation()->NodeOffset).first;
Block->second.Predecessors.emplace_back(CurrentBlock->Begin);
}
@@ -130,14 +130,14 @@ bool IRValidation::Run(OpDispatchBuilder *Disp) {
CurrentBlock->Successors.emplace_back(IterLocation());
}
OrderedNode *TargetNode = reinterpret_cast<OrderedNode*>(IterLocation()->GetPtr(ListBegin));
OrderedNode *TargetNode = IterLocation()->GetNode(ListBegin);
FEXCore::IR::IROp_Header *TargetOp = TargetNode->Op(DataBegin);
HadError |= TargetOp->Op != OP_BEGINBLOCK;
if (TargetOp->Op != OP_BEGINBLOCK) {
Errors << "Jump " << WrapperOp->ID() << ": Jump to Op that isn't the begining of a block" << std::endl;
}
else {
auto Block = OffsetToBlockMap.try_emplace(IterLocation()->NodeOffset, BlockInfo{}).first;
auto Block = OffsetToBlockMap.try_emplace(IterLocation()->NodeOffset).first;
Block->second.Predecessors.emplace_back(CurrentBlock->Begin);
}
break;
Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
+49 -30
View File
@@ -9,51 +9,70 @@ public:
};
bool RedundantFlagCalculationEliminination::Run(OpDispatchBuilder *Disp) {
std::array<OrderedNode*, 32> LastValidFlagStores{};
bool Changed = false;
auto CurrentIR = Disp->ViewIR();
uintptr_t ListBegin = CurrentIR.GetListData();
uintptr_t DataBegin = CurrentIR.GetData();
IR::NodeWrapperIterator Begin = CurrentIR.begin();
IR::NodeWrapperIterator End = CurrentIR.end();
auto Begin = CurrentIR.begin();
auto Op = Begin();
std::array<OrderedNode*, 32> LastValidFlagStores{};
OrderedNode *RealNode = Op->GetNode(ListBegin);
auto HeaderOp = RealNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
if (IROp->Op == OP_BEGINBLOCK ||
IROp->Op == OP_ENDBLOCK ||
IROp->Op == OP_JUMP ||
IROp->Op == OP_CONDJUMP ||
IROp->Op == OP_EXITFUNCTION) {
// We don't track across block boundaries
LastValidFlagStores.fill(nullptr);
}
while (1) {
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
if (IROp->Op == OP_STOREFLAG) {
auto Op = IROp->CW<IR::IROp_StoreFlag>();
// We grab these nodes this way so we can iterate easily
auto CodeBegin = CurrentIR.at(BlockIROp->Begin);
auto CodeLast = CurrentIR.at(BlockIROp->Last);
while (1) {
auto CodeOp = CodeBegin();
OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
auto IROp = CodeNode->Op(DataBegin);
// If we have had a valid flag store previously and it hasn't been touched until this new store
// Then just delete the old one and let DCE to take care of the rest
if (LastValidFlagStores[Op->Flag] != nullptr) {
Disp->Unlink(LastValidFlagStores[Op->Flag]);
Changed = true;
if (IROp->Op == OP_STOREFLAG) {
auto Op = IROp->CW<IR::IROp_StoreFlag>();
// If we have had a valid flag store previously and it hasn't been touched until this new store
// Then just delete the old one and let DCE to take care of the rest
if (LastValidFlagStores[Op->Flag] != nullptr) {
Disp->Unlink(LastValidFlagStores[Op->Flag]);
Changed = true;
}
// Set this node as the last one valid for this flag
LastValidFlagStores[Op->Flag] = RealNode;
}
else if (IROp->Op == OP_LOADFLAG) {
auto Op = IROp->CW<IR::IROp_LoadFlag>();
// If we loaded a flag then we can't track past this
LastValidFlagStores[Op->Flag] = nullptr;
}
// Set this node as the last one valid for this flag
LastValidFlagStores[Op->Flag] = RealNode;
}
else if (IROp->Op == OP_LOADFLAG) {
auto Op = IROp->CW<IR::IROp_LoadFlag>();
// If we loaded a flag then we can't track past this
LastValidFlagStores[Op->Flag] = nullptr;
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
break;
}
++CodeBegin;
}
++Begin;
if (BlockIROp->Next.ID() == 0) {
break;
} else {
BlockNode = BlockIROp->Next.GetNode(ListBegin);
}
// We don't track across block boundaries
LastValidFlagStores.fill(nullptr);
}
return Changed;
Source/Interface/IR/Passes/RegisterAllocationPass.cpp
+59 -55
View File
@@ -1,15 +1,13 @@
#include "Common/BitSet.h"
#include "Common/Profiler.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include <iterator>
PROFILER_DEFINE(RA, "Passes", "RA", 0);
namespace FEXCore::IR {
constexpr uint32_t INVALID_REG = ~0U;
constexpr uint32_t INVALID_CLASS = ~0U;
constexpr uint32_t DEFAULT_INTERFERENCE_LIST_SIZE = 128;
struct Register {
};
@@ -17,7 +15,7 @@ namespace FEXCore::IR {
struct RegisterClass {
uint32_t RegisterBase;
uint32_t NumberOfRegisters{0};
BitSet<uint32_t> Registers;
BitSet<uint64_t> Registers;
};
struct RegisterAllocationPass::RegisterNode {
@@ -26,7 +24,7 @@ namespace FEXCore::IR {
uint32_t InterferenceCount;
uint32_t InterferenceListSize;
uint32_t *InterferenceList;
BitSet<uint32_t> Interference;
BitSet<uint64_t> Interference;
};
static_assert(std::is_pod<RegisterAllocationPass::RegisterNode>::value, "We want this to be POD");
@@ -96,7 +94,7 @@ namespace FEXCore::IR {
for (uint32_t i = 0; i < NodeCount; ++i) {
Graph->Nodes[i].Register = INVALID_REG;
Graph->Nodes[i].RegisterClass = INVALID_CLASS;
Graph->Nodes[i].InterferenceListSize = 32;
Graph->Nodes[i].InterferenceListSize = DEFAULT_INTERFERENCE_LIST_SIZE;
Graph->Nodes[i].InterferenceList = reinterpret_cast<uint32_t*>(calloc(Graph->Nodes[i].InterferenceListSize, sizeof(uint32_t)));
Graph->Nodes[i].InterferenceCount = 0;
Graph->Nodes[i].Interference.Allocate(NodeCount);
@@ -125,7 +123,7 @@ namespace FEXCore::IR {
for (uint32_t i = OldNodeCount; i < NodeCount; ++i) {
Graph->Nodes[i].Register = INVALID_REG;
Graph->Nodes[i].RegisterClass = INVALID_CLASS;
Graph->Nodes[i].InterferenceListSize = 32;
Graph->Nodes[i].InterferenceListSize = DEFAULT_INTERFERENCE_LIST_SIZE;
Graph->Nodes[i].InterferenceList = reinterpret_cast<uint32_t*>(calloc(Graph->Nodes[i].InterferenceListSize, sizeof(uint32_t)));
Graph->Nodes[i].InterferenceCount = 0;
Graph->Nodes[i].Interference.Allocate(NodeCount);
@@ -165,22 +163,6 @@ namespace FEXCore::IR {
Graph->Nodes[Node].RegisterClass = Class;
}
void RegisterAllocationPass::AddNodeInterference(uint32_t Node1, uint32_t Node2) {
auto AddInterference = [&](uint32_t Node1, uint32_t Node2) {
RegisterNode *Node = &Graph->Nodes[Node1];
Node->Interference.Set(Node2);
if (Node->InterferenceListSize <= Node->InterferenceCount) {
Node->InterferenceListSize *= 2;
Node->InterferenceList = reinterpret_cast<uint32_t*>(realloc(Node->InterferenceList, Node->InterferenceListSize * sizeof(uint32_t)));
}
Node->InterferenceList[Node->InterferenceCount] = Node2;
++Node->InterferenceCount;
};
AddInterference(Node1, Node2);
AddInterference(Node2, Node1);
}
uint32_t RegisterAllocationPass::GetNodeRegister(uint32_t Node) {
return Graph->Nodes[Node].Register;
}
@@ -210,8 +192,8 @@ namespace FEXCore::IR {
while (Begin != End) {
using namespace FEXCore::IR;
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
OrderedNodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
if (IROp->HasDest) {
@@ -266,7 +248,8 @@ namespace FEXCore::IR {
}
}
void RegisterAllocationPass::CalculateLiveRange(IRListView<false> *CurrentIR, uint32_t Nodes) {
void RegisterAllocationPass::CalculateLiveRange(IRListView<false> *CurrentIR) {
size_t Nodes = CurrentIR->GetSSACount();
if (Nodes > LiveRanges.size()) {
LiveRanges.resize(Nodes);
}
@@ -281,14 +264,14 @@ namespace FEXCore::IR {
constexpr uint32_t DEFAULT_REMAT_COST = 1000;
while (Begin != End) {
using namespace FEXCore::IR;
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
OrderedNodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
uint32_t Node = WrapperOp->ID();
// If the destination hasn't yet been set then set it now
if (IROp->HasDest && LiveRanges[Node].Begin == ~0U) {
if (IROp->HasDest) {
LogMan::Throw::A(LiveRanges[Node].Begin == ~0U, "Node begin already defined?");
LiveRanges[Node].Begin = Node;
// Default to ending right where it starts
LiveRanges[Node].End = Node;
@@ -312,13 +295,27 @@ namespace FEXCore::IR {
++Begin;
}
}
void RegisterAllocationPass::CalculateNodeInterference(uint32_t NodeCount) {
auto AddInterference = [&](uint32_t Node1, uint32_t Node2) {
RegisterNode *Node = &Graph->Nodes[Node1];
Node->Interference.Set(Node2);
if (Node->InterferenceListSize <= Node->InterferenceCount) {
Node->InterferenceListSize *= 2;
Node->InterferenceList = reinterpret_cast<uint32_t*>(realloc(Node->InterferenceList, Node->InterferenceListSize * sizeof(uint32_t)));
}
Node->InterferenceList[Node->InterferenceCount] = Node2;
++Node->InterferenceCount;
};
// Now that we have all the live ranges calculated we need to add them to our interference graph
for (uint32_t i = 0; i < Nodes; ++i) {
for (uint32_t j = i + 1; j < Nodes; ++j) {
for (uint32_t i = 0; i < NodeCount; ++i) {
for (uint32_t j = i + 1; j < NodeCount; ++j) {
if (!(LiveRanges[i].Begin >= LiveRanges[j].End ||
LiveRanges[j].Begin >= LiveRanges[i].End)) {
AddNodeInterference(i, j);
AddInterference(i, j);
AddInterference(j, i);
}
}
}
@@ -342,10 +339,10 @@ namespace FEXCore::IR {
if (Reg == ~0U) {
auto RegisterNode = GetRegisterNode(i);
LogMan::Msg::E("\t%%ssa%d with no-RA has live range [%d, %d): Remat cost: %d", i, LiveRanges[i].Begin, LiveRanges[i].End, LiveRanges[i].RematCost);
// LogMan::Msg::E("\t%%ssa%d with no-RA has live range [%d, %d): Remat cost: %d", i, LiveRanges[i].Begin, LiveRanges[i].End, LiveRanges[i].RematCost);
for (uint32_t j = 0; j < RegisterNode->InterferenceCount; ++j) {
uint32_t InterferenceNode = RegisterNode->InterferenceList[j];
LogMan::Msg::E("\t\tInterferes with %%ssa%d: live range[%d, %d): Remat cost: %d", InterferenceNode, LiveRanges[InterferenceNode].Begin, LiveRanges[InterferenceNode].End, LiveRanges[InterferenceNode].RematCost);
// LogMan::Msg::E("\t\tInterferes with %%ssa%d: live range[%d, %d): Remat cost: %d", InterferenceNode, LiveRanges[InterferenceNode].Begin, LiveRanges[InterferenceNode].End, LiveRanges[InterferenceNode].RematCost);
}
Graph->SpillStack.emplace_back(SpillStackUnit{i, CurrentNode->RegisterClass});
}
@@ -375,8 +372,8 @@ namespace FEXCore::IR {
while (Begin != End) {
using namespace FEXCore::IR;
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
OrderedNodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
LogMan::Msg::D("\t\t%%ssa%d Do we have %%ssa%d", WrapperOp->ID(), Node->Wrapped(ListBegin).ID());
@@ -406,8 +403,8 @@ namespace FEXCore::IR {
auto LastCursor = Disp->GetWriteCursor();
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
OrderedNodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
auto Iter = IsInSpillStack(&Graph->SpillStack, WrapperOp->ID());
if (Iter != Graph->SpillStack.end()) {
@@ -424,8 +421,8 @@ namespace FEXCore::IR {
// We want to end the live range of this value here and continue it on first use
auto ConstantRegisterNode = GetRegisterNode(InterferenceNode);
auto *ConstantLiveRange = &LiveRanges[InterferenceNode];
NodeWrapper ConstantOp = NodeWrapper::WrapOffset(InterferenceNode * sizeof(OrderedNode));
OrderedNode *ConstantNode = reinterpret_cast<OrderedNode*>(ConstantOp.GetPtr(ListBegin));
OrderedNodeWrapper ConstantOp = OrderedNodeWrapper::WrapOffset(InterferenceNode * sizeof(OrderedNode));
OrderedNode *ConstantNode = ConstantOp.GetNode(ListBegin);
FEXCore::IR::IROp_Constant const *ConstantIROp = ConstantNode->Op(DataBegin)->C<IR::IROp_Constant>();
LogMan::Throw::A(ConstantIROp->Header.Op == OP_CONSTANT, "This needs to be const");
@@ -435,8 +432,8 @@ namespace FEXCore::IR {
auto FirstUseLocation = FindFirstUse(Disp, ConstantNode, NextIter, End);
if (FirstUseLocation != End) {
// LogMan::Throw::A(FirstUseLocation != End, "Failure to find op use");
NodeWrapper *FirstUseOp = FirstUseLocation();
OrderedNode *FirstUseOrderedNode = reinterpret_cast<OrderedNode*>(FirstUseOp->GetPtr(ListBegin));
OrderedNodeWrapper *FirstUseOp = FirstUseLocation();
OrderedNode *FirstUseOrderedNode = FirstUseOp->GetNode(ListBegin);
Disp->SetWriteCursor(FirstUseOrderedNode);
auto FilledConstant = Disp->_Constant(ConstantIROp->Constant);
Disp->ReplaceAllUsesWithInclusive(ConstantNode, FilledConstant, FirstUseLocation, End);
@@ -507,13 +504,13 @@ namespace FEXCore::IR {
auto *InterferenceLiveRange = &LiveRanges[InterferenceNode];
// If the interference's live range is past this op's live range then we can dump it
if (1) {
NodeWrapper InterferenceOp = NodeWrapper::WrapOffset(InterferenceNode * sizeof(OrderedNode));
OrderedNode *InterferenceOrderedNode = reinterpret_cast<OrderedNode*>(InterferenceOp.GetPtr(ListBegin));
OrderedNodeWrapper InterferenceOp = OrderedNodeWrapper::WrapOffset(InterferenceNode * sizeof(OrderedNode));
OrderedNode *InterferenceOrderedNode = InterferenceOp.GetNode(ListBegin);
FEXCore::IR::IROp_Header *InterferenceIROp = InterferenceOrderedNode->Op(DataBegin);
auto PrevIter = Begin;
--PrevIter;
Disp->SetWriteCursor(reinterpret_cast<OrderedNode*>(PrevIter()->GetPtr(ListBegin)));
Disp->SetWriteCursor(PrevIter()->GetNode(ListBegin));
auto SpillOp = Disp->_SpillRegister(InterferenceOrderedNode, SpillSlotCount, {InterferenceRegisterNode->RegisterClass});
SpillOp.first->Header.Size = InterferenceIROp->Size;
SpillOp.first->Header.Elements = InterferenceIROp->Elements;
@@ -525,8 +522,8 @@ namespace FEXCore::IR {
auto FirstUseLocation = FindFirstUse(Disp, InterferenceOrderedNode, NextIter, End);
if (FirstUseLocation != End) {
// LogMan::Throw::A(FirstUseLocation != End, "Failure to find op use");
NodeWrapper *FirstUseOp = FirstUseLocation();
OrderedNode *FirstUseOrderedNode = reinterpret_cast<OrderedNode*>(FirstUseOp->GetPtr(ListBegin));
OrderedNodeWrapper *FirstUseOp = FirstUseLocation();
OrderedNode *FirstUseOrderedNode = FirstUseOp->GetNode(ListBegin);
Disp->SetWriteCursor(FirstUseOrderedNode);
auto FilledInterference = Disp->_FillRegister(SpillSlotCount, {InterferenceRegisterNode->RegisterClass});
@@ -558,32 +555,39 @@ namespace FEXCore::IR {
}
bool RegisterAllocationPass::Run(OpDispatchBuilder *Disp) {
PROFILER_SCOPE(RA);
PROFILER_COUNTER_ADD("RA/counter", 1);
bool Changed = false;
constexpr uint32_t RATries = 1;
SpillSlotCount = 0;
HasSpills = false;
HadFullRA = false;
for (uint32_t i = 0; i < RATries; ++i) {
auto CurrentIR = Disp->ViewIR();
uintptr_t ListSize = CurrentIR.GetListSize();
uint32_t SSACount = ListSize / sizeof(IR::OrderedNode);
uint32_t SSACount = CurrentIR.GetSSACount();
ResetRegisterGraph(SSACount);
FindNodeClasses(&CurrentIR);
CalculateLiveRange(&CurrentIR, SSACount);
CalculateLiveRange(&CurrentIR);
CalculateNodeInterference(SSACount);
AllocateRegisters();
if (!Graph->SpillStack.empty()) {
Disp->ShouldDump = true;
Changed = true;
//ClearSpillList(Disp);
if (Config_SupportsSpills) {
Disp->ShouldDump = true;
Changed = true;
ClearSpillList(Disp);
}
else {
HadFullRA = false;
SpillSlotCount = 0;
}
return Changed;
}
else {
// We managed to RA, leave now
HadFullRA = true;
Disp->ShouldDump = false;
return Changed;
}
Source/Interface/IR/Passes/RegisterAllocationPass.h
+8 -2
View File
@@ -30,16 +30,18 @@ public:
void FreeRegisterGraph();
void ResetRegisterGraph(uint32_t NodeCount);
void SetNodeClass(uint32_t Node, uint32_t Class);
void AddNodeInterference(uint32_t Node1, uint32_t Node2);
uint32_t GetNodeRegister(uint32_t Node);
void AllocateRegisters();
/** @} */
bool HasFullRA() const { return HadFullRA; }
bool HadSpills() const { return HasSpills; }
uint32_t SpillSlots() const { return SpillSlotCount; }
void SetSupportsSpills(bool Supports) { Config_SupportsSpills = Supports; }
private:
RegisterGraph *Graph;
void FindNodeClasses(IRListView<false> *CurrentIR);
@@ -52,7 +54,8 @@ private:
std::vector<LiveRange> LiveRanges;
void CalculateLiveRange(IRListView<false> *CurrentIR, uint32_t Nodes);
void CalculateLiveRange(IRListView<false> *CurrentIR);
void CalculateNodeInterference(uint32_t NodeCount);
void ClearSpillList(OpDispatchBuilder *Disp);
@@ -61,6 +64,9 @@ private:
bool HasSpills {};
uint32_t SpillSlotCount {};
bool HadFullRA {};
bool Config_SupportsSpills {true};
};
}
Source/Interface/IR/Passes/SyscallOptimization.cpp
+40 -13
View File
@@ -16,24 +16,51 @@ bool SyscallOptimization::Run(OpDispatchBuilder *Disp) {
uintptr_t ListBegin = CurrentIR.GetListData();
uintptr_t DataBegin = CurrentIR.GetData();
IR::NodeWrapperIterator Begin = CurrentIR.begin();
IR::NodeWrapperIterator End = CurrentIR.end();
auto Begin = CurrentIR.begin();
auto Op = Begin();
while (Begin != End) {
NodeWrapper *WrapperOp = Begin();
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(WrapperOp->GetPtr(ListBegin));
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
OrderedNode *RealNode = Op->GetNode(ListBegin);
auto HeaderOp = RealNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
while (1) {
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
// We grab these nodes this way so we can iterate easily
auto CodeBegin = CurrentIR.at(BlockIROp->Begin);
auto CodeLast = CurrentIR.at(BlockIROp->Last);
while (1) {
auto CodeOp = CodeBegin();
OrderedNode *CodeNode = CodeOp->GetNode(ListBegin);
auto IROp = CodeNode->Op(DataBegin);
if (IROp->Op == FEXCore::IR::OP_SYSCALL) {
Disp->ShouldDump = true;
// Is the first argument a constant?
uint64_t Constant;
if (Disp->IsValueConstant(IROp->Args[0], &Constant)) {
LogMan::Msg::D("Whoa. Syscall argument is constant: %ld", Constant);
Changed = true;
}
if (IROp->Op == FEXCore::IR::OP_SYSCALL) {
// Is the first argument a constant?
uint64_t Constant;
if (Disp->IsValueConstant(IROp->Args[0], &Constant)) {
// LogMan::Msg::A("Whoa. Syscall argument is constant: %ld", Constant);
Changed = true;
}
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
if (CodeBegin == CodeLast) {
break;
}
++CodeBegin;
}
++Begin;
if (BlockIROp->Next.ID() == 0) {
break;
} else {
BlockNode = BlockIROp->Next.GetNode(ListBegin);
}
}
return Changed;
include/FEXCore/Core/CodeLoader.h
+3
View File
@@ -34,6 +34,9 @@ public:
*/
virtual uint64_t DefaultRIP() const = 0;
virtual void GetInitLocations(std::vector<uint64_t> *Locations) {}
virtual uint64_t InitializeThreadSlot(std::function<void(void const*, uint64_t)> Writer) const { return 0; };
using MemoryLayout = std::tuple<uint64_t, uint64_t, uint64_t>;
/**
* @brief Gets the default memory layout of the memory object being loaded
include/FEXCore/IR/IR.h
+67 -62
View File
@@ -6,8 +6,19 @@
namespace FEXCore::IR {
/**
* @brief The IROp_Header is an dynamically sized array
* At the end it contains a uint8_t for the number of arguments that Op has
* Then there is an unsized array of NodeWrapper arguments for the number of arguments this op has
* The op structures that are including the header must ensure that they pad themselves correctly to the number of arguments used
*/
struct IROp_Header;
class OrderedNode;
/**
* @brief This is a very simple wrapper for our node pointers
* You probably don't want to use this directly
* Use OpNodeWrapper and OrderedNodeWrapper types below instead
*
* This is necessary to allow two things
* - Reduce memory usage by having the pointer be an 32bit offset rather than the whole 64bit pointer
@@ -19,46 +30,53 @@ namespace FEXCore::IR {
* - We have to have the base offset live somewhere else
* - Has to be POD and trivially copyable
* - Makes every real node access turn in to a [Base + Offset] access
* - Can be confusing if you're mixing OpNodeWrapper and OrderedNodeWrapper usage
*/
struct NodeWrapper final {
template<typename Type>
struct NodeWrapperBase final {
// On x86-64 using a uint64_t type is more efficient since RIP addressing gives you [<Base> + <Index> + <imm offset>]
// On AArch64 using uint32_t is just more memory efficient. 32bit or 64bit offset doesn't matter
// We use uint32_t to be more memory efficient (Cuts our node list size in half)
using NodeOffsetType = uint32_t;
NodeOffsetType NodeOffset;
static NodeWrapper WrapOffset(NodeOffsetType Offset) {
NodeWrapper Wrapped;
static NodeWrapperBase WrapOffset(NodeOffsetType Offset) {
NodeWrapperBase Wrapped;
Wrapped.NodeOffset = Offset;
return Wrapped;
}
static NodeWrapper WrapPtr(uintptr_t Base, uintptr_t Value) {
NodeWrapper Wrapped;
static NodeWrapperBase WrapPtr(uintptr_t Base, uintptr_t Value) {
NodeWrapperBase Wrapped;
Wrapped.SetOffset(Base, Value);
return Wrapped;
}
static void *UnwrapNode(uintptr_t Base, NodeWrapper Node) {
return Node.GetPtr(Base);
static void *UnwrapNode(uintptr_t Base, NodeWrapperBase Node) {
return Node.GetNode(Base);
}
uint32_t ID() const;
explicit NodeWrapper() = default;
void *GetPtr(uintptr_t Base) { return reinterpret_cast<void*>(Base + NodeOffset); }
void const *GetPtr(uintptr_t Base) const { return reinterpret_cast<void*>(Base + NodeOffset); }
explicit NodeWrapperBase() = default;
Type *GetNode(uintptr_t Base) { return reinterpret_cast<Type*>(Base + NodeOffset); }
Type const *GetNode(uintptr_t Base) const { return reinterpret_cast<Type*>(Base + NodeOffset); }
void SetOffset(uintptr_t Base, uintptr_t Value) { NodeOffset = Value - Base; }
bool operator==(NodeWrapper const &rhs) { return NodeOffset == rhs.NodeOffset; }
bool operator==(NodeWrapperBase const &rhs) { return NodeOffset == rhs.NodeOffset; }
};
static_assert(std::is_pod<NodeWrapper>::value);
static_assert(sizeof(NodeWrapper) == sizeof(uint32_t));
static_assert(std::is_pod<NodeWrapperBase<OrderedNode>>::value);
static_assert(sizeof(NodeWrapperBase<OrderedNode>) == sizeof(uint32_t));
using OpNodeWrapper = NodeWrapperBase<IROp_Header>;
using OrderedNodeWrapper = NodeWrapperBase<OrderedNode>;
struct OrderedNodeHeader {
NodeWrapper Value;
NodeWrapper Next;
NodeWrapper Previous;
OpNodeWrapper Value;
OrderedNodeWrapper Next;
OrderedNodeWrapper Previous;
};
static_assert(sizeof(OrderedNodeHeader) == sizeof(uint32_t) * 3);
@@ -70,7 +88,7 @@ static_assert(sizeof(OrderedNodeHeader) == sizeof(uint32_t) * 3);
*/
class NodeWrapperIterator final {
public:
using value_type = NodeWrapper;
using value_type = OrderedNodeWrapper;
using size_type = std::size_t;
using difference_type = std::ptrdiff_t;
using reference = value_type&;
@@ -83,9 +101,9 @@ public:
using const_reverse_iterator = const_iterator;
using iterator_category = std::bidirectional_iterator_tag;
using NodeType = NodeWrapper;
using NodePtr = NodeWrapper*;
using NodeRef = NodeWrapper&;
using NodeType = value_type;
using NodePtr = value_type*;
using NodeRef = value_type&;
NodeWrapperIterator(uintptr_t Base) : BaseList {Base} {}
explicit NodeWrapperIterator(uintptr_t Base, NodeType Ptr) : BaseList {Base}, Node {Ptr} {}
@@ -99,13 +117,13 @@ public:
}
NodeWrapperIterator operator++() {
OrderedNodeHeader *RealNode = reinterpret_cast<OrderedNodeHeader*>(Node.GetPtr(BaseList));
OrderedNodeHeader *RealNode = reinterpret_cast<OrderedNodeHeader*>(Node.GetNode(BaseList));
Node = RealNode->Next;
return *this;
}
NodeWrapperIterator operator--() {
OrderedNodeHeader *RealNode = reinterpret_cast<OrderedNodeHeader*>(Node.GetPtr(BaseList));
OrderedNodeHeader *RealNode = reinterpret_cast<OrderedNodeHeader*>(Node.GetNode(BaseList));
Node = RealNode->Previous;
return *this;
}
@@ -123,14 +141,6 @@ private:
NodeType Node{};
};
/**
* @brief The IROp_Header is an dynamically sized array
* At the end it contains a uint8_t for the number of arguments that Op has
* Then there is an unsized array of NodeWrapper arguments for the number of arguments this op has
* The op structures that are including the header must ensure that they pad themselves correctly to the number of arguments used
*/
struct IROp_Header;
/**
* @brief This is a node in our IR representation
* Is a doubly linked list node that lives in a representation of a linearly allocated node list
@@ -153,6 +163,8 @@ class OrderedNode final {
OrderedNodeHeader Header;
uint32_t NumUses;
using value_type = OrderedNodeWrapper;
OrderedNode() = default;
/**
@@ -163,7 +175,7 @@ class OrderedNode final {
*
* @return Pointer to the node being added
*/
NodeWrapper append(uintptr_t Base, NodeWrapper Node) {
value_type append(uintptr_t Base, value_type Node) {
// Set Next Node's Previous to incoming node
SetPrevious(Base, Header.Next, Node);
@@ -179,7 +191,7 @@ class OrderedNode final {
}
OrderedNode *append(uintptr_t Base, OrderedNode *Node) {
NodeWrapper WNode = Node->Wrapped(Base);
value_type WNode = Node->Wrapped(Base);
// Set Next Node's Previous to incoming node
SetPrevious(Base, Header.Next, WNode);
@@ -201,7 +213,7 @@ class OrderedNode final {
*
* @return Pointer to the node being added
*/
NodeWrapper prepend(uintptr_t Base, NodeWrapper Node) {
value_type prepend(uintptr_t Base, value_type Node) {
// Set the previous node's next to the incoming node
SetNext(Base, Header.Previous, Node);
@@ -217,7 +229,7 @@ class OrderedNode final {
}
OrderedNode *prepend(uintptr_t Base, OrderedNode *Node) {
NodeWrapper WNode = Node->Wrapped(Base);
value_type WNode = Node->Wrapped(Base);
// Set the previous node's next to the incoming node
SetNext(Base, Header.Previous, WNode);
@@ -241,10 +253,10 @@ class OrderedNode final {
size_t size(uintptr_t Base) const {
size_t Size = 1;
// Walk the list forward until we hit a sentinal
NodeWrapper Current = Header.Next;
value_type Current = Header.Next;
while (Current.NodeOffset != 0) {
++Size;
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(Current.GetPtr(Base));
OrderedNode *RealNode = Current.GetNode(Base);
Current = RealNode->Header.Next;
}
return Size;
@@ -258,41 +270,36 @@ class OrderedNode final {
SetPrevious(Base, Header.Next, Header.Previous);
}
IROp_Header const* Op(uintptr_t Base) const { return reinterpret_cast<IROp_Header const*>(Header.Value.GetPtr(Base)); }
IROp_Header *Op(uintptr_t Base) { return reinterpret_cast<IROp_Header*>(Header.Value.GetPtr(Base)); }
IROp_Header const* Op(uintptr_t Base) const { return Header.Value.GetNode(Base); }
IROp_Header *Op(uintptr_t Base) { return Header.Value.GetNode(Base); }
uint32_t GetUses() const { return NumUses; }
void AddUse() { ++NumUses; }
void RemoveUse() { --NumUses; }
using iterator = NodeWrapperIterator;
iterator begin(uint64_t Base) noexcept { return iterator(Base, Wrapped(Base)); }
iterator end(uint64_t Base, uint64_t End) noexcept { return iterator(Base, WrappedOffset(End)); }
NodeWrapper Wrapped(uintptr_t Base) {
NodeWrapper Tmp;
value_type Wrapped(uintptr_t Base) {
value_type Tmp;
Tmp.SetOffset(Base, reinterpret_cast<uintptr_t>(this));
return Tmp;
}
private:
NodeWrapper WrappedOffset(uint32_t Offset) {
NodeWrapper Tmp;
value_type WrappedOffset(uint32_t Offset) {
value_type Tmp;
Tmp.NodeOffset = Offset;
return Tmp;
}
static void SetPrevious(uintptr_t Base, NodeWrapper Node, NodeWrapper New) {
static void SetPrevious(uintptr_t Base, value_type Node, value_type New) {
if (Node.NodeOffset == 0) return;
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(Node.GetPtr(Base));
OrderedNode *RealNode = Node.GetNode(Base);
RealNode->Header.Previous = New;
}
static void SetNext(uintptr_t Base, NodeWrapper Node, NodeWrapper New) {
static void SetNext(uintptr_t Base, value_type Node, value_type New) {
if (Node.NodeOffset == 0) return;
OrderedNode *RealNode = reinterpret_cast<OrderedNode*>(Node.GetPtr(Base));
OrderedNode *RealNode = Node.GetNode(Base);
RealNode->Header.Next = New;
}
@@ -304,26 +311,24 @@ static_assert(std::is_trivially_copyable<OrderedNode>::value);
static_assert(offsetof(OrderedNode, Header) == 0);
static_assert(sizeof(OrderedNode) == (sizeof(OrderedNodeHeader) + sizeof(uint32_t)));
struct RegisterClassType final {
uint32_t Val;
operator uint32_t() {
return Val;
}
};
#define IROP_ENUM
#define IROP_STRUCTS
#define IROP_SIZES
#include "IRDefines.inc"
template <class T>
struct Wrapper final {
T *first;
OrderedNode *Node; ///< Actual offset of this IR in ths list
operator Wrapper<IROp_Header>() const { return Wrapper<IROp_Header> {reinterpret_cast<IROp_Header*>(first), Node}; }
operator OrderedNode *() { return Node; }
operator NodeWrapper () { return Node->Header.Value; }
};
template<bool>
class IRListView;
void Dump(std::stringstream *out, IRListView<false> const* IR);
inline uint32_t NodeWrapper::ID() const { return NodeOffset / sizeof(IR::OrderedNode); }
template<typename Type>
inline uint32_t NodeWrapperBase<Type>::ID() const { return NodeOffset / sizeof(IR::OrderedNode); }
};
include/FEXCore/IR/IntrusiveIRList.h
+13 -4
View File
@@ -22,7 +22,7 @@ class IntrusiveAllocator final {
IntrusiveAllocator(IntrusiveAllocator &&) = delete;
IntrusiveAllocator(size_t Size)
: MemorySize {Size} {
Data = reinterpret_cast<uintptr_t>(calloc(Size, 1));
Data = reinterpret_cast<uintptr_t>(malloc(Size));
}
~IntrusiveAllocator() {
@@ -35,7 +35,7 @@ class IntrusiveAllocator final {
}
void *Allocate(size_t Size) {
assert(CheckSize(Size) && "Failure");
LogMan::Throw::A(CheckSize(Size), "Ran out of space in IntrusiveAllocator during allocation");
size_t NewOffset = CurrentOffset + Size;
uintptr_t NewPointer = Data + CurrentOffset;
CurrentOffset = NewOffset;
@@ -95,12 +95,13 @@ public:
size_t GetDataSize() const { return DataSize; }
size_t GetListSize() const { return ListSize; }
size_t GetSSACount() const { return ListSize / sizeof(OrderedNode); }
using iterator = NodeWrapperIterator;
iterator begin() const noexcept
{
NodeWrapper Wrapped;
OrderedNodeWrapper Wrapped;
Wrapped.NodeOffset = sizeof(OrderedNode);
return iterator(reinterpret_cast<uintptr_t>(ListData), Wrapped);
}
@@ -112,11 +113,19 @@ public:
*/
iterator end() const noexcept
{
NodeWrapper Wrapped;
OrderedNodeWrapper Wrapped;
Wrapped.NodeOffset = 0;
return iterator(reinterpret_cast<uintptr_t>(ListData), Wrapped);
}
/**
* @brief Convert a OrderedNodeWrapper to an interator that we can iterate over
* @return Iterator for this op
*/
iterator at(OrderedNodeWrapper Node) const noexcept {
return iterator(reinterpret_cast<uintptr_t>(ListData), Node);
}
private:
void *IRData;
void *ListData;