steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-14 13:00:41 +02:00
Code Issues Packages Projects Releases 67 Wiki Activity

Merge pull request #722 from FEX-Emu/skmp/ra-data

Browse Source 
RA: Split RAData from RAPass, Optimize data packing, Cleanups
This commit is contained in:
Ryan Houdek authored and GitHub committed 2021-01-29 05:16:18 -08:00
2 parents 810143d304 972219043d
commit f23fe030a1
23 files changed
+506 -433
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

External/FEXCore/Scripts/json_ir_generator.py
Vendored
+1 -1
View File
@@ -236,7 +236,7 @@ def print_ir_arg_printer(ops, defines):
if (SSAArgs != 0):
for i in range(0, SSAArgs):
LastArg = (SSAArgs - i - 1) == 0 and not HasArgs
output_file.write("\tPrintArg(out, IR, Op->Header.Args[%d], RAPass);\n" % i)
output_file.write("\tPrintArg(out, IR, Op->Header.Args[%d], RAData);\n" % i)
if not (LastArg):
output_file.write("\t*out << \", \";\n")
External/FEXCore/Source/Interface/Context/Context.h
Vendored
+4 -2
View File
@@ -139,7 +139,9 @@ namespace FEXCore::Context {
FEXCore::Core::ThreadState *GetThreadState();
void LoadEntryList();
std::tuple<void *, FEXCore::Core::DebugData *> CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
std::tuple<FEXCore::IR::IRListView<true> *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t> GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
std::tuple<void *, FEXCore::IR::IRListView<true> *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool> CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
uintptr_t CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
uintptr_t CompileFallbackBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
@@ -158,7 +160,7 @@ namespace FEXCore::Context {
#endif
protected:
void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache);
private:
void WaitForIdleWithTimeout();
External/FEXCore/Source/Interface/Core/CompileService.cpp
Vendored
+17 -30
View File
@@ -33,7 +33,7 @@ namespace FEXCore {
WorkerThread.join();
}
void CompileService::ClearCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
void CompileService::ClearCache(FEXCore::Core::InternalThreadState *Thread) {
// On cache clear we need to spin down the execution thread to ensure it isn't trying to give us more work items
if (CompileMutex.try_lock()) {
// We can only clear these things if we pulled the compile mutex
@@ -57,14 +57,9 @@ namespace FEXCore {
}
}
if (GuestRIP == 0) {
CompileThreadData->IRLists.clear();
}
else {
auto IR = CompileThreadData->IRLists.find(GuestRIP)->second.release();
CompileThreadData->IRLists.clear();
CompileThreadData->IRLists.try_emplace(GuestRIP, IR);
}
LogMan::Throw::A(CompileThreadData->IRLists.size() == 0, "Compile service must never have IRLists");
LogMan::Throw::A(CompileThreadData->RALists.size() == 0, "Compile service must never have RALists");
LogMan::Throw::A(CompileThreadData->DebugData.size() == 0, "Compile service must never have DebugData");
CompileMutex.unlock();
}
@@ -76,11 +71,6 @@ namespace FEXCore {
SelectedThread->IntBackend->ClearCache();
}
void CompileService::RemoveCodeEntry(uint64_t GuestRIP) {
CompileThreadData->IRLists.erase(GuestRIP);
CompileThreadData->DebugData.erase(GuestRIP);
CompileThreadData->LookupCache->Erase(GuestRIP);
}
CompileService::WorkItem *CompileService::CompileCode(uint64_t RIP) {
// Tell the worker thread to compile code for us
@@ -132,29 +122,26 @@ namespace FEXCore {
// If we had a work item then work on it
if (Item) {
// Does the block cache already contain this RIP?
void *CompiledCode = reinterpret_cast<void*>(CompileThreadData->LookupCache->FindBlock(Item->RIP));
FEXCore::Core::DebugData *DebugData = nullptr;
// Make sure it's not in lookup cache by accident
LogMan::Throw::A(CompileThreadData->LookupCache->FindBlock(Item->RIP) == 0, "Compile Service must never have entries in the LookupCache");
// Code isn't in cache, compile now
// Set our thread state's RIP
CompileThreadData->State.State.rip = Item->RIP;
if (!CompiledCode) {
// Code isn't in cache, compile now
// Set our thread state's RIP
CompileThreadData->State.State.rip = Item->RIP;
auto [Code, Data] = CTX->CompileCode(CompileThreadData.get(), Item->RIP);
CompiledCode = Code;
DebugData = Data;
}
auto [CodePtr, IRList, DebugData, RAData, Generated] = CTX->CompileCode(CompileThreadData.get(), Item->RIP);
if (!CompiledCode) {
LogMan::Throw::A(Generated == true, "Compile Service doesn't have IR Cache");
if (!CodePtr) {
// XXX: We currently have the expectation that compile service code will be significantly smaller than regular thread's code
ERROR_AND_DIE("Couldn't compile code for thread at RIP: 0x%lx", Item->RIP);
}
CompileThreadData->LookupCache->AddBlockMapping(Item->RIP, CompiledCode);
Item->CodePtr = CompiledCode;
Item->IRList = CompileThreadData->IRLists.find(Item->RIP)->second.get();
Item->CodePtr = CodePtr;
Item->IRList = IRList;
Item->DebugData = DebugData;
Item->RAData = RAData;
GCArray.emplace_back(Item);
Item->ServiceWorkDone.NotifyAll();
External/FEXCore/Source/Interface/Core/CompileService.h
Vendored
+5 -2
View File
@@ -16,6 +16,9 @@ namespace Core {
struct InternalThreadState;
}
namespace IR {
class RegisterAllocationData;
};
class CompileService final {
public:
CompileService(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread);
@@ -29,6 +32,7 @@ class CompileService final {
// Outgoing
void *CodePtr{};
FEXCore::IR::IRListView<true> *IRList{};
FEXCore::IR::RegisterAllocationData *RAData{};
FEXCore::Core::DebugData *DebugData{};
// Communication
@@ -37,8 +41,7 @@ class CompileService final {
};
WorkItem *CompileCode(uint64_t RIP);
void ClearCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
void RemoveCodeEntry(uint64_t GuestRIP);
void ClearCache(FEXCore::Core::InternalThreadState *Thread);
private:
FEXCore::Context::Context *CTX;
External/FEXCore/Source/Interface/Core/Core.cpp
Vendored
+237 -204
View File
@@ -111,7 +111,7 @@ namespace DefaultFallbackCore {
void Initialize() override {}
bool NeedsOpDispatch() override { return false; }
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) override {
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override {
LogMan::Msg::E("Fell back to default code handler at RIP: 0x%lx", ThreadState->State.State.rip);
return nullptr;
}
@@ -378,7 +378,7 @@ namespace FEXCore::Context {
// Walk the threads and tell them to clear their caches
// Useful when our block size is set to a large number and we need to step a single instruction
for (auto &Thread : Threads) {
ClearCodeCache(Thread, 0);
ClearCodeCache(Thread, true);
}
}
CoreRunningMode PreviousRunningMode = this->Config.RunningMode;
@@ -464,6 +464,8 @@ namespace FEXCore::Context {
// Run the passmanager over the IR from the dispatcher
Thread->PassManager->Run(IR);
Thread->IRLists.try_emplace(Addr, IR->CreateIRCopy());
Thread->DebugData.try_emplace(Addr, new Core::DebugData());
Thread->RALists.try_emplace(Addr, Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->PullAllocationData() : nullptr);
};
LocalLoader->AddIR(IRHandler);
@@ -563,239 +565,250 @@ namespace FEXCore::Context {
Thread->LookupCache->AddBlockMapping(Address, Ptr);
}
void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache) {
Thread->LookupCache->ClearCache();
Thread->CPUBackend->ClearCache();
Thread->IntBackend->ClearCache();
if (Thread->CompileService) {
Thread->CompileService->ClearCache(Thread, GuestRIP);
}
if (GuestRIP == 0) {
if (AlsoClearIRCache) {
Thread->IRLists.clear();
}
else {
auto IR = Thread->IRLists.find(GuestRIP)->second.release();
Thread->IRLists.clear();
Thread->IRLists.try_emplace(GuestRIP, IR);
Thread->RALists.clear();
Thread->DebugData.clear();
}
}
std::tuple<void *, FEXCore::Core::DebugData *> Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
std::tuple<FEXCore::IR::IRListView<true> *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t> Context::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
uint8_t const *GuestCode{};
GuestCode = reinterpret_cast<uint8_t const*>(GuestRIP);
GuestCode = reinterpret_cast<uint8_t const*>(GuestRIP);
// Do we already have this in the IR cache?
auto IR = Thread->IRLists.find(GuestRIP);
FEXCore::IR::IRListView<true> *IRList {};
FEXCore::Core::DebugData *DebugData {};
bool HadDispatchError {false};
if (IR == Thread->IRLists.end()) {
bool HadDispatchError {false};
uint64_t TotalInstructions {0};
uint64_t TotalInstructionsLength {0};
uint64_t TotalInstructions {0};
uint64_t TotalInstructionsLength {0};
if (!Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP)) {
if (Config.BreakOnFrontendFailure) {
LogMan::Msg::E("Had Frontend decoder error");
Stop(false /* Ignore Current Thread */);
}
return { nullptr, nullptr, 0, 0 };
}
if (!Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP)) {
if (Config.BreakOnFrontendFailure) {
LogMan::Msg::E("Had Frontend decoder error");
Stop(false /* Ignore Current Thread */);
}
return { nullptr, nullptr };
auto CodeBlocks = Thread->FrontendDecoder->GetDecodedBlocks();
Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks);
for (size_t j = 0; j < CodeBlocks->size(); ++j) {
FEXCore::Frontend::Decoder::DecodedBlocks const &Block = CodeBlocks->at(j);
// Set the block entry point
Thread->OpDispatcher->SetNewBlockIfChanged(Block.Entry);
uint64_t BlockInstructionsLength {};
// Reset any block-specific state
Thread->OpDispatcher->StartNewBlock();
uint64_t InstsInBlock = Block.NumInstructions;
if (Block.HasInvalidInstruction) {
uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry));
break;
}
auto CodeBlocks = Thread->FrontendDecoder->GetDecodedBlocks();
for (size_t i = 0; i < InstsInBlock; ++i) {
FEXCore::X86Tables::X86InstInfo const* TableInfo {nullptr};
FEXCore::X86Tables::DecodedInst const* DecodedInfo {nullptr};
Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks);
TableInfo = Block.DecodedInstructions[i].TableInfo;
DecodedInfo = &Block.DecodedInstructions[i];
for (size_t j = 0; j < CodeBlocks->size(); ++j) {
FEXCore::Frontend::Decoder::DecodedBlocks const &Block = CodeBlocks->at(j);
// Set the block entry point
Thread->OpDispatcher->SetNewBlockIfChanged(Block.Entry);
if (Config.SMCChecks) {
auto ExistingCodePtr = reinterpret_cast<uint64_t*>(Block.Entry + BlockInstructionsLength);
auto CodeChanged = Thread->OpDispatcher->_ValidateCode(ExistingCodePtr[0], ExistingCodePtr[1], (uintptr_t)ExistingCodePtr, DecodedInfo->InstSize);
uint64_t BlockInstructionsLength {};
auto InvalidateCodeCond = Thread->OpDispatcher->_CondJump(CodeChanged);
// Reset any block-specific state
Thread->OpDispatcher->StartNewBlock();
auto CurrentBlock = Thread->OpDispatcher->GetCurrentBlock();
auto CodeWasChangedBlock = Thread->OpDispatcher->CreateNewCodeBlockAtEnd();
Thread->OpDispatcher->SetTrueJumpTarget(InvalidateCodeCond, CodeWasChangedBlock);
uint64_t InstsInBlock = Block.NumInstructions;
Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock);
Thread->OpDispatcher->_RemoveCodeEntry(GuestRIP);
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(Block.Entry + BlockInstructionsLength));
auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
if (Block.HasInvalidInstruction) {
uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry));
break;
Thread->OpDispatcher->SetFalseJumpTarget(InvalidateCodeCond, NextOpBlock);
Thread->OpDispatcher->SetCurrentCodeBlock(NextOpBlock);
}
for (size_t i = 0; i < InstsInBlock; ++i) {
FEXCore::X86Tables::X86InstInfo const* TableInfo {nullptr};
FEXCore::X86Tables::DecodedInst const* DecodedInfo {nullptr};
TableInfo = Block.DecodedInstructions[i].TableInfo;
DecodedInfo = &Block.DecodedInstructions[i];
if (Config.SMCChecks) {
auto ExistingCodePtr = reinterpret_cast<uint64_t*>(Block.Entry + BlockInstructionsLength);
auto CodeChanged = Thread->OpDispatcher->_ValidateCode(ExistingCodePtr[0], ExistingCodePtr[1], (uintptr_t)ExistingCodePtr, DecodedInfo->InstSize);
auto InvalidateCodeCond = Thread->OpDispatcher->_CondJump(CodeChanged);
auto CurrentBlock = Thread->OpDispatcher->GetCurrentBlock();
auto CodeWasChangedBlock = Thread->OpDispatcher->CreateNewCodeBlockAtEnd();
Thread->OpDispatcher->SetTrueJumpTarget(InvalidateCodeCond, CodeWasChangedBlock);
Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock);
Thread->OpDispatcher->_RemoveCodeEntry(GuestRIP);
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(Block.Entry + BlockInstructionsLength));
auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
Thread->OpDispatcher->SetFalseJumpTarget(InvalidateCodeCond, NextOpBlock);
Thread->OpDispatcher->SetCurrentCodeBlock(NextOpBlock);
}
if (TableInfo->OpcodeDispatcher) {
auto Fn = TableInfo->OpcodeDispatcher;
std::invoke(Fn, Thread->OpDispatcher, DecodedInfo);
if (Thread->OpDispatcher->HadDecodeFailure()) {
if (Config.BreakOnFrontendFailure) {
LogMan::Msg::E("Had OpDispatcher error at 0x%lx", GuestRIP);
Stop(false /* Ignore Current Thread */);
}
HadDispatchError = true;
if (TableInfo->OpcodeDispatcher) {
auto Fn = TableInfo->OpcodeDispatcher;
std::invoke(Fn, Thread->OpDispatcher, DecodedInfo);
if (Thread->OpDispatcher->HadDecodeFailure()) {
if (Config.BreakOnFrontendFailure) {
LogMan::Msg::E("Had OpDispatcher error at 0x%lx", GuestRIP);
Stop(false /* Ignore Current Thread */);
}
else {
BlockInstructionsLength += DecodedInfo->InstSize;
TotalInstructionsLength += DecodedInfo->InstSize;
++TotalInstructions;
}
}
else {
LogMan::Msg::E("Missing OpDispatcher at 0x%lx{'%s'}", Block.Entry + BlockInstructionsLength, TableInfo->Name);
HadDispatchError = true;
}
// If we had a dispatch error then leave early
if (HadDispatchError) {
if (TotalInstructions == 0) {
// Couldn't handle any instruction in op dispatcher
Thread->OpDispatcher->ResetWorkingList();
return { nullptr, nullptr };
}
else {
uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
// We had some instructions. Early exit
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry + BlockInstructionsLength));
break;
}
else {
BlockInstructionsLength += DecodedInfo->InstSize;
TotalInstructionsLength += DecodedInfo->InstSize;
++TotalInstructions;
}
}
else {
LogMan::Msg::E("Missing OpDispatcher at 0x%lx{'%s'}", Block.Entry + BlockInstructionsLength, TableInfo->Name);
HadDispatchError = true;
}
if (Thread->OpDispatcher->FinishOp(DecodedInfo->PC + DecodedInfo->InstSize, i + 1 == InstsInBlock)) {
// If we had a dispatch error then leave early
if (HadDispatchError) {
if (TotalInstructions == 0) {
// Couldn't handle any instruction in op dispatcher
Thread->OpDispatcher->ResetWorkingList();
return { nullptr, nullptr, 0, 0 };
}
else {
uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
// We had some instructions. Early exit
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry + BlockInstructionsLength));
break;
}
}
}
Thread->OpDispatcher->Finalize();
auto IRDumper = [Thread, GuestRIP](IR::RegisterAllocationPass* RA) {
FILE* f = nullptr;
bool CloseAfter = false;
if (Thread->CTX->Config.DumpIR=="stderr") {
f = stderr;
}
else if (Thread->CTX->Config.DumpIR=="stdout") {
f = stdout;
}
else {
std::stringstream fileName;
fileName << Thread->CTX->Config.DumpIR << "/" << std::hex << GuestRIP << (RA ? "-post.ir" : "-pre.ir");
f = fopen(fileName.str().c_str(), "w");
CloseAfter = true;
}
if (f) {
std::stringstream out;
auto NewIR = Thread->OpDispatcher->ViewIR();
FEXCore::IR::Dump(&out, &NewIR, RA);
fprintf(f,"IR-%s 0x%lx:\n%s\n@@@@@\n", RA ? "post" : "pre", GuestRIP, out.str().c_str());
if (CloseAfter) {
fclose(f);
}
}
};
if (Thread->CTX->Config.DumpIR != "no") {
IRDumper(nullptr);
}
if (Thread->CTX->Config.ValidateIRarser) {
// Convert to text, Parse, Convert to text again and make sure the texts match
std::stringstream out;
static auto compaction = IR::CreateIRCompaction();
compaction->Run(Thread->OpDispatcher.get());
auto NewIR = Thread->OpDispatcher->ViewIR();
Dump(&out, &NewIR, nullptr);
out.seekg(0);
auto reparsed = IR::Parse(&out);
if (reparsed == nullptr) {
LogMan::Msg::A("Failed to parse ir\n");
} else {
std::stringstream out2;
auto NewIR2 = reparsed->ViewIR();
Dump(&out2, &NewIR2, nullptr);
if (out.str() != out2.str()) {
printf("one:\n %s\n", out.str().c_str());
printf("two:\n %s\n", out2.str().c_str());
LogMan::Msg::A("Parsed ir doesn't match\n");
}
delete reparsed;
if (Thread->OpDispatcher->FinishOp(DecodedInfo->PC + DecodedInfo->InstSize, i + 1 == InstsInBlock)) {
break;
}
}
// Run the passmanager over the IR from the dispatcher
Thread->PassManager->Run(Thread->OpDispatcher.get());
if (Thread->CTX->Config.DumpIR != "no") {
IRDumper(Thread->PassManager->GetRAPass());
}
if (Thread->OpDispatcher->ShouldDump) {
std::stringstream out;
auto NewIR = Thread->OpDispatcher->ViewIR();
FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass());
printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str());
}
// Create a copy of the IR and place it in this thread's IR cache
auto AddedIR = Thread->IRLists.try_emplace(GuestRIP, Thread->OpDispatcher->CreateIRCopy());
Thread->OpDispatcher->ResetWorkingList();
auto Debugit = &Thread->DebugData.try_emplace(GuestRIP).first->second;
Debugit->GuestCodeSize = TotalInstructionsLength;
Debugit->GuestInstructionCount = TotalInstructions;
IRList = AddedIR.first->second.get();
DebugData = Debugit;
Thread->Stats.BlocksCompiled.fetch_add(1);
}
else {
IRList = IR->second.get();
auto Debugit = Thread->DebugData.find(GuestRIP);
if (Debugit != Thread->DebugData.end()) {
DebugData = &Debugit->second;
Thread->OpDispatcher->Finalize();
auto IRDumper = [Thread, GuestRIP](IR::RegisterAllocationData* RA) {
FILE* f = nullptr;
bool CloseAfter = false;
if (Thread->CTX->Config.DumpIR=="stderr") {
f = stderr;
}
else if (Thread->CTX->Config.DumpIR=="stdout") {
f = stdout;
}
else {
std::stringstream fileName;
fileName << Thread->CTX->Config.DumpIR << "/" << std::hex << GuestRIP << (RA ? "-post.ir" : "-pre.ir");
f = fopen(fileName.str().c_str(), "w");
CloseAfter = true;
}
if (f) {
std::stringstream out;
auto NewIR = Thread->OpDispatcher->ViewIR();
FEXCore::IR::Dump(&out, &NewIR, RA);
fprintf(f,"IR-%s 0x%lx:\n%s\n@@@@@\n", RA ? "post" : "pre", GuestRIP, out.str().c_str());
if (CloseAfter) {
fclose(f);
}
}
};
if (Thread->CTX->Config.DumpIR != "no") {
IRDumper(nullptr);
}
if (Thread->CTX->Config.ValidateIRarser) {
// Convert to text, Parse, Convert to text again and make sure the texts match
std::stringstream out;
static auto compaction = IR::CreateIRCompaction();
compaction->Run(Thread->OpDispatcher.get());
auto NewIR = Thread->OpDispatcher->ViewIR();
Dump(&out, &NewIR, nullptr);
out.seekg(0);
auto reparsed = IR::Parse(&out);
if (reparsed == nullptr) {
LogMan::Msg::A("Failed to parse ir\n");
} else {
std::stringstream out2;
auto NewIR2 = reparsed->ViewIR();
Dump(&out2, &NewIR2, nullptr);
if (out.str() != out2.str()) {
printf("one:\n %s\n", out.str().c_str());
printf("two:\n %s\n", out2.str().c_str());
LogMan::Msg::A("Parsed ir doesn't match\n");
}
delete reparsed;
}
}
// Run the passmanager over the IR from the dispatcher
Thread->PassManager->Run(Thread->OpDispatcher.get());
if (Thread->CTX->Config.DumpIR != "no") {
IRDumper(Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr);
}
if (Thread->OpDispatcher->ShouldDump) {
std::stringstream out;
auto NewIR = Thread->OpDispatcher->ViewIR();
FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->GetAllocationData() : nullptr);
printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str());
}
auto RAData = Thread->PassManager->GetRAPass() ? Thread->PassManager->GetRAPass()->PullAllocationData() : nullptr;
auto IRList = Thread->OpDispatcher->CreateIRCopy();
Thread->OpDispatcher->ResetWorkingList();
return {IRList, RAData.release(), TotalInstructions, TotalInstructionsLength};
}
std::tuple<void *, FEXCore::IR::IRListView<true> *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool> Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
FEXCore::IR::IRListView<true> *IRList {};
FEXCore::Core::DebugData *DebugData {};
FEXCore::IR::RegisterAllocationData *RAData {};
bool GeneratedIR {};
// Do we already have this in the IR cache?
auto IR = Thread->IRLists.find(GuestRIP);
if (IR != Thread->IRLists.end()) {
// Entry already exists
// pull in the data
IRList = IR->second.get();
DebugData = Thread->DebugData.find(GuestRIP)->second.get();
RAData = Thread->RALists.find(GuestRIP)->second.get();
GeneratedIR = false;
} else {
// Generate IR + Meta Info
auto [IRCopy, RACopy, TotalInstructions, TotalInstructionsLength] = GenerateIR(Thread, GuestRIP);
// Setup pointers to internal structures
IRList = IRCopy;
RAData = RACopy;
DebugData = new FEXCore::Core::DebugData();
// Initialize metadata
DebugData->GuestCodeSize = TotalInstructionsLength;
DebugData->GuestInstructionCount = TotalInstructions;
// Increment stats
Thread->Stats.BlocksCompiled.fetch_add(1);
// These blocks aren't already in the cache
GeneratedIR = true;
}
// Attempt to get the CPU backend to compile this code
return { Thread->CPUBackend->CompileCode(IRList, DebugData), DebugData };
return { Thread->CPUBackend->CompileCode(IRList, DebugData, RAData), IRList, DebugData, RAData, GeneratedIR};
}
uintptr_t Context::CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
@@ -806,28 +819,42 @@ namespace FEXCore::Context {
return HostCode;
}
void *CodePtr;
FEXCore::Core::DebugData *DebugData;
void *CodePtr {};
FEXCore::IR::IRListView<true> *IRList {};
FEXCore::Core::DebugData *DebugData {};
FEXCore::IR::RegisterAllocationData *RAData {};
bool DecrementRefCount = false;
bool GeneratedIR {};
if (Thread->CompileBlockReentrantRefCount != 0) {
if (!Thread->CompileService) {
Thread->CompileService = std::make_shared<FEXCore::CompileService>(this, Thread);
Thread->CompileService->Initialize();
}
auto WorkItem = Thread->CompileService->CompileCode(GuestRIP);
WorkItem->ServiceWorkDone.Wait();
// Return here with the data in place
Thread->IRLists.try_emplace(GuestRIP, WorkItem->IRList->CreateCopy());
CodePtr = WorkItem->CodePtr;
IRList = WorkItem->IRList;
DebugData = WorkItem->DebugData;
RAData = WorkItem->RAData;
WorkItem->SafeToClear = true;
// The compile service will always generate IR + DebugData + RAData
// Remove the entries here to make sure we don't fail to insert later on
RemoveCodeEntry(Thread, GuestRIP);
GeneratedIR = true;
} else {
++Thread->CompileBlockReentrantRefCount;
DecrementRefCount = true;
auto [Code, Data] = CompileCode(Thread, GuestRIP);
auto [Code, IR, Data, RA, Generated] = CompileCode(Thread, GuestRIP);
CodePtr = Code;
IRList = IR;
DebugData = Data;
RAData = RA;
GeneratedIR = Generated;
}
if (CodePtr != nullptr) {
@@ -844,8 +871,17 @@ namespace FEXCore::Context {
}
#endif
// Insert to caches if we generated IR
if (GeneratedIR) {
Thread->IRLists.emplace(GuestRIP, IRList);
Thread->DebugData.emplace(GuestRIP, DebugData);
Thread->RALists.emplace(GuestRIP, RAData);
}
if (DecrementRefCount)
--Thread->CompileBlockReentrantRefCount;
// Insert to lookup cache
AddBlockMapping(Thread, GuestRIP, CodePtr);
return (uintptr_t)CodePtr;
@@ -861,7 +897,7 @@ namespace FEXCore::Context {
// We have ONE more chance to try and fallback to the fallback CPU backend
// This will most likely fail since regular code use won't be using a fallback core.
// It's mainly for testing new instruction encodings
void *CodePtr = Thread->FallbackBackend->CompileCode(nullptr, nullptr);
void *CodePtr = Thread->FallbackBackend->CompileCode(nullptr, nullptr, nullptr);
AddBlockMapping(Thread, GuestRIP, CodePtr);
return (uintptr_t)CodePtr;
}
@@ -918,12 +954,9 @@ namespace FEXCore::Context {
void Context::RemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
Thread->IRLists.erase(GuestRIP);
Thread->RALists.erase(GuestRIP);
Thread->DebugData.erase(GuestRIP);
Thread->LookupCache->Erase(GuestRIP);
if (Thread->CompileService) {
Thread->CompileService->RemoveCodeEntry(GuestRIP);
}
}
// Debug interface
External/FEXCore/Source/Interface/Core/Interpreter/InterpreterClass.h
Vendored
+1 -1
View File
@@ -22,7 +22,7 @@ public:
explicit InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
~InterpreterCore() override;
std::string GetName() override { return "Interpreter"; }
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) override;
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp
Vendored
+4 -2
View File
@@ -476,7 +476,7 @@ Res InterpreterCore::GetSrc(void* SSAData, IR::OrderedNodeWrapper Src) {
return reinterpret_cast<Res>(DstPtr);
}
void *InterpreterCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData) {
void *InterpreterCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
return reinterpret_cast<void*>(InterpreterExecution);
}
@@ -4719,10 +4719,12 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
}
}
#ifndef NDEBUG
auto DebugData = Thread->DebugData.find(Thread->State.State.rip);
if (DebugData != Thread->DebugData.end()) {
Thread->Stats.InstructionsExecuted.fetch_add(DebugData->second.GuestInstructionCount);
Thread->Stats.InstructionsExecuted.fetch_add(DebugData->second->GuestInstructionCount);
}
#endif
}
FEXCore::CPU::CPUBackend *CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
Vendored
+26 -31
View File
@@ -683,27 +683,21 @@ void JITCore::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant) {
}
}
struct PhysReg { uint32_t Class; uint32_t VId; };
static IR::PhysicalRegister GetPhys(IR::RegisterAllocationData *RAData, uint32_t Node) {
auto PhyReg = RAData->GetNodeRegister(Node);
static PhysReg GetPhys(IR::RegisterAllocationPass *RAPass, uint32_t Node) {
uint64_t Reg = RAPass->GetNodeRegister(Node);
auto rv = PhysReg {uint32_t(Reg>>32), (uint32_t)Reg};
LogMan::Throw::A(!PhyReg.IsInvalid(), "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
if (rv.VId != ~0U)
return rv;
else
LogMan::Msg::A("Couldn't Allocate register for node: ssa%d. Class: %d", Node, Reg >> 32);
return PhysReg { ~0U, ~0U};
return PhyReg;
}
template<>
aarch64::Register JITCore::GetReg<JITCore::RA_32>(uint32_t Node) {
auto Reg = GetPhys(RAPass, Node);
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::GPRFixedClass.Val) {
return SRA64[Reg.VId].W();
return SRA64[Reg.Reg].W();
} else if (Reg.Class == IR::GPRClass.Val) {
return RA64[Reg.VId].W();
return RA64[Reg.Reg].W();
} else {
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
@@ -711,11 +705,11 @@ auto Reg = GetPhys(RAPass, Node);
template<>
aarch64::Register JITCore::GetReg<JITCore::RA_64>(uint32_t Node) {
auto Reg = GetPhys(RAPass, Node);
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::GPRFixedClass.Val) {
return SRA64[Reg.VId];
return SRA64[Reg.Reg];
} else if (Reg.Class == IR::GPRClass.Val) {
return RA64[Reg.VId];
return RA64[Reg.Reg];
} else {
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
@@ -723,33 +717,33 @@ aarch64::Register JITCore::GetReg<JITCore::RA_64>(uint32_t Node) {
template<>
std::pair<aarch64::Register, aarch64::Register> JITCore::GetSrcPair<JITCore::RA_32>(uint32_t Node) {
uint32_t Reg = GetPhys(RAPass, Node).VId;
uint32_t Reg = GetPhys(RAData, Node).Reg;
return RA32Pair[Reg];
}
template<>
std::pair<aarch64::Register, aarch64::Register> JITCore::GetSrcPair<JITCore::RA_64>(uint32_t Node) {
uint32_t Reg = GetPhys(RAPass, Node).VId;
uint32_t Reg = GetPhys(RAData, Node).Reg;
return RA64Pair[Reg];
}
aarch64::VRegister JITCore::GetSrc(uint32_t Node) {
auto Reg = GetPhys(RAPass, Node);
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::FPRFixedClass.Val) {
return SRAFPR[Reg.VId];
return SRAFPR[Reg.Reg];
} else if (Reg.Class == IR::FPRClass.Val) {
return RAFPR[Reg.VId];
return RAFPR[Reg.Reg];
} else {
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
}
aarch64::VRegister JITCore::GetDst(uint32_t Node) {
auto Reg = GetPhys(RAPass, Node);
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::FPRFixedClass.Val) {
return SRAFPR[Reg.VId];
return SRAFPR[Reg.Reg];
} else if (Reg.Class == IR::FPRClass.Val) {
return RAFPR[Reg.VId];
return RAFPR[Reg.Reg];
} else {
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
@@ -770,8 +764,7 @@ bool JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Va
}
FEXCore::IR::RegisterClassType JITCore::GetRegClass(uint32_t Node) {
auto Class = static_cast<uint32_t>(RAPass->GetNodeRegister(Node) >> 32);
return FEXCore::IR::RegisterClassType {Class};
return FEXCore::IR::RegisterClassType {GetPhys(RAData, Node).Class};
}
@@ -787,11 +780,13 @@ bool JITCore::IsGPR(uint32_t Node) {
return Class == IR::GPRClass || Class == IR::GPRFixedClass;
}
void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData) {
void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
using namespace aarch64;
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
this->RAData = RAData;
auto HeaderOp = IR->GetHeader();
#ifndef NDEBUG
@@ -803,7 +798,7 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
// Fairly excessive buffer range to make sure we don't overflow
uint32_t BufferRange = SSACount * 16;
if ((GetCursorOffset() + BufferRange) > CurrentCodeBuffer->Size) {
State->CTX->ClearCodeCache(State, HeaderOp->Entry);
State->CTX->ClearCodeCache(State, false);
}
// AAPCS64
@@ -861,9 +856,9 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
LoadConstant(x0, ThreadSharedData.InterpreterFallbackHelperAddress);
br(x0);
} else {
LogMan::Throw::A(RAPass->HasFullRA(), "Arm64 JIT only works with RA");
//LogMan::Throw::A(RAData->HasFullRA(), "Arm64 JIT only works with RA");
SpillSlots = RAPass->SpillSlots();
SpillSlots = RAData->SpillSlots();
if (SpillSlots) {
sub(sp, sp, SpillSlots * 16);
@@ -1097,7 +1092,7 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) {
Literal l_VirtualMemory {VirtualMemorySize};
Literal l_PagePtr {Thread->LookupCache->GetPagePointer()};
Literal l_CTX {reinterpret_cast<uintptr_t>(CTX)};
Literal l_Interpreter {reinterpret_cast<uint64_t>(State->IntBackend->CompileCode(nullptr, nullptr))};
Literal l_Interpreter {reinterpret_cast<uint64_t>(State->IntBackend->CompileCode(nullptr, nullptr, nullptr))};
Literal l_Sleep {reinterpret_cast<uint64_t>(SleepThread)};
uintptr_t CompileBlockPtr{};
External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
Vendored
+2 -1
View File
@@ -78,7 +78,7 @@ public:
~JITCore() override;
std::string GetName() override { return "JIT"; }
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) override;
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -232,6 +232,7 @@ private:
CompilerSharedData ThreadSharedData;
IR::RegisterAllocationPass *RAPass;
IR::RegisterAllocationData *RAData;
uint32_t SpillSlots{};
External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp
Vendored
+31 -37
View File
@@ -460,27 +460,20 @@ void JITCore::ClearCache() {
}
}
uint32_t JITCore::GetPhys(uint32_t Node) {
uint64_t Reg = RAPass->GetNodeRegister(Node);
IR::PhysicalRegister JITCore::GetPhys(uint32_t Node) {
auto PhyReg = RAData->GetNodeRegister(Node);
if ((uint32_t)Reg != ~0U)
return Reg;
else
LogMan::Msg::A("Couldn't Allocate register for node: ssa%d. Class: %d", Node, Reg >> 32);
LogMan::Throw::A(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
return ~0U;
return PhyReg;
}
bool JITCore::IsFPR(uint32_t Node) {
auto Class = RAPass->GetNodeRegister(Node) >> 32;
return Class == IR::FPRClass.Val;
return RAData->GetNodeRegister(Node).Class == IR::FPRClass.Val;
}
bool JITCore::IsGPR(uint32_t Node) {
auto Class = RAPass->GetNodeRegister(Node) >> 32;
return Class == IR::GPRClass.Val;
return RAData->GetNodeRegister(Node).Class == IR::GPRClass.Val;
}
template<uint8_t RAType>
@@ -489,17 +482,17 @@ Xbyak::Reg JITCore::GetSrc(uint32_t Node) {
// r10
// Callee Saved
// rbx, rbp, r12, r13, r14, r15
uint32_t Reg = GetPhys(Node);
auto PhyReg = GetPhys(Node);
if (RAType == RA_64)
return RA64[Reg].cvt64();
return RA64[PhyReg.Reg].cvt64();
else if (RAType == RA_XMM)
return RAXMM[Reg];
return RAXMM[PhyReg.Reg];
else if (RAType == RA_32)
return RA64[Reg].cvt32();
return RA64[PhyReg.Reg].cvt32();
else if (RAType == RA_16)
return RA64[Reg].cvt16();
return RA64[PhyReg.Reg].cvt16();
else if (RAType == RA_8)
return RA64[Reg].cvt8();
return RA64[PhyReg.Reg].cvt8();
}
template
@@ -515,23 +508,23 @@ template
Xbyak::Reg JITCore::GetSrc<JITCore::RA_8>(uint32_t Node);
Xbyak::Xmm JITCore::GetSrc(uint32_t Node) {
uint32_t Reg = GetPhys(Node);
return RAXMM_x[Reg];
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
template<uint8_t RAType>
Xbyak::Reg JITCore::GetDst(uint32_t Node) {
uint32_t Reg = GetPhys(Node);
auto PhyReg = GetPhys(Node);
if (RAType == RA_64)
return RA64[Reg].cvt64();
return RA64[PhyReg.Reg].cvt64();
else if (RAType == RA_XMM)
return RAXMM[Reg];
return RAXMM[PhyReg.Reg];
else if (RAType == RA_32)
return RA64[Reg].cvt32();
return RA64[PhyReg.Reg].cvt32();
else if (RAType == RA_16)
return RA64[Reg].cvt16();
return RA64[PhyReg.Reg].cvt16();
else if (RAType == RA_8)
return RA64[Reg].cvt8();
return RA64[PhyReg.Reg].cvt8();
}
template
@@ -548,11 +541,11 @@ Xbyak::Reg JITCore::GetDst<JITCore::RA_8>(uint32_t Node);
template<uint8_t RAType>
std::pair<Xbyak::Reg, Xbyak::Reg> JITCore::GetSrcPair(uint32_t Node) {
uint32_t Reg = GetPhys(Node);
auto PhyReg = GetPhys(Node);
if (RAType == RA_64)
return RA64Pair[Reg];
return RA64Pair[PhyReg.Reg];
else if (RAType == RA_32)
return {RA64Pair[Reg].first.cvt32(), RA64Pair[Reg].second.cvt32()};
return {RA64Pair[PhyReg.Reg].first.cvt32(), RA64Pair[PhyReg.Reg].second.cvt32()};
}
template
@@ -562,8 +555,8 @@ template
std::pair<Xbyak::Reg, Xbyak::Reg> JITCore::GetSrcPair<JITCore::RA_32>(uint32_t Node);
Xbyak::Xmm JITCore::GetDst(uint32_t Node) {
uint32_t Reg = GetPhys(Node);
return RAXMM_x[Reg];
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
bool JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
@@ -610,16 +603,17 @@ std::tuple<JITCore::SetCC, JITCore::CMovCC, JITCore::JCC> JITCore::GetCC(IR::Con
}
}
void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData) {
void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
this->RAData = RAData;
auto HeaderOp = IR->GetHeader();
// Fairly excessive buffer range to make sure we don't overflow
uint32_t BufferRange = SSACount * 16;
if ((getSize() + BufferRange) > CurrentCodeBuffer->Size) {
ThreadState->CTX->ClearCodeCache(ThreadState, HeaderOp->Entry);
ThreadState->CTX->ClearCodeCache(ThreadState, false);
}
void *Entry = getCurr<void*>();
@@ -650,9 +644,9 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
mov(rax, (uintptr_t)ThreadSharedData.InterpreterFallbackHelperAddress);
jmp(rax);
} else {
LogMan::Throw::A(RAPass->HasFullRA(), "Needs RA");
LogMan::Throw::A(RAData != nullptr, "Needs RA");
SpillSlots = RAPass->SpillSlots();
SpillSlots = RAData->SpillSlots();
if (SpillSlots) {
sub(rsp, SpillSlots * 16);
@@ -1001,7 +995,7 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) {
ThreadSharedData.InterpreterFallbackHelperAddress = getCurr<void*>();
// This will get called so our stack is now misaligned
mov(rdi, STATE);
mov(rax, reinterpret_cast<uint64_t>(ThreadState->IntBackend->CompileCode(nullptr, nullptr)));
mov(rax, reinterpret_cast<uint64_t>(ThreadState->IntBackend->CompileCode(nullptr, nullptr, nullptr)));
call(rax);
External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h
Vendored
+4 -2
View File
@@ -14,6 +14,7 @@ using namespace Xbyak;
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/IR/IR.h>
#include <FEXCore/IR/IntrusiveIRList.h>
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include <tuple>
@@ -58,7 +59,7 @@ public:
explicit JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, CodeBuffer Buffer, bool CompileThread);
~JITCore() override;
std::string GetName() override { return "JIT"; }
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) override;
void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -106,7 +107,7 @@ private:
constexpr static uint8_t RA_64 = 3;
constexpr static uint8_t RA_XMM = 4;
uint32_t GetPhys(uint32_t Node);
IR::PhysicalRegister GetPhys(uint32_t Node);
bool IsFPR(uint32_t Node);
bool IsGPR(uint32_t Node);
@@ -128,6 +129,7 @@ private:
void CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread);
IR::RegisterAllocationPass *RAPass;
FEXCore::IR::RegisterAllocationData *RAData;
#ifdef BLOCKSTATS
bool GetSamplingData {true};
External/FEXCore/Source/Interface/IR/IR.json
Vendored
+2
View File
@@ -30,6 +30,8 @@
"static constexpr FEXCore::IR::RegisterClassType ComplexClass {5}",
"static constexpr FEXCore::IR::RegisterClassType InvalidClass {7}",
"",
"static constexpr uint8_t InvalidReg {31}",
"",
"static const FEXCore::IR::TypeDefinition i8 {TypeDefinition::Create(1, 0)}",
"static const FEXCore::IR::TypeDefinition i16 {TypeDefinition::Create(2, 0)}",
"static const FEXCore::IR::TypeDefinition i32 {TypeDefinition::Create(4, 0)}",
External/FEXCore/Source/Interface/IR/IRDumper.cpp
Vendored
+15 -18
View File
@@ -74,18 +74,17 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView<false>
*out << "Unknown Registerclass " << Arg;
}
static void PrintArg(std::stringstream *out, IRListView<false> const* IR, OrderedNodeWrapper Arg, IR::RegisterAllocationPass *RAPass) {
static void PrintArg(std::stringstream *out, IRListView<false> const* IR, OrderedNodeWrapper Arg, IR::RegisterAllocationData *RAData) {
auto [CodeNode, IROp] = IR->at(Arg)();
if (Arg.ID() == 0) {
*out << "%Invalid";
} else {
*out << "%ssa" << std::to_string(Arg.ID());
if (RAPass) {
uint64_t RegClass = RAPass->GetNodeRegister(Arg.ID());
FEXCore::IR::RegisterClassType Class {uint32_t(RegClass >> 32)};
uint32_t Reg = RegClass;
switch (Class) {
if (RAData) {
auto PhyReg = RAData->GetNodeRegister(Arg.ID());
switch (PhyReg.Class) {
case FEXCore::IR::GPRClass.Val: *out << "(GPR"; break;
case FEXCore::IR::GPRFixedClass.Val: *out << "(GPRFixed"; break;
case FEXCore::IR::FPRClass.Val: *out << "(FPR"; break;
@@ -96,8 +95,8 @@ static void PrintArg(std::stringstream *out, IRListView<false> const* IR, Ordere
default: *out << "(Unknown"; break;
}
if (Class != FEXCore::IR::InvalidClass.Val) {
*out << std::dec << Reg << ")";
if (PhyReg.Class != FEXCore::IR::InvalidClass.Val) {
*out << std::dec << (uint32_t)PhyReg.Reg << ")";
} else {
*out << ")";
}
@@ -147,7 +146,7 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView<false>
}
}
void Dump(std::stringstream *out, IRListView<false> const* IR, IR::RegisterAllocationPass *RAPass) {
void Dump(std::stringstream *out, IRListView<false> const* IR, IR::RegisterAllocationData *RAData) {
auto HeaderOp = IR->GetHeader();
int8_t CurrentIndent = 0;
@@ -204,11 +203,9 @@ void Dump(std::stringstream *out, IRListView<false> const* IR, IR::RegisterAlloc
*out << "%ssa" << std::to_string(ID);
if (RAPass) {
uint64_t RegClass = RAPass->GetNodeRegister(ID);
FEXCore::IR::RegisterClassType Class {uint32_t(RegClass >> 32)};
uint32_t Reg = RegClass;
switch (Class) {
if (RAData) {
auto PhyReg = RAData->GetNodeRegister(ID);
switch (PhyReg.Class) {
case FEXCore::IR::GPRClass.Val: *out << "(GPR"; break;
case FEXCore::IR::GPRFixedClass.Val: *out << "(GPRFixed"; break;
case FEXCore::IR::FPRClass.Val: *out << "(FPR"; break;
@@ -218,8 +215,8 @@ void Dump(std::stringstream *out, IRListView<false> const* IR, IR::RegisterAlloc
case FEXCore::IR::InvalidClass.Val: *out << "(Invalid"; break;
default: *out << "(Unknown"; break;
}
if (Class != FEXCore::IR::InvalidClass.Val) {
*out << std::dec << Reg << ")";
if (PhyReg.Class != FEXCore::IR::InvalidClass.Val) {
*out << std::dec << (uint32_t)PhyReg.Reg << ")";
} else {
*out << ")";
}
@@ -264,9 +261,9 @@ void Dump(std::stringstream *out, IRListView<false> const* IR, IR::RegisterAlloc
auto [NodeNode, IROp] = NodeBegin();
auto PhiOp = IROp->C<IR::IROp_PhiValue>();
*out << "[ ";
PrintArg(out, IR, PhiOp->Value, RAPass);
PrintArg(out, IR, PhiOp->Value, RAData);
*out << ", ";
PrintArg(out, IR, PhiOp->Block, RAPass);
PrintArg(out, IR, PhiOp->Block, RAData);
*out << " ]";
if (PhiOp->Next.ID())
External/FEXCore/Source/Interface/IR/PassManager.cpp
Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
#include "Interface/IR/PassManager.h"
#include "Interface/IR/Passes.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include "Interface/IR/PassManager.h"
#include <FEXCore/Config/Config.h>
External/FEXCore/Source/Interface/IR/Passes.h
Vendored
+1
View File
@@ -3,6 +3,7 @@
namespace FEXCore::IR {
class Pass;
class RegisterAllocationPass;
class RegisterAllocationData;
FEXCore::IR::Pass* CreateConstProp(bool InlineConstants);
FEXCore::IR::Pass* CreateContextLoadStoreElimination();
External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp
Vendored
+7 -7
View File
@@ -50,9 +50,9 @@ bool IRValidation::Run(IREmitter *IREmit) {
auto HeaderOp = CurrentIR.GetHeader();
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
IR::RegisterAllocationPass * RAPass{};
IR::RegisterAllocationData * RAData{};
if (Manager->HasRAPass() && !HeaderOp->ShouldInterpret) {
RAPass = Manager->GetRAPass();
RAData = Manager->GetRAPass() ? Manager->GetRAPass()->GetAllocationData() : nullptr;
}
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
@@ -81,12 +81,12 @@ bool IRValidation::Run(IREmitter *IREmit) {
Warnings << "%ssa" << ID << ": Destination created but had no uses" << std::endl;
}
if (RAPass) {
if (RAData) {
// If we have a register allocator then the destination needs to be assigned a register and class
uint64_t Reg = RAPass->GetNodeRegister(ID);
auto PhyReg = RAData->GetNodeRegister(ID);
FEXCore::IR::RegisterClassType ExpectedClass = IR::GetRegClass(IROp->Op);
FEXCore::IR::RegisterClassType AssignedClass = FEXCore::IR::RegisterClassType{uint32_t(Reg >> 32)};
FEXCore::IR::RegisterClassType AssignedClass = FEXCore::IR::RegisterClassType{PhyReg.Class};
// If no register class was assigned
if (AssignedClass == IR::InvalidClass) {
@@ -95,7 +95,7 @@ bool IRValidation::Run(IREmitter *IREmit) {
}
// If no physical register was assigned
if ((uint32_t)Reg == ~0U) {
if (PhyReg.Reg == IR::InvalidReg) {
HadError |= true;
Errors << "%ssa" << ID << ": Had destination but with no register assigned" << std::endl;
}
@@ -260,7 +260,7 @@ bool IRValidation::Run(IREmitter *IREmit) {
HadWarning = false;
if (HadError || HadWarning) {
FEXCore::IR::Dump(&Out, &CurrentIR, RAPass);
FEXCore::IR::Dump(&Out, &CurrentIR, RAData);
if (HadError) {
Out << "Errors:" << std::endl << Errors.str() << std::endl;
External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp
Vendored
+78 -85
View File
@@ -9,13 +9,16 @@
#define SRA_DEBUG(...) // printf(__VA_ARGS__)
namespace {
constexpr uint32_t INVALID_REG = 31;
constexpr uint32_t INVALID_CLASS = 7;
constexpr uint32_t DEFAULT_INTERFERENCE_LIST_COUNT = 120;
using FEXCore::IR::PhysicalRegister;
constexpr uint32_t INVALID_REG = FEXCore::IR::InvalidReg;
constexpr uint32_t INVALID_CLASS = FEXCore::IR::InvalidClass.Val;
constexpr uint32_t DEFAULT_INTERFERENCE_LIST_COUNT = 122;
constexpr uint32_t DEFAULT_INTERFERENCE_SPAN_COUNT = 30;
constexpr uint32_t DEFAULT_NODE_COUNT = 8192;
constexpr uint64_t INVALID_REGCLASS = (((uint64_t)INVALID_CLASS) << 32) | (INVALID_REG);
const PhysicalRegister INVALID_REGCLASS = PhysicalRegister::Invalid();
// BucketList is an optimized container, it includes an inline array of Size
// and can overflow to a linked list of further buckets
@@ -158,11 +161,10 @@ namespace {
struct RegisterNode {
struct VolatileHeader {
uint64_t RegAndClass;
uint32_t BlockID;
uint32_t SpillSlot;
RegisterNode *PhiPartner;
} Head { INVALID_REGCLASS, ~0U, ~0U, nullptr };
} Head { ~0U, ~0U, nullptr };
BucketList<DEFAULT_INTERFERENCE_LIST_COUNT, uint32_t> Interferences;
};
@@ -179,8 +181,8 @@ namespace {
uint32_t Begin{~0U};
uint32_t End{~0U};
uint32_t RematCost{0};
int64_t PrefferedRegister{-1};
uint32_t PreWritten{0};
PhysicalRegister PrefferedRegister{INVALID_REGCLASS};
bool Written{false};
bool Global{false};
};
@@ -193,6 +195,7 @@ namespace {
};
struct RegisterGraph {
std::unique_ptr<FEXCore::IR::RegisterAllocationData, FEXCore::IR::RegisterAllocationDataDeleter> AllocData;
RegisterSet Set;
std::vector<RegisterNode> Nodes;
uint32_t NodeCount;
@@ -221,41 +224,27 @@ namespace {
Graph->Set.Classes[Class].PhysicalCount = Count;
}
void SetConflict(RegisterGraph *Graph, uint64_t RegAndClass, uint64_t RegAndClassConflict) {
uint32_t Reg = (uint32_t)RegAndClass;
uint32_t Class = RegAndClass >> 32;
uint32_t RegConflict = (uint32_t)RegAndClassConflict;
uint32_t ClassConflict = RegAndClassConflict >> 32;
uint32_t Index = ((Class << 3) | ClassConflict << 5) | Reg;
void SetConflict(RegisterGraph *Graph, PhysicalRegister RegAndClass, PhysicalRegister ConflictRegAndClass) {
uint32_t Index = (ConflictRegAndClass.Class << 8) | RegAndClass.Raw;
Graph->Set.Conflicts[Index] |= 1 << RegConflict;
Graph->Set.Conflicts[Index] |= 1 << ConflictRegAndClass.Reg;
}
bool IsConflict(RegisterGraph *Graph, uint64_t RegAndClass, uint64_t ConflictRegAndClass) {
uint32_t Reg = (uint32_t)RegAndClass;
uint32_t Class = RegAndClass >> 32;
uint32_t RegConflict = (uint32_t)ConflictRegAndClass;
uint32_t ClassConflict = ConflictRegAndClass >> 32;
uint32_t Index = ((Class << 3) | ClassConflict << 5) | Reg;
return (Graph->Set.Conflicts[Index] >> RegConflict) & 1;
bool IsConflict(RegisterGraph *Graph, PhysicalRegister RegAndClass, PhysicalRegister ConflictRegAndClass) {
uint32_t Index = (ConflictRegAndClass.Class << 8) | RegAndClass.Raw;
return (Graph->Set.Conflicts[Index] >> ConflictRegAndClass.Reg) & 1;
}
uint32_t GetConflicts(RegisterGraph *Graph, uint64_t RegAndClass, uint32_t ConflictClass) {
uint32_t Reg = (uint32_t)RegAndClass;
uint32_t Class = RegAndClass >> 32;
uint32_t Index = ((Class << 3) | ConflictClass << 5) | Reg;
uint32_t GetConflicts(RegisterGraph *Graph, PhysicalRegister RegAndClass, FEXCore::IR::RegisterClassType ConflictClass) {
uint32_t Index = (ConflictClass.Val << 8) | RegAndClass.Raw;
return Graph->Set.Conflicts[Index];
}
void VirtualAddRegisterConflict(RegisterGraph *Graph, FEXCore::IR::RegisterClassType ClassConflict, uint32_t RegConflict, FEXCore::IR::RegisterClassType Class, uint32_t Reg) {
auto RegAndClass = ((uint64_t)Class << 32) | Reg;
auto RegAndClassConflict = ((uint64_t)ClassConflict << 32) | RegConflict;
auto RegAndClass = PhysicalRegister(Class, Reg);
auto RegAndClassConflict = PhysicalRegister(ClassConflict, RegConflict);
// Conflict must go both ways
SetConflict(Graph, RegAndClass, RegAndClassConflict);
@@ -270,12 +259,15 @@ namespace {
NodeCount = AlignUp(NodeCount, sizeof(uint64_t));
Graph->Nodes.clear();
Graph->Nodes.resize(NodeCount);
Graph->NodeCount = NodeCount;
Graph->VisitedNodePredecessors.clear();
Graph->AllocData.reset();
Graph->AllocData.reset((FEXCore::IR::RegisterAllocationData*)malloc(FEXCore::IR::RegisterAllocationData::Size(NodeCount)));
memset(&Graph->AllocData->Map[0], INVALID_REGCLASS.Raw, NodeCount);
Graph->NodeCount = NodeCount;
}
void SetNodeClass(RegisterGraph *Graph, uint32_t Node, FEXCore::IR::RegisterClassType Class) {
Graph->Nodes[Node].Head.RegAndClass = ((uint64_t)Class << 32) | (Graph->Nodes[Node].Head.RegAndClass & ~0U);
Graph->AllocData->Map[Node].Class = Class.Val;
}
void SetNodePartner(RegisterGraph *Graph, uint32_t Node, uint32_t Partner) {
@@ -283,25 +275,21 @@ namespace {
}
bool DoesNodeConflictWithRegAndClass(RegisterGraph *Graph, RegisterNode const *InterferenceNode, uint64_t RegAndClass) {
return IsConflict(Graph, InterferenceNode->Head.RegAndClass, RegAndClass);
}
/**
* @brief Individual node interference check
*/
bool DoesNodeInterfereWithRegister(RegisterGraph *Graph, RegisterNode const *Node, uint64_t RegAndClass) {
bool DoesNodeInterfereWithRegister(RegisterGraph *Graph, RegisterNode const *Node, PhysicalRegister RegAndClass) {
// Walk the node's interference list and see if it interferes with this register
return Node->Interferences.Find([Graph, RegAndClass](uint32_t InterferenceNodeId) {
RegisterNode *InterferenceNode = &Graph->Nodes[InterferenceNodeId];
return DoesNodeConflictWithRegAndClass(Graph, InterferenceNode, RegAndClass);
auto InterferenceRegAndClass = Graph->AllocData->Map[InterferenceNodeId];
return IsConflict(Graph, InterferenceRegAndClass, RegAndClass);
});
}
/**
* @brief Node set walking for PHI node interference checking
*/
bool DoesNodeSetInterfereWithRegister(RegisterGraph *Graph, std::vector<RegisterNode*> const &Nodes, uint64_t RegAndClass) {
bool DoesNodeSetInterfereWithRegister(RegisterGraph *Graph, std::vector<RegisterNode*> const &Nodes, PhysicalRegister RegAndClass) {
for (auto it : Nodes) {
if (DoesNodeInterfereWithRegister(Graph, it, RegAndClass)) {
return true;
@@ -369,7 +357,9 @@ namespace {
for (auto [CodeNode, IROp] : IR->GetAllCode()) {
// If the destination hasn't yet been set then set it now
if (IROp->HasDest) {
SetNodeClass(Graph, IR->GetID(CodeNode), GetRegClassFromNode(IR, IROp));
Graph->AllocData->Map[IR->GetID(CodeNode)] = PhysicalRegister(GetRegClassFromNode(IR, IROp), INVALID_REG);
} else {
//Graph->AllocData->Map[IR->GetID(CodeNode)] = PhysicalRegister::Invalid();
}
}
}
@@ -390,7 +380,8 @@ namespace FEXCore::IR {
* @brief Returns the register and class encoded together
* Top 32bits is the class, lower 32bits is the register
*/
uint64_t GetNodeRegister(uint32_t Node) override;
RegisterAllocationData* GetAllocationData() override;
std::unique_ptr<RegisterAllocationData, RegisterAllocationDataDeleter> PullAllocationData() override;
private:
bool OptimizeSRA;
uint32_t SpillPointId;
@@ -466,8 +457,12 @@ namespace FEXCore::IR {
VirtualAddRegisterConflict(Graph, ClassConflict, RegConflict, Class, Reg);
}
uint64_t ConstrainedRAPass::GetNodeRegister(uint32_t Node) {
return Graph->Nodes[Node].Head.RegAndClass;
RegisterAllocationData* ConstrainedRAPass::GetAllocationData() {
return Graph->AllocData.get();
}
std::unique_ptr<RegisterAllocationData, RegisterAllocationDataDeleter> ConstrainedRAPass::PullAllocationData() {
return std::move(Graph->AllocData);
}
void ConstrainedRAPass::RecursiveLiveRangeExpansion(FEXCore::IR::IRListView<false> *IR, uint32_t Node, uint32_t DefiningBlockID, LiveRange *LiveRange, const std::unordered_set<uint32_t> &Predecessors, std::unordered_set<uint32_t> &VisitedPredecessors) {
@@ -617,13 +612,13 @@ namespace FEXCore::IR {
if (Offset >= beginGpr && Offset < endGpr) {
auto reg = (Offset - beginGpr) / 8;
return (uint64_t(GPRFixedClass.Val)<<32) | reg;
return PhysicalRegister(GPRFixedClass, reg);
} else if (Offset >= beginFpr && Offset < endFpr) {
auto reg = (Offset - beginFpr) / 16;
return (uint64_t(FPRFixedClass.Val)<<32) | reg;
return PhysicalRegister(FPRFixedClass, reg);
} else {
LogMan::Throw::A(false, "Unexpected Offset %d", Offset);
return ~0UL;
return INVALID_REGCLASS;
}
};
@@ -652,16 +647,13 @@ namespace FEXCore::IR {
};
// Get a StaticMap entry from reg and class
auto GetStaticMapFromReg = [&](int64_t RegAndClass) {
uint32_t Class = RegAndClass >> 32;
uint32_t Reg = RegAndClass;
if (Class == GPRFixedClass.Val) {
return &StaticMaps[Reg];
} else if (Class == FPRFixedClass.Val) {
return &StaticMaps[GprSize + Reg];
auto GetStaticMapFromReg = [&](IR::PhysicalRegister PhyReg) {
if (PhyReg.Class == GPRFixedClass.Val) {
return &StaticMaps[PhyReg.Reg];
} else if (PhyReg.Class == FPRFixedClass.Val) {
return &StaticMaps[GprSize + PhyReg.Reg];
} else {
LogMan::Throw::A(false, "Unexpected Class %d", Class);
LogMan::Throw::A(false, "Unexpected Class %d", PhyReg.Class);
return (LiveRange**)nullptr;
}
};
@@ -675,7 +667,7 @@ namespace FEXCore::IR {
//int -1 /*vreg*/ = (int)(Op->Offset / 8) - 1;
if (IsPreWritable(IROp->Size, Op->StaticClass)
&& LiveRanges[Op->Value.ID()].PrefferedRegister == -1
&& LiveRanges[Op->Value.ID()].PrefferedRegister.IsInvalid()
&& !LiveRanges[Op->Value.ID()].Global) {
//pre-write and sra-allocate in the defining node - this might be undone if a read before the actual store happens
@@ -707,7 +699,7 @@ namespace FEXCore::IR {
// ACCESSED after write, let's not SRA this one
if (LiveRanges[ArgNode].Written) {
SRA_DEBUG("Demoting ssa%d because accessed after write in ssa%d\n", ArgNode, Node);
LiveRanges[ArgNode].PrefferedRegister = -1;
LiveRanges[ArgNode].PrefferedRegister = INVALID_REGCLASS;
auto ArgNodeNode = IR->GetNode(IROp->Args[i]);
SetNodeClass(Graph, ArgNode, GetRegClassFromNode(IR, ArgNodeNode->Op(IR->GetData())));
}
@@ -717,7 +709,7 @@ namespace FEXCore::IR {
if (IROp->HasDest) {
// If this is a pre-write, update the StaticMap so we track writes
if (LiveRanges[Node].PrefferedRegister != -1) {
if (!LiveRanges[Node].PrefferedRegister.IsInvalid()) {
SRA_DEBUG("ssa%d is a pre-write\n", Node);
auto StaticMap = GetStaticMapFromReg(LiveRanges[Node].PrefferedRegister);
if ((*StaticMap)) {
@@ -741,13 +733,13 @@ namespace FEXCore::IR {
uint32_t ID = (*StaticMap) - &LiveRanges[0];
SRA_DEBUG("ssa%d cannot be a pre-write because ssa%d reads from sra%d before storereg", ID, Node, -1 /*vreg*/);
(*StaticMap)->PrefferedRegister = -1;
(*StaticMap)->PrefferedRegister = INVALID_REGCLASS;
(*StaticMap)->PreWritten = 0;
SetNodeClass(Graph, ID, Op->Class);
}
// if not sra-allocated and full size, sra-allocate
if (!LiveRanges[Node].Global && LiveRanges[Node].PrefferedRegister == -1) {
if (!LiveRanges[Node].Global && LiveRanges[Node].PrefferedRegister.IsInvalid()) {
// only full size reads can be aliased
if (IsAliasable(IROp->Size, Op->StaticClass, Op->Offset)) {
@@ -890,13 +882,11 @@ namespace FEXCore::IR {
// Now that we have all the live ranges calculated we need to add them to our interference graph
auto GetClass = [](uint64_t RegAndClass) {
uint32_t Class = RegAndClass >> 32;
if (Class == IR::GPRPairClass.Val)
auto GetClass = [](PhysicalRegister PhyReg) {
if (PhyReg.Class == IR::GPRPairClass.Val)
return IR::GPRClass.Val;
else
return Class;
return (uint32_t)PhyReg.Class;
};
SpanStart.resize(NodeCount);
@@ -905,7 +895,7 @@ namespace FEXCore::IR {
if (LiveRanges[i].Begin != ~0U) {
LogMan::Throw::A(LiveRanges[i].Begin < LiveRanges[i].End , "Span must Begin before Ending");
auto Class = GetClass(Graph->Nodes[i].Head.RegAndClass);
auto Class = GetClass(Graph->AllocData->Map[i]);
SpanStart[LiveRanges[i].Begin].Append(INFO_MAKE(i, Class));
SpanEnd[LiveRanges[i].End] .Append(INFO_MAKE(i, Class));
}
@@ -941,13 +931,14 @@ namespace FEXCore::IR {
void ConstrainedRAPass::AllocateVirtualRegisters() {
for (uint32_t i = 0; i < Graph->NodeCount; ++i) {
RegisterNode *CurrentNode = &Graph->Nodes[i];
if (CurrentNode->Head.RegAndClass == INVALID_REGCLASS)
auto &CurrentRegAndClass = Graph->AllocData->Map[i];
if (CurrentRegAndClass == INVALID_REGCLASS)
continue;
auto LiveRange = &LiveRanges[i];
FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentNode->Head.RegAndClass >> 32)};
uint64_t RegAndClass = INVALID_REGCLASS;
FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{CurrentRegAndClass.Class};
auto RegAndClass = INVALID_REGCLASS;
RegisterClass *RAClass = &Graph->Set.Classes[RegClass];
if (CurrentNode->Head.PhiPartner) {
@@ -986,35 +977,34 @@ namespace FEXCore::IR {
}
else {
if (LiveRange->PrefferedRegister != -1) {
if (!LiveRange->PrefferedRegister.IsInvalid()) {
RegAndClass = LiveRange->PrefferedRegister;
} else {
uint32_t RegisterConflicts = 0;
CurrentNode->Interferences.Iterate([&](const uint32_t InterferenceNode) {
RegisterConflicts |= GetConflicts(Graph, Graph->Nodes[InterferenceNode].Head.RegAndClass, RegClass);
RegisterConflicts |= GetConflicts(Graph, Graph->AllocData->Map[InterferenceNode], {RegClass});
});
RegisterConflicts = (~RegisterConflicts) & RAClass->CountMask;
int Reg = ffs(RegisterConflicts);
if (Reg != 0) {
RegAndClass = (static_cast<uint64_t>(RegClass) << 32) + Reg -1;
RegAndClass = PhysicalRegister({RegClass}, Reg-1);
}
}
// If we failed to find a virtual register then use INVALID_REG and mark allocation as failed
if (RegAndClass == INVALID_REGCLASS) {
RegAndClass = (static_cast<uint64_t>(RegClass.Val) << 32);
RegAndClass |= INVALID_REG;
if (RegAndClass.IsInvalid()) {
RegAndClass = IR::PhysicalRegister(RegClass, INVALID_REG);
HadFullRA = false;
SpillPointId = i;
CurrentNode->Head.RegAndClass = RegAndClass;
CurrentRegAndClass = RegAndClass;
// Must spill and restart
return;
}
CurrentNode->Head.RegAndClass = RegAndClass;
CurrentRegAndClass = RegAndClass;
}
}
}
@@ -1308,7 +1298,7 @@ namespace FEXCore::IR {
if (InterferenceIdToSpill == 0) {
int j = 0;
LogMan::Msg::D("node %%ssa%d, was dumped in to virtual reg %d. Live Range[%d, %d)",
CurrentLocation, RegisterNode->Head.RegAndClass,
CurrentLocation, -1,
OpLiveRange->Begin, OpLiveRange->End);
RegisterNode->Interferences.Iterate([&](uint32_t InterferenceNode) {
@@ -1356,12 +1346,13 @@ namespace FEXCore::IR {
uint32_t Node = IR.GetID(CodeNode);
RegisterNode *CurrentNode = &Graph->Nodes[Node];
auto &CurrentRegAndClass = Graph->AllocData->Map[Node];
LiveRange *OpLiveRange = &LiveRanges[Node];
// If this node is allocated above the number of physical registers we have then we need to search the interference list and spill the one
// that is cheapest
FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{uint32_t(CurrentNode->Head.RegAndClass >> 32)};
bool NeedsToSpill = (uint32_t)CurrentNode->Head.RegAndClass == INVALID_REG;
FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{CurrentRegAndClass.Class};
bool NeedsToSpill = CurrentRegAndClass.Reg == INVALID_REG;
if (NeedsToSpill) {
bool Spilled = false;
@@ -1391,11 +1382,11 @@ namespace FEXCore::IR {
if (!Spilled) {
uint32_t InterferenceNode = FindNodeToSpill(IREmit, CurrentNode, Node, OpLiveRange);
if (InterferenceNode != ~0U) {
FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{uint32_t(Graph->Nodes[InterferenceNode].Head.RegAndClass >> 32)};
FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{Graph->AllocData->Map[InterferenceNode].Class};
uint32_t SpillSlot = FindSpillSlot(InterferenceNode, InterferenceRegClass);
RegisterNode *InterferenceRegisterNode = &Graph->Nodes[InterferenceNode];
LogMan::Throw::A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!");
LogMan::Throw::A((InterferenceRegisterNode->Head.RegAndClass & ~0U) != ~0U, "Interference node never assigned a register?");
//LogMan::Throw::A(InterferenceRegisterNode->Head.RegAndClass.Reg != INVALID_REG, "Interference node never assigned a register?");
LogMan::Throw::A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?");
LogMan::Throw::A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently");
LogMan::Throw::A(InterferenceRegClass == RegClass, "Class doesn't match");
@@ -1469,7 +1460,7 @@ namespace FEXCore::IR {
if (OptimizeSRA)
OptimizeStaticRegisters(&IR);
// Linear foward scan based interference calculation is faster for smaller blocks
// Linear forward scan based interference calculation is faster for smaller blocks
// Smarter block based interference calculation is faster for larger blocks
/*if (SSACount >= 2048) {
CalculateBlockInterferences(&IR);
@@ -1533,6 +1524,8 @@ namespace FEXCore::IR {
CompactionPass->Run(IREmit);
}
Graph->AllocData->SpillSlotCount = Graph->SpillStack.size();
return Changed;
}
External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h
Vendored
+8 -4
View File
@@ -1,5 +1,6 @@
#pragma once
#include "Interface/IR/PassManager.h"
#include <FEXCore/IR/RegisterAllocationData.h>
#include <vector>
namespace FEXCore::IR {
@@ -9,7 +10,6 @@ class IRListView;
class RegisterAllocationPass : public FEXCore::IR::Pass {
public:
bool HasFullRA() const { return HadFullRA; }
uint32_t SpillSlots() const { return SpillSlotCount; }
virtual void AllocateRegisterSet(uint32_t RegisterCount, uint32_t ClassCount) = 0;
virtual void AddRegisters(FEXCore::IR::RegisterClassType Class, uint32_t RegisterCount) = 0;
@@ -33,10 +33,14 @@ class RegisterAllocationPass : public FEXCore::IR::Pass {
* @{ */
/**
* @brief Returns the register and class encoded together
* Top 32bits is the class, lower 32bits is the register
* @brief Returns the register and class map array
*/
virtual uint64_t GetNodeRegister(uint32_t Node) = 0;
virtual RegisterAllocationData *GetAllocationData() = 0;
/**
* @brief Returns and transfers ownership of the register and class map array
*/
virtual std::unique_ptr<RegisterAllocationData, RegisterAllocationDataDeleter> PullAllocationData() = 0;
/** @} */
protected:
External/FEXCore/include/FEXCore/Core/CPUBackend.h
Vendored
+2 -1
View File
@@ -7,6 +7,7 @@ namespace FEXCore {
namespace IR {
template<bool Copy>
class IRListView;
class RegisterAllocationData;
}
namespace Core {
@@ -44,7 +45,7 @@ class LLVMCore;
* @return An executable function pointer that is theoretically compiled from this point.
* Is actually a function pointer of type `void (FEXCore::Core::ThreadState *Thread)
*/
virtual void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) = 0;
virtual void *CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) = 0;
/**
* @brief Function for mapping memory in to the CPUBackend's visible space. Allows setting up virtual mappings if required
External/FEXCore/include/FEXCore/Debug/InternalThreadState.h
Vendored
+4 -1
View File
@@ -3,7 +3,9 @@
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/IR/IntrusiveIRList.h>
#include <FEXCore/IR/RegisterAllocationData.h>
#include <FEXCore/Utils/Event.h>
#include <map>
#include <thread>
@@ -78,7 +80,8 @@ namespace FEXCore::Core {
std::unique_ptr<FEXCore::LookupCache> LookupCache;
std::unordered_map<uint64_t, std::unique_ptr<FEXCore::IR::IRListView<true>>> IRLists;
std::unordered_map<uint64_t, FEXCore::Core::DebugData> DebugData;
std::unordered_map<uint64_t, std::unique_ptr<FEXCore::IR::RegisterAllocationData, FEXCore::IR::RegisterAllocationDataDeleter>> RALists;
std::unordered_map<uint64_t, std::unique_ptr<FEXCore::Core::DebugData>> DebugData;
std::unique_ptr<FEXCore::Frontend::Decoder> FrontendDecoder;
std::unique_ptr<FEXCore::IR::PassManager> PassManager;
External/FEXCore/include/FEXCore/IR/IR.h
Vendored
+2 -1
View File
@@ -7,6 +7,7 @@
namespace FEXCore::IR {
class RegisterAllocationPass;
class RegisterAllocationData;
/**
* @brief The IROp_Header is an dynamically sized array
@@ -459,7 +460,7 @@ template<bool>
class IRListView;
class IREmitter;
void Dump(std::stringstream *out, IRListView<false> const* IR, IR::RegisterAllocationPass *RAPass);
void Dump(std::stringstream *out, IRListView<false> const* IR, IR::RegisterAllocationData *RAData);
IREmitter* Parse(std::istream *in);
template<typename Type>
External/FEXCore/include/FEXCore/IR/RegisterAllocationData.h
Vendored
+52
View File
@@ -0,0 +1,52 @@
#pragma once
#include "IR.h"
namespace FEXCore::IR {
union PhysicalRegister {
uint8_t Raw;
struct {
uint8_t Reg: 5;
uint8_t Class: 3;
};
bool operator==(const PhysicalRegister &Other) const {
return Raw == Other.Raw;
}
PhysicalRegister(RegisterClassType Class, uint8_t Reg) : Reg(Reg), Class(Class.Val) { }
static const PhysicalRegister Invalid() {
return PhysicalRegister(InvalidClass, InvalidReg);
}
bool IsInvalid() {
return *this == Invalid();
}
};
static_assert(sizeof(PhysicalRegister) == 1);
class RegisterAllocationData;
struct RegisterAllocationDataDeleter {
void operator()(RegisterAllocationData* r) {
free(r);
}
};
class RegisterAllocationData {
public:
uint32_t SpillSlotCount {};
PhysicalRegister Map[0];
PhysicalRegister GetNodeRegister(uint32_t Node) const {
return Map[Node];
}
uint32_t SpillSlots() const { return SpillSlotCount; }
static size_t Size(uint32_t NodeCount) {
return sizeof(RegisterAllocationData) + NodeCount * sizeof(Map[0]);
}
};
}
Source/CommonCore/HostFactory.cpp
+2 -2
View File
@@ -30,7 +30,7 @@ namespace HostFactory {
explicit HostCore(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread, bool Fallback);
~HostCore() override;
std::string GetName() override { return "Host Core"; }
void* CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) override;
void* CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void *HostPtr, uint64_t VirtualGuestPtr, uint64_t Size) override {
return HostPtr;
@@ -177,7 +177,7 @@ namespace HostFactory {
ready();
}
void* HostCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) {
void* HostCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
return nullptr;
}