mirror of
https://github.com/FEX-Emu/FEX.git
synced 2026-10-10 01:00:18 +02:00
Ryan Houdek
199cfd76d8
I had to change how blocks are represented to make it easier to parse This required a fairly substantial refactor that makes it so blocks are represented differently and we can walk them sequentially. This will make future analysis easier to deal with. Had to rewrite the passes and core's parsing of the IR afterwards. Moved RA in to a optimization pass to be shared between the JIT backends This works because x86-64 and AArch64 RA can be identical. Still doesn't support PHI nodes or spilling correctly, this is the first step in the process of getting there.
1963 lines
68 KiB
C++
1963 lines
68 KiB
C++
#include "Interface/Context/Context.h"
|
|
#include "Interface/Core/DebugData.h"
|
|
#include "Interface/Core/LLVMJIT/LLVMMemoryManager.h"
|
|
#include "Interface/HLE/Syscalls.h"
|
|
|
|
#include <FEXCore/Core/CPUBackend.h>
|
|
|
|
#include <llvm-c/Core.h>
|
|
#include <llvm/ExecutionEngine/ExecutionEngine.h>
|
|
#include <llvm/InitializePasses.h>
|
|
#include <llvm/IR/IRBuilder.h>
|
|
#include <llvm/IR/IRPrintingPasses.h>
|
|
#include <llvm/IR/LLVMContext.h>
|
|
#include <llvm/IR/LegacyPassManager.h>
|
|
#include <llvm/IR/Verifier.h>
|
|
#include <llvm/Passes/PassBuilder.h>
|
|
#include <llvm/Support/raw_ostream.h>
|
|
#include <llvm/Support/TargetSelect.h>
|
|
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
|
|
#include <llvm/Transforms/Scalar.h>
|
|
#include <llvm/Transforms/Vectorize.h>
|
|
#include <vector>
|
|
|
|
#define DESTMAP_AS_MAP 1
|
|
#if DESTMAP_AS_MAP
|
|
using DestMapType = std::unordered_map<uint64_t, llvm::Value*>;
|
|
#else
|
|
using DestMapType = std::vector<llvm::Value*>;
|
|
#endif
|
|
|
|
namespace FEXCore::CPU {
|
|
|
|
static void CPUIDRun_Thunk(CPUIDEmu::FunctionResults *Results, FEXCore::CPUIDEmu *Class, uint32_t Function) {
|
|
*Results = Class->RunFunction(Function);
|
|
}
|
|
|
|
static void SetExitState_Thunk(FEXCore::Core::InternalThreadState *Thread) {
|
|
Thread->State.RunningEvents.ShouldStop = true;
|
|
}
|
|
|
|
class LLVMJITCore final : public CPUBackend {
|
|
public:
|
|
explicit LLVMJITCore(FEXCore::Core::InternalThreadState *Thread);
|
|
~LLVMJITCore() override;
|
|
std::string GetName() override { return "JIT"; }
|
|
void* CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) override ;
|
|
|
|
void *MapRegion(void *HostPtr, uint64_t GuestPtr, uint64_t Size) override {
|
|
return HostPtr;
|
|
}
|
|
|
|
bool NeedsOpDispatch() override { return true; }
|
|
|
|
private:
|
|
void HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrapperIterator *Node);
|
|
llvm::Value *CreateContextGEP(uint64_t Offset, uint8_t Size);
|
|
llvm::Value *CreateContextPtr(uint64_t Offset, uint8_t Size);
|
|
llvm::Value *CreateMemoryLoad(llvm::Value *Ptr);
|
|
void CreateMemoryStore(llvm::Value *Ptr, llvm::Value *Val);
|
|
|
|
void ValidateMemoryInVM(uint64_t Ptr, uint8_t Size, bool Load);
|
|
template<typename Type>
|
|
Type MemoryLoad_Validate(uint64_t Ptr);
|
|
template<typename Type>
|
|
void MemoryStore_Validate(uint64_t Ptr, Type Val);
|
|
|
|
void DebugPrint(uint64_t Val);
|
|
void DebugPrint128(__uint128_t Val);
|
|
|
|
FEXCore::Core::InternalThreadState *ThreadState;
|
|
FEXCore::Context::Context *CTX;
|
|
|
|
struct LLVMState {
|
|
LLVMContextRef ContextRef;
|
|
llvm::Module *MainModule;
|
|
llvm::EngineBuilder *MainEngineBuilder;
|
|
llvm::IRBuilder<> *IRBuilder;
|
|
LLVMMemoryManager *MemManager;
|
|
std::vector<llvm::ExecutionEngine*> Functions;
|
|
};
|
|
|
|
struct LLVMCurrentState {
|
|
llvm::Function *SyscallFunction;
|
|
llvm::Function *CPUIDFunction;
|
|
llvm::Function *ExitVMFunction;
|
|
llvm::Function *ValuePrinter;
|
|
|
|
llvm::Function *ValidateLoad8;
|
|
llvm::Function *ValidateLoad16;
|
|
llvm::Function *ValidateLoad32;
|
|
llvm::Function *ValidateLoad64;
|
|
llvm::Function *ValidateLoad128;
|
|
|
|
llvm::Function *ValidateStore8;
|
|
llvm::Function *ValidateStore16;
|
|
llvm::Function *ValidateStore32;
|
|
llvm::Function *ValidateStore64;
|
|
llvm::Function *ValidateStore128;
|
|
|
|
llvm::Function *DebugPrint;
|
|
llvm::Function *DebugPrint128;
|
|
|
|
llvm::Type *CPUStateType;
|
|
llvm::GlobalVariable *CPUStateVar;
|
|
llvm::LoadInst *CPUState;
|
|
|
|
llvm::BasicBlock *CurrentBlock;
|
|
std::vector<llvm::BasicBlock*> Blocks;
|
|
bool CurrentBlockHasTerm{false};
|
|
llvm::BasicBlock *ExitBlock;
|
|
};
|
|
|
|
LLVMState JITState;
|
|
LLVMCurrentState JITCurrentState;
|
|
llvm::LLVMContext *Con;
|
|
llvm::Function *Func;
|
|
|
|
// Intrinsics
|
|
llvm::CallInst *Popcount(llvm::Value *Arg) { return JITState.IRBuilder->CreateUnaryIntrinsic(llvm::Intrinsic::ctpop, Arg); }
|
|
llvm::CallInst *BSwap(llvm::Value *Arg) { return JITState.IRBuilder->CreateUnaryIntrinsic(llvm::Intrinsic::bswap, Arg); }
|
|
llvm::CallInst *CTTZ(llvm::Value *Arg) {
|
|
std::vector<llvm::Type*> ArgTypes = {
|
|
Arg->getType(),
|
|
};
|
|
std::vector<llvm::Value*> Args = {
|
|
Arg,
|
|
JITState.IRBuilder->getInt1(true),
|
|
};
|
|
|
|
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::cttz, ArgTypes, Args);
|
|
}
|
|
|
|
llvm::CallInst *CTLZ(llvm::Value *Arg) {
|
|
std::vector<llvm::Type*> ArgTypes = {
|
|
Arg->getType(),
|
|
};
|
|
std::vector<llvm::Value*> Args = {
|
|
Arg,
|
|
JITState.IRBuilder->getInt1(true),
|
|
};
|
|
|
|
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::ctlz, ArgTypes, Args);
|
|
}
|
|
|
|
llvm::CallInst *FSHL(llvm::Value *Val, llvm::Value *Val2, llvm::Value *Amt) {
|
|
std::vector<llvm::Type*> ArgTypes = {
|
|
Val->getType(),
|
|
};
|
|
std::vector<llvm::Value*> Args = {
|
|
Val,
|
|
Val2,
|
|
Amt,
|
|
};
|
|
|
|
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::fshl, ArgTypes, Args);
|
|
}
|
|
|
|
llvm::CallInst *FSHR(llvm::Value *Val, llvm::Value *Val2, llvm::Value *Amt) {
|
|
std::vector<llvm::Type*> ArgTypes = {
|
|
Val->getType(),
|
|
};
|
|
std::vector<llvm::Value*> Args = {
|
|
Val,
|
|
Val2,
|
|
Amt,
|
|
};
|
|
|
|
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::fshr, ArgTypes, Args);
|
|
}
|
|
llvm::CallInst *CycleCounter() {
|
|
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::readcyclecounter, {}, {});
|
|
}
|
|
|
|
void CreateDebugPrint(llvm::Value *Val) {
|
|
std::vector<llvm::Value*> Args;
|
|
Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast<uint64_t>(this)));
|
|
Args.emplace_back(Val);
|
|
if (Val->getType()->getIntegerBitWidth() > 64)
|
|
JITState.IRBuilder->CreateCall(JITCurrentState.DebugPrint128, Args);
|
|
else
|
|
JITState.IRBuilder->CreateCall(JITCurrentState.DebugPrint, Args);
|
|
}
|
|
|
|
void CreateGlobalVariables(llvm::ExecutionEngine *Engine, llvm::Module *FunctionModule);
|
|
|
|
llvm::Value *CastVectorToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize);
|
|
llvm::Value *CastToOpaqueStructure(llvm::Value *Arg, llvm::Type *DstType);
|
|
void SetDest(IR::OrderedNodeWrapper Op, llvm::Value *Val);
|
|
llvm::Value *GetSrc(IR::OrderedNodeWrapper Src);
|
|
|
|
DestMapType DestMap;
|
|
FEXCore::IR::IRListView<true> const *CurrentIR;
|
|
|
|
std::unordered_map<IR::OrderedNodeWrapper::NodeOffsetType, llvm::BasicBlock*> JumpTargets;
|
|
std::unordered_map<IR::OrderedNodeWrapper::NodeOffsetType, llvm::BasicBlock*> ForwardJumpTargets;
|
|
|
|
// Target Machines
|
|
const std::string arch = "x86-64";
|
|
const std::string cpu = "skylake";
|
|
const llvm::Triple TargetTriple{"x86_64", "unknown", "linux", "gnu"};
|
|
const llvm::SmallVector<std::string, 0> Attrs;
|
|
llvm::TargetMachine *LLVMTarget;
|
|
};
|
|
|
|
LLVMJITCore::LLVMJITCore(FEXCore::Core::InternalThreadState *Thread)
|
|
: ThreadState {Thread}
|
|
, CTX {Thread->CTX} {
|
|
|
|
llvm::InitializeNativeTarget();
|
|
llvm::InitializeNativeTargetAsmPrinter();
|
|
|
|
JITState.ContextRef = LLVMContextCreate();
|
|
Con = *llvm::unwrap(&JITState.ContextRef);
|
|
JITState.MainModule = new llvm::Module("Main Module", *Con);
|
|
JITState.IRBuilder = new llvm::IRBuilder<>(*Con);
|
|
JITState.MainEngineBuilder = new llvm::EngineBuilder(std::unique_ptr<llvm::Module>(JITState.MainModule));
|
|
JITState.MainEngineBuilder->setEngineKind(llvm::EngineKind::JIT);
|
|
LLVMTarget = JITState.MainEngineBuilder->selectTarget(
|
|
TargetTriple,
|
|
arch, cpu, Attrs);
|
|
|
|
JITState.MemManager = new LLVMMemoryManager();
|
|
CTX->Config.LLVM_MemoryValidation = false;
|
|
#if !DESTMAP_AS_MAP
|
|
DestMap.resize(0x1000);
|
|
#endif
|
|
}
|
|
|
|
LLVMJITCore::~LLVMJITCore() {
|
|
// MainEngineBuilder takes overship of MainModule
|
|
delete JITState.MainEngineBuilder;
|
|
delete JITState.IRBuilder;
|
|
// Causes fault when destroying MCJIT
|
|
//for (auto Module : JITState.Functions) {
|
|
// delete Module;
|
|
//}
|
|
LLVMContextDispose(JITState.ContextRef);
|
|
}
|
|
|
|
void LLVMJITCore::ValidateMemoryInVM(uint64_t Ptr, uint8_t Size, bool Load) {
|
|
uint64_t VirtualBase = CTX->MemoryMapper.GetBaseOffset<uint64_t>(0);
|
|
uint64_t VirtualEnd = VirtualBase + (1ULL << 36ULL);
|
|
if (Ptr < VirtualBase || (Ptr + Size) >= VirtualEnd) {
|
|
LogMan::Msg::A("Invalid memory load at 0x%016lx. Wasn't within virtual range [0x%016lx, 0x%015lx)", Ptr, VirtualBase, VirtualEnd);
|
|
}
|
|
LogMan::Msg::D("%s guestmem: 0x%lx", Load ? "Loading from" : "Storing", Ptr - VirtualBase);
|
|
}
|
|
|
|
void LLVMJITCore::DebugPrint(uint64_t Val) {
|
|
LogMan::Msg::I(">>>> Value in Arg: 0x%lx, %ld", Val, Val);
|
|
}
|
|
|
|
void LLVMJITCore::DebugPrint128(__uint128_t Val) {
|
|
LogMan::Msg::I(">>>Val: %016lx, %016lx", static_cast<uint64_t>(Val >> 64), static_cast<uint64_t>(Val));
|
|
}
|
|
|
|
template<typename Type>
|
|
Type LLVMJITCore::MemoryLoad_Validate(uint64_t Ptr) {
|
|
ValidateMemoryInVM(Ptr, sizeof(Type), true);
|
|
Type *TypedAddr = reinterpret_cast<Type*>(Ptr);
|
|
Type Ret = TypedAddr[0];
|
|
uint64_t Data;
|
|
memcpy(&Data, &Ret, sizeof(Data));
|
|
LogMan::Msg::D("\tLoading: 0x%016lx", Data);
|
|
return Ret;
|
|
}
|
|
|
|
template<typename Type>
|
|
void LLVMJITCore::MemoryStore_Validate(uint64_t Ptr, Type Val) {
|
|
ValidateMemoryInVM(Ptr, sizeof(Type), false);
|
|
Type *TypedAddr = reinterpret_cast<Type*>(Ptr);
|
|
TypedAddr[0] = Val;
|
|
uint64_t Data;
|
|
memcpy(&Data, &Val, sizeof(Data));
|
|
LogMan::Msg::D("\tStoring: 0x%016lx", Data);
|
|
}
|
|
|
|
llvm::Value *LLVMJITCore::CreateMemoryLoad(llvm::Value *Ptr) {
|
|
if (CTX->Config.LLVM_MemoryValidation) {
|
|
std::vector<llvm::Value*> Args;
|
|
Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast<uint64_t>(this)));
|
|
Args.emplace_back(Ptr);
|
|
|
|
unsigned PtrSize = Ptr->getType()->getPointerElementType()->getIntegerBitWidth();
|
|
switch (PtrSize) {
|
|
case 8: return JITState.IRBuilder->CreateCall(JITCurrentState.ValidateLoad8, Args);
|
|
case 16: return JITState.IRBuilder->CreateCall(JITCurrentState.ValidateLoad16, Args);
|
|
case 32: return JITState.IRBuilder->CreateCall(JITCurrentState.ValidateLoad32, Args);
|
|
case 64: return JITState.IRBuilder->CreateCall(JITCurrentState.ValidateLoad64, Args);
|
|
case 128: return JITState.IRBuilder->CreateCall(JITCurrentState.ValidateLoad128, Args);
|
|
default: LogMan::Msg::A("Unknown Load Size: %d", PtrSize); break;
|
|
}
|
|
}
|
|
|
|
return JITState.IRBuilder->CreateLoad(Ptr);
|
|
}
|
|
|
|
void LLVMJITCore::CreateMemoryStore(llvm::Value *Ptr, llvm::Value *Val) {
|
|
if (CTX->Config.LLVM_MemoryValidation) {
|
|
std::vector<llvm::Value*> Args;
|
|
Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast<uint64_t>(this)));
|
|
Args.emplace_back(Ptr);
|
|
Args.emplace_back(Val);
|
|
|
|
unsigned PtrSize = Ptr->getType()->getPointerElementType()->getIntegerBitWidth();
|
|
switch (PtrSize) {
|
|
case 8: JITState.IRBuilder->CreateCall(JITCurrentState.ValidateStore8, Args); break;
|
|
case 16: JITState.IRBuilder->CreateCall(JITCurrentState.ValidateStore16, Args); break;
|
|
case 32: JITState.IRBuilder->CreateCall(JITCurrentState.ValidateStore32, Args); break;
|
|
case 64: JITState.IRBuilder->CreateCall(JITCurrentState.ValidateStore64, Args); break;
|
|
case 128: JITState.IRBuilder->CreateCall(JITCurrentState.ValidateStore128, Args); break;
|
|
default: LogMan::Msg::A("Unknown Store Size: %d", PtrSize); break;
|
|
}
|
|
return;
|
|
}
|
|
|
|
JITState.IRBuilder->CreateStore(Val, Ptr);
|
|
}
|
|
|
|
|
|
void LLVMJITCore::CreateGlobalVariables(llvm::ExecutionEngine *Engine, llvm::Module *FunctionModule) {
|
|
using namespace llvm;
|
|
Type *voidTy = Type::getVoidTy(*Con);
|
|
Type *i8 = Type::getInt8Ty(*Con);
|
|
Type *i16 = Type::getInt16Ty(*Con);
|
|
Type *i32 = Type::getInt32Ty(*Con);
|
|
Type *i64 = Type::getInt64Ty(*Con);
|
|
Type *i128 = Type::getInt128Ty(*Con);
|
|
|
|
// Syscall Function
|
|
{
|
|
auto FuncType = FunctionType::get(i64,
|
|
{
|
|
i64, // Technically a this pointer
|
|
i64,
|
|
ArrayType::get(i64, 7)->getPointerTo(),
|
|
},
|
|
false);
|
|
JITCurrentState.SyscallFunction = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"Syscall",
|
|
FunctionModule);
|
|
using ClassPtrType = uint64_t (FEXCore::SyscallHandler::*)(FEXCore::Core::InternalThreadState *, FEXCore::HLE::SyscallArguments *);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &FEXCore::SyscallHandler::HandleSyscall;
|
|
Engine->addGlobalMapping(JITCurrentState.SyscallFunction, Ptr.Data);
|
|
}
|
|
|
|
// CPUID Function
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{
|
|
ArrayType::get(i32, 4)->getPointerTo(),
|
|
i64, // Technically this is a pointer
|
|
i32, // CPUID Function
|
|
},
|
|
false);
|
|
JITCurrentState.CPUIDFunction = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"CPUID",
|
|
FunctionModule);
|
|
using ClassPtrType = void (*)(FEXCore::CPUIDEmu::FunctionResults*, FEXCore::CPUIDEmu*, uint32_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &CPUIDRun_Thunk;
|
|
Engine->addGlobalMapping(JITCurrentState.CPUIDFunction, Ptr.Data);
|
|
}
|
|
|
|
// Exit VM function
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{
|
|
i64, // Technically this is a pointer
|
|
},
|
|
false);
|
|
JITCurrentState.ExitVMFunction = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"ExitVM",
|
|
FunctionModule);
|
|
using ClassPtrType = void (*)(FEXCore::Core::InternalThreadState *Thread);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &SetExitState_Thunk;
|
|
Engine->addGlobalMapping(JITCurrentState.ExitVMFunction, Ptr.Data);
|
|
}
|
|
|
|
if (CTX->Config.LLVM_MemoryValidation) {
|
|
// Memory validate load 8
|
|
{
|
|
auto FuncType = FunctionType::get(i8,
|
|
{i64, // this pointer
|
|
i8->getPointerTo()}, false);
|
|
JITCurrentState.ValidateLoad8 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"LoadValidate8",
|
|
FunctionModule);
|
|
using ClassPtrType = uint8_t (LLVMJITCore::*)(uint64_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryLoad_Validate<uint8_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateLoad8, Ptr.Data);
|
|
}
|
|
// Memory validate load 16
|
|
{
|
|
auto FuncType = FunctionType::get(i16,
|
|
{i64, // this pointer
|
|
i16->getPointerTo()}, false);
|
|
JITCurrentState.ValidateLoad16 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"LoadValidate16",
|
|
FunctionModule);
|
|
using ClassPtrType = uint16_t (LLVMJITCore::*)(uint64_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryLoad_Validate<uint16_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateLoad16, Ptr.Data);
|
|
}
|
|
// Memory validate load 32
|
|
{
|
|
auto FuncType = FunctionType::get(i32,
|
|
{i64, // this pointer
|
|
i32->getPointerTo()}, false);
|
|
|
|
JITCurrentState.ValidateLoad32 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"LoadValidate32",
|
|
FunctionModule);
|
|
using ClassPtrType = uint32_t (LLVMJITCore::*)(uint64_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryLoad_Validate<uint32_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateLoad32, Ptr.Data);
|
|
}
|
|
// Memory validate load 64
|
|
{
|
|
auto FuncType = FunctionType::get(i64,
|
|
{i64, // this pointer
|
|
i64->getPointerTo()}, false);
|
|
JITCurrentState.ValidateLoad64 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"LoadValidate64",
|
|
FunctionModule);
|
|
using ClassPtrType = uint64_t (LLVMJITCore::*)(uint64_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryLoad_Validate<uint64_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateLoad64, Ptr.Data);
|
|
}
|
|
// Memory validate load 128
|
|
{
|
|
auto FuncType = FunctionType::get(i128,
|
|
{i64, // this pointer
|
|
i128->getPointerTo()}, false);
|
|
JITCurrentState.ValidateLoad128 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"LoadValidate128",
|
|
FunctionModule);
|
|
using ClassPtrType = __uint128_t (LLVMJITCore::*)(uint64_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryLoad_Validate<__uint128_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateLoad128, Ptr.Data);
|
|
}
|
|
|
|
// Memory validate Store 8
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{i64, // this pointer
|
|
i8->getPointerTo(),
|
|
i8}, false);
|
|
JITCurrentState.ValidateStore8 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"StoreValidate8",
|
|
FunctionModule);
|
|
using ClassPtrType = void (LLVMJITCore::*)(uint64_t, uint8_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryStore_Validate<uint8_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateStore8, Ptr.Data);
|
|
}
|
|
|
|
// Memory validate Store 16
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{i64, // this pointer
|
|
i16->getPointerTo(),
|
|
i16}, false);
|
|
JITCurrentState.ValidateStore16 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"StoreValidate16",
|
|
FunctionModule);
|
|
using ClassPtrType = void (LLVMJITCore::*)(uint64_t, uint16_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryStore_Validate<uint16_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateStore16, Ptr.Data);
|
|
}
|
|
|
|
// Memory validate Store 32
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{i64, // this pointer
|
|
i32->getPointerTo(),
|
|
i32}, false);
|
|
JITCurrentState.ValidateStore32 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"StoreValidate32",
|
|
FunctionModule);
|
|
using ClassPtrType = void (LLVMJITCore::*)(uint64_t, uint32_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryStore_Validate<uint32_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateStore32, Ptr.Data);
|
|
}
|
|
|
|
// Memory validate Store 64
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{i64, // this pointer
|
|
i64->getPointerTo(),
|
|
i64}, false);
|
|
JITCurrentState.ValidateStore64 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"StoreValidate64",
|
|
FunctionModule);
|
|
using ClassPtrType = void (LLVMJITCore::*)(uint64_t, uint64_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryStore_Validate<uint64_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateStore64, Ptr.Data);
|
|
}
|
|
|
|
// Memory validate Store 128
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{i64, // this pointer
|
|
i128->getPointerTo(),
|
|
i128}, false);
|
|
JITCurrentState.ValidateStore128 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"StoreValidate128",
|
|
FunctionModule);
|
|
using ClassPtrType = void (LLVMJITCore::*)(uint64_t, __uint128_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::MemoryStore_Validate<__uint128_t>;
|
|
Engine->addGlobalMapping(JITCurrentState.ValidateStore128, Ptr.Data);
|
|
}
|
|
}
|
|
|
|
// Value Print
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{i64, // this pointer
|
|
i64}, false);
|
|
JITCurrentState.DebugPrint = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"PrintVal",
|
|
FunctionModule);
|
|
using ClassPtrType = void (LLVMJITCore::*)(uint64_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::DebugPrint;
|
|
Engine->addGlobalMapping(JITCurrentState.DebugPrint, Ptr.Data);
|
|
}
|
|
|
|
// Value Print 128
|
|
{
|
|
auto FuncType = FunctionType::get(voidTy,
|
|
{i64, // this pointer
|
|
i128}, false);
|
|
JITCurrentState.DebugPrint128 = Function::Create(FuncType,
|
|
Function::ExternalLinkage,
|
|
"PrintVal128",
|
|
FunctionModule);
|
|
using ClassPtrType = void (LLVMJITCore::*)(__uint128_t);
|
|
union PtrCast {
|
|
ClassPtrType ClassPtr;
|
|
void* Data;
|
|
};
|
|
PtrCast Ptr;
|
|
Ptr.ClassPtr = &LLVMJITCore::DebugPrint128;
|
|
Engine->addGlobalMapping(JITCurrentState.DebugPrint128, Ptr.Data);
|
|
}
|
|
|
|
// JIT State
|
|
{
|
|
JITCurrentState.CPUStateType = StructType::create(*Con,
|
|
{
|
|
i64, // RIP
|
|
ArrayType::get(i64, 16), // Gregs
|
|
i64, // Pad to ensure alignment
|
|
ArrayType::get(i128, 16), // XMMs
|
|
i64, i64, // GS, FS
|
|
ArrayType::get(i8, 32), //rflags
|
|
},
|
|
"CPUStateType");
|
|
|
|
FunctionModule->getOrInsertGlobal("X86State::State", JITCurrentState.CPUStateType->getPointerTo());
|
|
JITCurrentState.CPUStateVar = FunctionModule->getNamedGlobal("X86State::State");
|
|
JITCurrentState.CPUStateVar->setConstant(true);
|
|
JITCurrentState.CPUStateVar->setInitializer(
|
|
ConstantInt::getIntegerValue(
|
|
JITCurrentState.CPUStateType->getPointerTo(),
|
|
APInt(64, reinterpret_cast<uint64_t>(&ThreadState->State))));
|
|
JITCurrentState.CPUState = JITState.IRBuilder->CreateLoad(JITCurrentState.CPUStateVar, false, "X86State::State::Local");
|
|
}
|
|
}
|
|
|
|
llvm::Value *LLVMJITCore::CreateContextGEP(uint64_t Offset, uint8_t Size) {
|
|
std::vector<llvm::Value*> GEPValues = {
|
|
JITState.IRBuilder->getInt32(0), // First value in the pointer to CPUState
|
|
};
|
|
|
|
if (Offset == 0) { // RIP
|
|
if (Size != 8) return nullptr;
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32(0));
|
|
}
|
|
else if (Offset >= offsetof(FEXCore::Core::CPUState, gregs) && Offset < offsetof(FEXCore::Core::CPUState, xmm)) {
|
|
if (Size != 8 || Offset % 8 != 0) return nullptr;
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32(1));
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32((Offset - offsetof(FEXCore::Core::CPUState, gregs)) / 8));
|
|
}
|
|
else if (Offset >= offsetof(FEXCore::Core::CPUState, xmm) && Offset < offsetof(FEXCore::Core::CPUState, gs)) {
|
|
if (Size != 16 || Offset % 16 != 0) return nullptr;
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32(3));
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32((Offset - offsetof(FEXCore::Core::CPUState, xmm)) / 16));
|
|
}
|
|
else if (Offset == offsetof(FEXCore::Core::CPUState, gs)) {
|
|
if (Size != 8) return nullptr;
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32(4));
|
|
}
|
|
else if (Offset == offsetof(FEXCore::Core::CPUState, fs)) {
|
|
if (Size != 8) return nullptr;
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32(5));
|
|
}
|
|
else if (Offset >= offsetof(FEXCore::Core::CPUState, flags)) {
|
|
if (Size != 1) return nullptr;
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32(6));
|
|
GEPValues.emplace_back(JITState.IRBuilder->getInt32(Offset - offsetof(FEXCore::Core::CPUState, flags[0])));
|
|
}
|
|
else
|
|
LogMan::Msg::A("Unknown X86State GEP: 0x%lx", Offset);
|
|
|
|
return JITState.IRBuilder->CreateGEP(JITCurrentState.CPUState, GEPValues, "Context::Value");
|
|
}
|
|
|
|
llvm::Value *LLVMJITCore::CreateContextPtr(uint64_t Offset, uint8_t Size) {
|
|
llvm::Type *i8 = llvm::Type::getInt8Ty(*Con);
|
|
llvm::Type *i16 = llvm::Type::getInt16Ty(*Con);
|
|
llvm::Type *i32 = llvm::Type::getInt32Ty(*Con);
|
|
llvm::Type *i64 = llvm::Type::getInt64Ty(*Con);
|
|
llvm::Type *i128 = llvm::Type::getInt128Ty(*Con);
|
|
|
|
// Let's try to create our pointer with GEP
|
|
// This can only happen if we are a full value from the context and is aligned correctly
|
|
llvm::Value *GEPResult = CreateContextGEP(Offset, Size);
|
|
if (GEPResult) return GEPResult;
|
|
|
|
llvm::Value *StateBasePtr = JITState.IRBuilder->CreatePtrToInt(JITCurrentState.CPUState, i64);
|
|
StateBasePtr = JITState.IRBuilder->CreateAdd(StateBasePtr, JITState.IRBuilder->getInt64(Offset));
|
|
|
|
// Convert back to pointer of correct size
|
|
switch (Size) {
|
|
case 1: return JITState.IRBuilder->CreateIntToPtr(StateBasePtr, i8->getPointerTo());
|
|
case 2: return JITState.IRBuilder->CreateIntToPtr(StateBasePtr, i16->getPointerTo());
|
|
case 4: return JITState.IRBuilder->CreateIntToPtr(StateBasePtr, i32->getPointerTo());
|
|
case 8: return JITState.IRBuilder->CreateIntToPtr(StateBasePtr, i64->getPointerTo());
|
|
case 16: return JITState.IRBuilder->CreateIntToPtr(StateBasePtr, i128->getPointerTo());
|
|
default: LogMan::Msg::A("Unknown context pointer size: %d", Size); break;
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
llvm::Value *LLVMJITCore::CastVectorToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize) {
|
|
uint8_t NumElements = RegisterSize / ElementSize;
|
|
llvm::Type *ElementType;
|
|
if (Integer) {
|
|
ElementType = llvm::Type::getIntNTy(*Con, ElementSize * 8);
|
|
}
|
|
else {
|
|
if (ElementSize == 4) {
|
|
ElementType = llvm::Type::getFloatTy(*Con);
|
|
}
|
|
else {
|
|
ElementType = llvm::Type::getDoubleTy(*Con);
|
|
}
|
|
}
|
|
|
|
llvm::Type *VectorType = llvm::VectorType::get(ElementType, NumElements);
|
|
|
|
// This happens frequently
|
|
// If the source argument isn't of vector type then BitCast fails moving from Scalar->Vector domains
|
|
// Need to create a vector and insert elements in to that vector from the scalar type instead
|
|
if (!Arg->getType()->isVectorTy()) {
|
|
return JITState.IRBuilder->CreateBitCast(Arg, VectorType);
|
|
}
|
|
|
|
return JITState.IRBuilder->CreateBitCast(Arg, VectorType);
|
|
}
|
|
|
|
llvm::Value *LLVMJITCore::CastToOpaqueStructure(llvm::Value *Arg, llvm::Type *DstType) {
|
|
if (Arg->getType()->isVectorTy()) {
|
|
// First do a bitcast from the vector type to the same size integer
|
|
unsigned ElementSize = Arg->getType()->getVectorElementType()->getIntegerBitWidth();
|
|
unsigned NumElements = Arg->getType()->getVectorNumElements();
|
|
auto NewIntegerType = llvm::Type::getIntNTy(*Con, ElementSize * NumElements);
|
|
Arg = JITState.IRBuilder->CreateBitCast(Arg, NewIntegerType);
|
|
}
|
|
|
|
return JITState.IRBuilder->CreateZExtOrTrunc(Arg, DstType->getPointerElementType());
|
|
}
|
|
|
|
void LLVMJITCore::SetDest(IR::OrderedNodeWrapper Op, llvm::Value *Val) {
|
|
DestMap[Op.ID()] = Val;
|
|
}
|
|
|
|
llvm::Value *LLVMJITCore::GetSrc(IR::OrderedNodeWrapper Src) {
|
|
#if DESTMAP_AS_MAP
|
|
LogMan::Throw::A(DestMap.find(Src.ID()) != DestMap.end(), "Op had Src but wasn't added to the dest map");
|
|
#endif
|
|
|
|
auto DstPtr = DestMap[Src.ID()];
|
|
LogMan::Throw::A(DstPtr != nullptr, "Destmap had slot but wasn't allocated memory");
|
|
return DstPtr;
|
|
}
|
|
|
|
void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrapperIterator *Node) {
|
|
using namespace llvm;
|
|
|
|
uintptr_t ListBegin = CurrentIR->GetListData();
|
|
uintptr_t DataBegin = CurrentIR->GetData();
|
|
|
|
IR::OrderedNodeWrapper *WrapperOp = (*Node)();
|
|
IR::OrderedNode *RealNode = WrapperOp->GetNode(ListBegin);
|
|
FEXCore::IR::IROp_Header *IROp = RealNode->Op(DataBegin);
|
|
uint8_t OpSize = IROp->Size;
|
|
|
|
switch (IROp->Op) {
|
|
case FEXCore::IR::IROps::OP_BEGINBLOCK: {
|
|
auto ForwardIt = ForwardJumpTargets.find(WrapperOp->NodeOffset);
|
|
if (ForwardIt != ForwardJumpTargets.end()) {
|
|
// This block has already been created for us, just move over to it
|
|
JITState.IRBuilder->SetInsertPoint(ForwardIt->second);
|
|
}
|
|
else {
|
|
auto Block = BasicBlock::Create(*Con, "BeginBlock", Func);
|
|
JITCurrentState.Blocks.emplace_back(Block);
|
|
|
|
// Blocks can be jump targets
|
|
JumpTargets[WrapperOp->NodeOffset] = Block;
|
|
|
|
// We need to do a jump from previous block to this block
|
|
// This ensures block fallthrough works
|
|
// Although if the previously block already had a terminator then skip the jump
|
|
if (!JITCurrentState.CurrentBlockHasTerm) {
|
|
JITState.IRBuilder->CreateBr(Block);
|
|
}
|
|
JITState.IRBuilder->SetInsertPoint(Block);
|
|
JITCurrentState.CurrentBlock = Block;
|
|
JITCurrentState.CurrentBlockHasTerm = false;
|
|
}
|
|
}
|
|
break;
|
|
case IR::OP_ENDBLOCK: {
|
|
auto Op = IROp->C<IR::IROp_EndBlock>();
|
|
|
|
if (Op->RIPIncrement) {
|
|
auto DownCountValue = JITState.IRBuilder->CreateGEP(JITCurrentState.CPUState,
|
|
{
|
|
JITState.IRBuilder->getInt32(0),
|
|
JITState.IRBuilder->getInt32(0),
|
|
},
|
|
"RIPIncrement");
|
|
auto LoadRIP = JITState.IRBuilder->CreateLoad(DownCountValue);
|
|
auto NewValue = JITState.IRBuilder->CreateAdd(LoadRIP, JITState.IRBuilder->getInt64(Op->RIPIncrement));
|
|
JITState.IRBuilder->CreateStore(NewValue, DownCountValue);
|
|
}
|
|
|
|
// If we hit an end block that isn't at the end of the stream that means we need to early exit
|
|
// Just set ourselves to the end regardless
|
|
if (CTX->Config.Multiblock) {
|
|
// Fall through to the next block
|
|
// Just in case some additional garbage needs to fall through
|
|
auto Block = BasicBlock::Create(*Con, "EndBlock_Fallthrough", Func);
|
|
JITCurrentState.Blocks.emplace_back(Block);
|
|
|
|
if (!JITCurrentState.CurrentBlockHasTerm) {
|
|
JITState.IRBuilder->CreateBr(Block);
|
|
}
|
|
|
|
JITState.IRBuilder->SetInsertPoint(Block);
|
|
JITCurrentState.CurrentBlock = Block;
|
|
JITCurrentState.CurrentBlockHasTerm = false;
|
|
|
|
}
|
|
else {
|
|
if (!JITCurrentState.CurrentBlockHasTerm) {
|
|
JITState.IRBuilder->CreateBr(JITCurrentState.ExitBlock);
|
|
auto Block = BasicBlock::Create(*Con, "EndBlock_Fallthrough", Func);
|
|
JITCurrentState.Blocks.emplace_back(Block);
|
|
|
|
JITState.IRBuilder->SetInsertPoint(Block);
|
|
JITCurrentState.CurrentBlock = Block;
|
|
JITCurrentState.CurrentBlockHasTerm = false;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case IR::OP_BREAK: {
|
|
std::vector<llvm::Value*> Args;
|
|
// We need to pull this argument from the ExecuteCodeFunction
|
|
Args.emplace_back(Func->args().begin());
|
|
|
|
JITState.IRBuilder->CreateCall(JITCurrentState.ExitVMFunction, Args);
|
|
JITState.IRBuilder->CreateBr(JITCurrentState.ExitBlock);
|
|
|
|
// Just in case some additional garbage needs to fall through
|
|
auto Block = BasicBlock::Create(*Con, "Break_Fallthrough", Func);
|
|
JITCurrentState.Blocks.emplace_back(Block);
|
|
|
|
JITState.IRBuilder->SetInsertPoint(Block);
|
|
JITCurrentState.CurrentBlock = Block;
|
|
JITCurrentState.CurrentBlockHasTerm = false;
|
|
break;
|
|
}
|
|
case IR::OP_EXITFUNCTION:
|
|
case IR::OP_ENDFUNCTION: {
|
|
JITState.IRBuilder->CreateBr(JITCurrentState.ExitBlock);
|
|
JITCurrentState.CurrentBlockHasTerm = true;
|
|
break;
|
|
}
|
|
case IR::OP_JUMP: {
|
|
auto Op = IROp->C<IR::IROp_Jump>();
|
|
auto JumpTarget = Op->Header.Args[0].NodeOffset;
|
|
llvm::BasicBlock *Target;
|
|
auto ForwardIt = ForwardJumpTargets.find(JumpTarget);
|
|
if (ForwardIt == ForwardJumpTargets.end()) {
|
|
// If the target doesn't yet exist then create it now
|
|
Target = BasicBlock::Create(*Con, "ForwardJump_Target", Func);
|
|
JITCurrentState.Blocks.emplace_back(Target);
|
|
ForwardJumpTargets[JumpTarget] = Target;
|
|
}
|
|
else {
|
|
// If we have the branch created already then we can just jump to it
|
|
Target = ForwardIt->second;
|
|
}
|
|
JITState.IRBuilder->CreateBr(Target);
|
|
JITCurrentState.CurrentBlockHasTerm = true;
|
|
break;
|
|
}
|
|
case IR::OP_CONDJUMP: {
|
|
auto Op = IROp->C<IR::IROp_CondJump>();
|
|
auto Cond = GetSrc(Op->Header.Args[0]);
|
|
auto JumpTarget = Op->Header.Args[1].NodeOffset;
|
|
|
|
auto Comp = JITState.IRBuilder->CreateICmpNE(Cond, JITState.IRBuilder->getInt64(0));
|
|
if (JumpTarget < WrapperOp->NodeOffset) {
|
|
// Backwards branch means the target block is already created
|
|
auto Block = BasicBlock::Create(*Con, "CondJump_FalseBlock", Func);
|
|
JITCurrentState.Blocks.emplace_back(Block);
|
|
|
|
JITState.IRBuilder->CreateCondBr(Comp, JumpTargets[JumpTarget], Block);
|
|
JITState.IRBuilder->SetInsertPoint(Block);
|
|
JITCurrentState.CurrentBlock = Block;
|
|
JITCurrentState.CurrentBlockHasTerm = false;
|
|
}
|
|
else {
|
|
// If we are forward jumping then we need to create two new blocks
|
|
// One for continuing execution and another for the true conditional path
|
|
// If the target already exists in the forward block map then we just use that
|
|
llvm::BasicBlock *TrueBlock;
|
|
auto ForwardIt = ForwardJumpTargets.find(JumpTarget);
|
|
if (ForwardIt == ForwardJumpTargets.end()) {
|
|
// Add the True path to our forward block map so when we hit it in the future we can just use it
|
|
auto Block = BasicBlock::Create(*Con, "CondJump_TrueBlock", Func);
|
|
JITCurrentState.Blocks.emplace_back(Block);
|
|
|
|
ForwardJumpTargets[JumpTarget] = Block;
|
|
TrueBlock = Block;
|
|
}
|
|
else {
|
|
TrueBlock = ForwardIt->second;
|
|
}
|
|
auto Block = BasicBlock::Create(*Con, "CondJump_FalseBlock", Func);
|
|
JITCurrentState.Blocks.emplace_back(Block);
|
|
|
|
JITState.IRBuilder->CreateCondBr(Comp, TrueBlock, Block);
|
|
JITState.IRBuilder->SetInsertPoint(Block);
|
|
JITCurrentState.CurrentBlock = Block;
|
|
JITCurrentState.CurrentBlockHasTerm = false;
|
|
}
|
|
break;
|
|
}
|
|
case IR::OP_MOV: {
|
|
auto Op = IROp->C<IR::IROp_Mov>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
SetDest(*WrapperOp, Src);
|
|
break;
|
|
}
|
|
case IR::OP_SELECT: {
|
|
auto Op = IROp->C<IR::IROp_Select>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
auto ArgTrue = GetSrc(Op->Header.Args[2]);
|
|
auto ArgFalse = GetSrc(Op->Header.Args[3]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
ArgFalse = JITState.IRBuilder->CreateZExtOrTrunc(ArgFalse, ArgTrue->getType());
|
|
|
|
Value *Cmp{};
|
|
switch (Op->Cond) {
|
|
case FEXCore::IR::COND_EQ:
|
|
Cmp = JITState.IRBuilder->CreateICmpEQ(Src1, Src2);
|
|
break;
|
|
case FEXCore::IR::COND_NEQ:
|
|
Cmp = JITState.IRBuilder->CreateICmpNE(Src1, Src2);
|
|
break;
|
|
case FEXCore::IR::COND_GE:
|
|
Cmp = JITState.IRBuilder->CreateICmpUGE(Src1, Src2);
|
|
break;
|
|
case FEXCore::IR::COND_LT:
|
|
Cmp = JITState.IRBuilder->CreateICmpULT(Src1, Src2);
|
|
break;
|
|
case FEXCore::IR::COND_GT:
|
|
Cmp = JITState.IRBuilder->CreateICmpUGT(Src1, Src2);
|
|
break;
|
|
case FEXCore::IR::COND_LE:
|
|
Cmp = JITState.IRBuilder->CreateICmpULE(Src1, Src2);
|
|
break;
|
|
default: LogMan::Msg::A("Unknown Select Op Type: %d", Op->Cond); break;
|
|
}
|
|
|
|
auto Result = JITState.IRBuilder->CreateSelect(Cmp, ArgTrue, ArgFalse);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case FEXCore::IR::IROps::OP_CONSTANT: {
|
|
auto Op = IROp->C<IR::IROp_Constant>();
|
|
auto Result = JITState.IRBuilder->getInt64(Op->Constant);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case FEXCore::IR::IROps::OP_SYSCALL: {
|
|
auto Op = IROp->C<IR::IROp_Syscall>();
|
|
|
|
std::vector<llvm::Value*> Args;
|
|
Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast<uint64_t>(&CTX->SyscallHandler)));
|
|
// We need to pull this argument from the ExecuteCodeFunction
|
|
Args.emplace_back(Func->args().begin());
|
|
|
|
auto LLVMArgs = JITState.IRBuilder->CreateAlloca(ArrayType::get(Type::getInt64Ty(*Con), 7));
|
|
for (unsigned i = 0; i < 7; ++i) {
|
|
auto Location = JITState.IRBuilder->CreateGEP(LLVMArgs,
|
|
{
|
|
JITState.IRBuilder->getInt32(0),
|
|
JITState.IRBuilder->getInt32(i),
|
|
},
|
|
"Arg");
|
|
auto Src = GetSrc(Op->Header.Args[i]);
|
|
JITState.IRBuilder->CreateStore(Src, Location);
|
|
}
|
|
Args.emplace_back(LLVMArgs);
|
|
|
|
auto Result = JITState.IRBuilder->CreateCall(JITCurrentState.SyscallFunction, Args);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_CPUID: {
|
|
auto Op = IROp->C<IR::IROp_CPUID>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
std::vector<llvm::Value*> Args{};
|
|
|
|
auto ReturnType = ArrayType::get(Type::getInt32Ty(*Con), 4);
|
|
auto LLVMArgs = JITState.IRBuilder->CreateAlloca(ReturnType);
|
|
Args.emplace_back(LLVMArgs);
|
|
Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast<uint64_t>(&CTX->CPUID)));
|
|
Args.emplace_back(Src);
|
|
JITState.IRBuilder->CreateCall(JITCurrentState.CPUIDFunction, Args);
|
|
auto Result = JITState.IRBuilder->CreateLoad(ReturnType, LLVMArgs);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
// The IR's current representation of vectors is actually an array
|
|
case IR::OP_EXTRACTELEMENT: {
|
|
auto Op = IROp->C<IR::IROp_ExtractElement>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
std::vector<unsigned> Idxs = {Op->Idx};
|
|
auto Result = JITState.IRBuilder->CreateExtractValue(Src, Idxs);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_LOADCONTEXT: {
|
|
auto Op = IROp->C<IR::IROp_LoadContext>();
|
|
auto Value = CreateContextPtr(Op->Offset, Op->Size);
|
|
llvm::Value *Load;
|
|
if ((Op->Offset % Op->Size) == 0)
|
|
Load = JITState.IRBuilder->CreateAlignedLoad(Value, Op->Size);
|
|
else
|
|
Load = JITState.IRBuilder->CreateLoad(Value);
|
|
SetDest(*WrapperOp, Load);
|
|
break;
|
|
}
|
|
case IR::OP_STORECONTEXT: {
|
|
auto Op = IROp->C<IR::IROp_StoreContext>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
auto Value = CreateContextPtr(Op->Offset, Op->Size);
|
|
Src = CastToOpaqueStructure(Src, Value->getType());
|
|
|
|
if ((Op->Offset % Op->Size) == 0)
|
|
JITState.IRBuilder->CreateAlignedStore(Src, Value, Op->Size);
|
|
else
|
|
JITState.IRBuilder->CreateStore(Src, Value);
|
|
break;
|
|
}
|
|
case IR::OP_LOADFLAG: {
|
|
auto Op = IROp->C<IR::IROp_LoadFlag>();
|
|
auto Value = CreateContextPtr(offsetof(FEXCore::Core::CPUState, flags) + Op->Flag, 1);
|
|
auto Load = JITState.IRBuilder->CreateLoad(Value);
|
|
SetDest(*WrapperOp, Load);
|
|
break;
|
|
}
|
|
case IR::OP_STOREFLAG: {
|
|
auto Op = IROp->C<IR::IROp_StoreFlag>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
auto Value = CreateContextPtr(offsetof(FEXCore::Core::CPUState, flags) + Op->Flag, 1);
|
|
Src = JITState.IRBuilder->CreateZExtOrTrunc(Src, Type::getInt8Ty(*Con));
|
|
Src = JITState.IRBuilder->CreateAnd(Src, JITState.IRBuilder->getInt8(1));
|
|
|
|
JITState.IRBuilder->CreateStore(Src, Value);
|
|
break;
|
|
}
|
|
case IR::OP_ADD: {
|
|
auto Op = IROp->C<IR::IROp_Add>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateAdd(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_SUB: {
|
|
auto Op = IROp->C<IR::IROp_Add>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateSub(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_XOR: {
|
|
auto Op = IROp->C<IR::IROp_Xor>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateXor(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_BFE: {
|
|
auto Op = IROp->C<IR::IROp_Bfe>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
LogMan::Throw::A(OpSize <= 16, "OpSize is too large for BFE: %d", OpSize);
|
|
|
|
auto BitWidth = Src->getType()->getIntegerBitWidth();
|
|
if (OpSize == 16) {
|
|
LogMan::Throw::A(Op->Width <= 64, "Can't extract width of %d", Op->Width);
|
|
|
|
// Generate our 128bit mask
|
|
auto SourceMask = JITState.IRBuilder->CreateShl(JITState.IRBuilder->getIntN(BitWidth, 1), JITState.IRBuilder->getIntN(BitWidth, Op->Width));
|
|
SourceMask = JITState.IRBuilder->CreateSub(SourceMask, JITState.IRBuilder->getIntN(BitWidth, 1));
|
|
|
|
// Shift the source in to the correct location
|
|
auto Result = JITState.IRBuilder->CreateLShr(Src, JITState.IRBuilder->getIntN(BitWidth, Op->lsb));
|
|
// Mask what we want
|
|
Result = JITState.IRBuilder->CreateAnd(Result, SourceMask);
|
|
SetDest(*WrapperOp, Result);
|
|
}
|
|
else {
|
|
uint64_t SourceMask = (1ULL << Op->Width) - 1;
|
|
if (Op->Width == 64)
|
|
SourceMask = ~0ULL;
|
|
|
|
auto Result = JITState.IRBuilder->CreateLShr(Src, JITState.IRBuilder->getIntN(BitWidth, Op->lsb));
|
|
Result = JITState.IRBuilder->CreateAnd(Result,
|
|
JITState.IRBuilder->getIntN(BitWidth, SourceMask));
|
|
SetDest(*WrapperOp, Result);
|
|
}
|
|
break;
|
|
}
|
|
case IR::OP_BFI: {
|
|
auto Op = IROp->C<IR::IROp_Bfi>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
uint64_t SourceMask = (1ULL << Op->Width) - 1;
|
|
if (Op->Width == 64)
|
|
SourceMask = ~0ULL;
|
|
uint64_t DestMask = ~(SourceMask << Op->lsb);
|
|
|
|
auto BitWidth = Src1->getType()->getIntegerBitWidth();
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
auto MaskedDest = JITState.IRBuilder->CreateAnd(Src1, JITState.IRBuilder->getIntN(BitWidth, DestMask));
|
|
auto MaskedSrc = JITState.IRBuilder->CreateAnd(Src2, JITState.IRBuilder->getIntN(BitWidth, SourceMask));
|
|
MaskedSrc = JITState.IRBuilder->CreateShl(MaskedSrc, JITState.IRBuilder->getIntN(BitWidth, Op->lsb));
|
|
|
|
auto Result = JITState.IRBuilder->CreateOr(MaskedDest, MaskedSrc);
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_LSHR: {
|
|
auto Op = IROp->C<IR::IROp_Lshr>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Our IR assumes defined behaviour for shifting all the bits out of the value
|
|
// So we need to ZEXT to the next size up and then trunc
|
|
auto OriginalType = Src1->getType();
|
|
auto BiggerType = Type::getIntNTy(*Con, 128);
|
|
Src1 = JITState.IRBuilder->CreateZExt(Src1, BiggerType);
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, BiggerType);
|
|
|
|
auto Result = JITState.IRBuilder->CreateLShr(Src1, Src2);
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, OriginalType);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_ASHR: {
|
|
auto Op = IROp->C<IR::IROp_Ashr>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Our IR assumes defined behaviour for shifting all the bits out of the value
|
|
// So we need to ZEXT to the next size up and then trunc
|
|
auto OriginalType = Src1->getType();
|
|
auto BiggerType = Type::getIntNTy(*Con, 128);
|
|
Src1 = JITState.IRBuilder->CreateSExt(Src1, BiggerType);
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, BiggerType);
|
|
|
|
auto Result = JITState.IRBuilder->CreateAShr(Src1, Src2);
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, OriginalType);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_LSHL: {
|
|
auto Op = IROp->C<IR::IROp_Lshl>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Our IR assumes defined behaviour for shifting all the bits out of the value
|
|
// So we need to ZEXT to the next size up and then trunc
|
|
auto OriginalType = Src1->getType();
|
|
auto BiggerType = Type::getIntNTy(*Con, 128);
|
|
Src1 = JITState.IRBuilder->CreateZExt(Src1, BiggerType);
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, BiggerType);
|
|
|
|
auto Result = JITState.IRBuilder->CreateShl(Src1, Src2);
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, OriginalType);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_AND: {
|
|
auto Op = IROp->C<IR::IROp_And>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateAnd(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_UMUL:
|
|
case IR::OP_MUL: {
|
|
auto Op = IROp->C<IR::IROp_Mul>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateMul(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_ROL: {
|
|
auto Op = IROp->C<IR::IROp_Rol>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
|
|
auto Result = FSHL(Src1, Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_ROR: {
|
|
auto Op = IROp->C<IR::IROp_Ror>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
auto Result = FSHR(Src1, Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_PRINT: {
|
|
auto Op = IROp->C<IR::IROp_Print>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
if (Src->getType()->getIntegerBitWidth() < 64) {
|
|
Src = JITState.IRBuilder->CreateZExtOrTrunc(Src, Type::getInt64Ty(*Con));
|
|
}
|
|
CreateDebugPrint(Src);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_CYCLECOUNTER: {
|
|
#ifdef DEBUG_CYCLES
|
|
SetDest(*WrapperOp, JITState.IRBuilder->getInt64(0));
|
|
#else
|
|
SetDest(*WrapperOp, CycleCounter());
|
|
#endif
|
|
break;
|
|
}
|
|
|
|
case IR::OP_POPCOUNT: {
|
|
auto Op = IROp->C<IR::IROp_Popcount>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
|
|
SetDest(*WrapperOp, Popcount(Src));
|
|
break;
|
|
}
|
|
case IR::OP_FINDLSB: {
|
|
auto Op = IROp->C<IR::IROp_FindLSB>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
|
|
unsigned SrcBitWidth = Src->getType()->getIntegerBitWidth();
|
|
llvm::Value *Result = CTTZ(Src);
|
|
|
|
// Need to compare source to zero, since we are expecting -1 on zero, llvm CTTZ returns undef on zero
|
|
auto Comp = JITState.IRBuilder->CreateICmpEQ(Src, JITState.IRBuilder->getIntN(SrcBitWidth, 0));
|
|
Result = JITState.IRBuilder->CreateSelect(Comp, JITState.IRBuilder->getIntN(SrcBitWidth, ~0ULL), Result);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_FINDMSB: {
|
|
auto Op = IROp->C<IR::IROp_FindMSB>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
|
|
unsigned SrcBitWidth = Src->getType()->getIntegerBitWidth();
|
|
llvm::Value *Result = CTLZ(Src);
|
|
|
|
Result = JITState.IRBuilder->CreateSub(JITState.IRBuilder->getIntN(SrcBitWidth, SrcBitWidth), Result);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_SEXT: {
|
|
auto Op = IROp->C<IR::IROp_Sext>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
llvm::Type *SourceType = Type::getIntNTy(*Con, Op->SrcSize);
|
|
llvm::Type *TargetType = Type::getIntNTy(*Con, OpSize * 8);
|
|
|
|
auto Result = JITState.IRBuilder->CreateSExtOrTrunc(Src, SourceType);
|
|
Result = JITState.IRBuilder->CreateSExt(Result, TargetType);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_ZEXT: {
|
|
auto Op = IROp->C<IR::IROp_Zext>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
llvm::Type *SourceType = Type::getIntNTy(*Con, Op->SrcSize);
|
|
llvm::Type *TargetType = Type::getIntNTy(*Con, OpSize * 8);
|
|
|
|
auto Result = JITState.IRBuilder->CreateZExtOrTrunc(Src, SourceType);
|
|
Result = JITState.IRBuilder->CreateZExt(Result, TargetType);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_OR: {
|
|
auto Op = IROp->C<IR::IROp_Or>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateOr(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_UDIV: {
|
|
auto Op = IROp->C<IR::IROp_UDiv>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
auto Divisor = GetSrc(Op->Header.Args[1]);
|
|
|
|
Divisor = JITState.IRBuilder->CreateZExtOrTrunc(Divisor, Src->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateUDiv(Src, Divisor);
|
|
SetDest(*WrapperOp, Result);
|
|
|
|
break;
|
|
}
|
|
case IR::OP_DIV: {
|
|
auto Op = IROp->C<IR::IROp_UDiv>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
auto Divisor = GetSrc(Op->Header.Args[1]);
|
|
|
|
Divisor = JITState.IRBuilder->CreateZExtOrTrunc(Divisor, Src->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateSDiv(Src, Divisor);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_UREM: {
|
|
auto Op = IROp->C<IR::IROp_URem>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
auto Divisor = GetSrc(Op->Header.Args[1]);
|
|
|
|
Divisor = JITState.IRBuilder->CreateZExtOrTrunc(Divisor, Src->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateURem(Src, Divisor);
|
|
SetDest(*WrapperOp, Result);
|
|
|
|
break;
|
|
}
|
|
case IR::OP_REM: {
|
|
auto Op = IROp->C<IR::IROp_Rem>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
auto Divisor = GetSrc(Op->Header.Args[1]);
|
|
|
|
Divisor = JITState.IRBuilder->CreateZExtOrTrunc(Divisor, Src->getType());
|
|
|
|
auto Result = JITState.IRBuilder->CreateSRem(Src, Divisor);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_LUDIV: {
|
|
auto Op = IROp->C<IR::IROp_LUDiv>();
|
|
// Each source is OpSize in size
|
|
// So you can have up to a 128bit divide from x86-64
|
|
auto SrcLow = GetSrc(Op->Header.Args[0]);
|
|
auto SrcHigh = GetSrc(Op->Header.Args[1]);
|
|
auto Divisor = GetSrc(Op->Header.Args[2]);
|
|
|
|
Type *iNormal = Type::getIntNTy(*Con, OpSize * 8);
|
|
Type *iLarge = Type::getIntNTy(*Con, OpSize * 8 * 2);
|
|
|
|
// Zero extend all values to large size
|
|
SrcLow = JITState.IRBuilder->CreateZExt(SrcLow, iLarge);
|
|
SrcHigh = JITState.IRBuilder->CreateZExt(SrcHigh, iLarge);
|
|
Divisor = JITState.IRBuilder->CreateZExt(Divisor, iLarge);
|
|
// Combine the split values
|
|
SrcHigh = JITState.IRBuilder->CreateShl(SrcHigh, JITState.IRBuilder->getIntN(OpSize * 8 * 2, OpSize * 8));
|
|
auto Dividend = JITState.IRBuilder->CreateOr(SrcHigh, SrcLow);
|
|
|
|
// Now do the divide
|
|
auto Result = JITState.IRBuilder->CreateUDiv(Dividend, Divisor);
|
|
|
|
// Now truncate back down origina size and store
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, iNormal);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_LDIV: {
|
|
auto Op = IROp->C<IR::IROp_LDiv>();
|
|
// Each source is OpSize in size
|
|
// So you can have up to a 128bit divide from x86-64
|
|
auto SrcLow = GetSrc(Op->Header.Args[0]);
|
|
auto SrcHigh = GetSrc(Op->Header.Args[1]);
|
|
auto Divisor = GetSrc(Op->Header.Args[2]);
|
|
|
|
Type *iNormal = Type::getIntNTy(*Con, OpSize * 8);
|
|
Type *iLarge = Type::getIntNTy(*Con, OpSize * 8 * 2);
|
|
|
|
// Zero extend all values to large size
|
|
SrcLow = JITState.IRBuilder->CreateZExt(SrcLow, iLarge);
|
|
SrcHigh = JITState.IRBuilder->CreateZExt(SrcHigh, iLarge);
|
|
Divisor = JITState.IRBuilder->CreateSExt(Divisor, iLarge);
|
|
// Combine the split values
|
|
SrcHigh = JITState.IRBuilder->CreateShl(SrcHigh, JITState.IRBuilder->getIntN(OpSize * 8 * 2, OpSize * 8));
|
|
auto Dividend = JITState.IRBuilder->CreateOr(SrcHigh, SrcLow);
|
|
|
|
// Now do the divide
|
|
auto Result = JITState.IRBuilder->CreateSDiv(Dividend, Divisor);
|
|
|
|
// Now truncate back down origina size and store
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, iNormal);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_LUREM: {
|
|
auto Op = IROp->C<IR::IROp_LURem>();
|
|
// Each source is OpOpSize in size
|
|
// So you can have up to a 128bit divide from x86-64
|
|
auto SrcLow = GetSrc(Op->Header.Args[0]);
|
|
auto SrcHigh = GetSrc(Op->Header.Args[1]);
|
|
auto Divisor = GetSrc(Op->Header.Args[2]);
|
|
|
|
Type *iNormal = Type::getIntNTy(*Con, OpSize * 8);
|
|
Type *iLarge = Type::getIntNTy(*Con, OpSize * 8 * 2);
|
|
|
|
// Zero extend all values to large size
|
|
SrcLow = JITState.IRBuilder->CreateZExt(SrcLow, iLarge);
|
|
SrcHigh = JITState.IRBuilder->CreateZExt(SrcHigh, iLarge);
|
|
Divisor = JITState.IRBuilder->CreateZExt(Divisor, iLarge);
|
|
// Combine the split values
|
|
SrcHigh = JITState.IRBuilder->CreateShl(SrcHigh, JITState.IRBuilder->getIntN(OpSize * 8 * 2, OpSize * 8));
|
|
auto Dividend = JITState.IRBuilder->CreateOr(SrcHigh, SrcLow);
|
|
|
|
// Now do the remainder
|
|
auto Result = JITState.IRBuilder->CreateURem(Dividend, Divisor);
|
|
|
|
// Now truncate back down origina size and store
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, iNormal);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_LREM: {
|
|
auto Op = IROp->C<IR::IROp_LRem>();
|
|
// Each source is OpOpSize in size
|
|
// So you can have up to a 128bit divide from x86-64
|
|
auto SrcLow = GetSrc(Op->Header.Args[0]);
|
|
auto SrcHigh = GetSrc(Op->Header.Args[1]);
|
|
auto Divisor = GetSrc(Op->Header.Args[2]);
|
|
|
|
Type *iNormal = Type::getIntNTy(*Con, OpSize * 8);
|
|
Type *iLarge = Type::getIntNTy(*Con, OpSize * 8 * 2);
|
|
|
|
// Zero extend all values to large size
|
|
SrcLow = JITState.IRBuilder->CreateZExt(SrcLow, iLarge);
|
|
SrcHigh = JITState.IRBuilder->CreateZExt(SrcHigh, iLarge);
|
|
Divisor = JITState.IRBuilder->CreateSExt(Divisor, iLarge);
|
|
// Combine the split values
|
|
SrcHigh = JITState.IRBuilder->CreateShl(SrcHigh, JITState.IRBuilder->getIntN(OpSize * 8 * 2, OpSize * 8));
|
|
auto Dividend = JITState.IRBuilder->CreateOr(SrcHigh, SrcLow);
|
|
|
|
// Now do the remainder
|
|
auto Result = JITState.IRBuilder->CreateSRem(Dividend, Divisor);
|
|
|
|
// Now truncate back down origina size and store
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, iNormal);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_UMULH: {
|
|
auto Op = IROp->C<IR::IROp_UMulH>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Type *iNormal = Type::getIntNTy(*Con, OpSize * 8);
|
|
Type *iLarge = Type::getIntNTy(*Con, OpSize * 8 * 2);
|
|
|
|
// Zero extend all values to larger value
|
|
Src1 = JITState.IRBuilder->CreateZExt(Src1, iLarge);
|
|
Src2 = JITState.IRBuilder->CreateZExt(Src2, iLarge);
|
|
|
|
// Do the large multiply
|
|
auto Result = JITState.IRBuilder->CreateMul(Src1, Src2);
|
|
Result = JITState.IRBuilder->CreateLShr(Result, JITState.IRBuilder->getIntN(OpSize * 8 * 2, OpSize * 8));
|
|
// Now truncate back down to origianl size and store
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, iNormal);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_MULH: {
|
|
auto Op = IROp->C<IR::IROp_MulH>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
Type *iNormal = Type::getIntNTy(*Con, OpSize * 8);
|
|
Type *iLarge = Type::getIntNTy(*Con, OpSize * 8 * 2);
|
|
|
|
// Sign extend all values to larger value
|
|
Src1 = JITState.IRBuilder->CreateSExt(Src1, iLarge);
|
|
Src2 = JITState.IRBuilder->CreateSExt(Src2, iLarge);
|
|
|
|
// Do the large multiply
|
|
auto Result = JITState.IRBuilder->CreateMul(Src1, Src2);
|
|
Result = JITState.IRBuilder->CreateLShr(Result, JITState.IRBuilder->getIntN(OpSize * 8 * 2, OpSize * 8));
|
|
|
|
// Now truncate back down to origianl size and store
|
|
Result = JITState.IRBuilder->CreateTrunc(Result, iNormal);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_REV: {
|
|
auto Op = IROp->C<IR::IROp_Rev>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
SetDest(*WrapperOp, BSwap(Src));
|
|
break;
|
|
}
|
|
case IR::OP_CREATEVECTOR2: {
|
|
auto Op = IROp->C<IR::IROp_CreateVector2>();
|
|
LogMan::Throw::A(OpSize <= 16, "Can't handle a vector of size: %d", OpSize);
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Value *Undef = UndefValue::get(VectorType::get(Src1->getType(), 2));
|
|
|
|
// Src1 = CastToOpaqueStructure(Src1, ElementType);
|
|
// Src2 = CastToOpaqueStructure(Src2, ElementType);
|
|
|
|
Undef = JITState.IRBuilder->CreateInsertElement(Undef, Src1, JITState.IRBuilder->getInt32(0));
|
|
Undef = JITState.IRBuilder->CreateInsertElement(Undef, Src2, JITState.IRBuilder->getInt32(1));
|
|
SetDest(*WrapperOp, Undef);
|
|
break;
|
|
}
|
|
case IR::OP_SPLATVECTOR2: {
|
|
auto Op = IROp->C<IR::IROp_SplatVector2>();
|
|
LogMan::Throw::A(OpSize <= 16, "Can't handle a vector of size: %d", OpSize);
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
|
|
auto Result = JITState.IRBuilder->CreateVectorSplat(2, Src);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_SPLATVECTOR3: {
|
|
auto Op = IROp->C<IR::IROp_SplatVector3>();
|
|
LogMan::Throw::A(OpSize <= 16, "Can't handle a vector of size: %d", OpSize);
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
|
|
auto Result = JITState.IRBuilder->CreateVectorSplat(3, Src);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_SPLATVECTOR4: {
|
|
auto Op = IROp->C<IR::IROp_SplatVector4>();
|
|
LogMan::Throw::A(OpSize <= 16, "Can't handle a vector of size: %d", OpSize);
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
|
|
auto Result = JITState.IRBuilder->CreateVectorSplat(4, Src);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_VOR: {
|
|
auto Op = IROp->C<IR::IROp_Or>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
auto Result = JITState.IRBuilder->CreateOr(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_VXOR: {
|
|
auto Op = IROp->C<IR::IROp_Or>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
auto Result = JITState.IRBuilder->CreateXor(Src1, Src2);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_VADD: {
|
|
auto Op = IROp->C<IR::IROp_VAdd>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
auto Result = JITState.IRBuilder->CreateAdd(Src1, Src2);
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_VSUB: {
|
|
auto Op = IROp->C<IR::IROp_VSub>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
auto Result = JITState.IRBuilder->CreateSub(Src1, Src2);
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_VUSHL: {
|
|
auto Op = IROp->C<IR::IROp_VUShl>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
// Now we will do a lshr <NumElements x i1> -> <NumElements x ElementSize>
|
|
auto Result = JITState.IRBuilder->CreateShl(Src1, Src2);
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_VUSHLS: {
|
|
auto Op = IROp->C<IR::IROp_VUShlS>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, Src1->getType()->getScalarType());
|
|
Src2 = JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, Src2);
|
|
|
|
// Now we will do a lshr <NumElements x i1> -> <NumElements x ElementSize>
|
|
auto Result = JITState.IRBuilder->CreateShl(Src1, Src2);
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_VUSHR: {
|
|
auto Op = IROp->C<IR::IROp_VUShr>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, Src2);
|
|
|
|
// Now we will do a lshr <NumElements x i1> -> <NumElements x ElementSize>
|
|
auto Result = JITState.IRBuilder->CreateLShr(Src1, Src2);
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_VCMPEQ: {
|
|
auto Op = IROp->C<IR::IROp_VCMPEQ>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
// Do an icmpeq, this will return a vector of <NumElements x i1>
|
|
auto Result = JITState.IRBuilder->CreateICmpEQ(Src1, Src2);
|
|
|
|
// Now we will do a sext <NumElements x i1> -> <NumElements x ElementSize>
|
|
Result = JITState.IRBuilder->CreateSExt(Result, Src1->getType());
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_VCMPGT: {
|
|
auto Op = IROp->C<IR::IROp_VCMPGT>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
// Do an icmpeq, this will return a vector of <NumElements x i1>
|
|
auto Result = JITState.IRBuilder->CreateICmpSGT(Src1, Src2);
|
|
|
|
// Now we will do a sext <NumElements x i1> -> <NumElements x ElementSize>
|
|
Result = JITState.IRBuilder->CreateSExt(Result, Src1->getType());
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_VZIP2:
|
|
case IR::OP_VZIP: {
|
|
auto Op = IROp->C<IR::IROp_VZip>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
unsigned NumElements = Op->RegisterSize / Op->ElementSize;
|
|
unsigned BaseElement = IROp->Op == IR::OP_VZIP2 ? NumElements / 2 : 0;
|
|
std::vector<uint32_t> VectorMask;
|
|
for (unsigned i = 0; i < NumElements; ++i) {
|
|
unsigned shfl = i % 2 ? (NumElements + (i >> 1)) : (i >> 1);
|
|
shfl += BaseElement;
|
|
VectorMask.emplace_back(shfl);
|
|
}
|
|
|
|
auto VectorMaskConstant = ConstantDataVector::get(*Con, VectorMask);
|
|
|
|
auto Result = JITState.IRBuilder->CreateShuffleVector(Src1, Src2, VectorMaskConstant);
|
|
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_VINSELEMENT: {
|
|
auto Op = IROp->C<IR::IROp_VInsElement>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
|
|
// Cast to the type we want
|
|
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
|
|
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
|
|
|
|
// Extract our source index
|
|
auto Source = JITState.IRBuilder->CreateExtractElement(Src2, JITState.IRBuilder->getInt32(Op->SrcIdx));
|
|
auto Result = JITState.IRBuilder->CreateInsertElement(Src1, Source, JITState.IRBuilder->getInt32(Op->DestIdx));
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_CAS: {
|
|
auto Op = IROp->C<IR::IROp_CAS>();
|
|
auto Src1 = GetSrc(Op->Header.Args[0]);
|
|
auto Src2 = GetSrc(Op->Header.Args[1]);
|
|
auto MemSrc = GetSrc(Op->Header.Args[2]);
|
|
|
|
MemSrc = JITState.IRBuilder->CreateAdd(MemSrc, JITState.IRBuilder->getInt64(CTX->MemoryMapper.GetBaseOffset<uint64_t>(0)));
|
|
// Cast the pointer type correctly
|
|
MemSrc = JITState.IRBuilder->CreateIntToPtr(MemSrc, Type::getIntNTy(*Con, OpSize * 8)->getPointerTo());
|
|
|
|
Src1 = JITState.IRBuilder->CreateZExtOrTrunc(Src1, MemSrc->getType()->getPointerElementType());
|
|
Src2 = JITState.IRBuilder->CreateZExtOrTrunc(Src2, MemSrc->getType()->getPointerElementType());
|
|
|
|
llvm::Value *Result = JITState.IRBuilder->CreateAtomicCmpXchg(MemSrc, Src1, Src2, llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering::SequentiallyConsistent);
|
|
|
|
// Result is a { <Type>, i1 } So we need to extract it first
|
|
// Behaves exactly like std::atomic::compare_exchange_strong(Desired (Src1), Src2) ? Src1 : Desired
|
|
Result = JITState.IRBuilder->CreateExtractValue(Result, {0});
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
|
|
case IR::OP_LOADMEM: {
|
|
auto Op = IROp->C<IR::IROp_LoadMem>();
|
|
auto Src = GetSrc(Op->Header.Args[0]);
|
|
|
|
Src = JITState.IRBuilder->CreateAdd(Src, JITState.IRBuilder->getInt64(CTX->MemoryMapper.GetBaseOffset<uint64_t>(0)));
|
|
// Cast the pointer type correctly
|
|
Src = JITState.IRBuilder->CreateIntToPtr(Src, Type::getIntNTy(*Con, Op->Size * 8)->getPointerTo());
|
|
auto Result = CreateMemoryLoad(Src);
|
|
SetDest(*WrapperOp, Result);
|
|
break;
|
|
}
|
|
case IR::OP_STOREMEM: {
|
|
auto Op = IROp->C<IR::IROp_StoreMem>();
|
|
|
|
auto Dst = GetSrc(Op->Header.Args[0]);
|
|
auto Src = GetSrc(Op->Header.Args[1]);
|
|
|
|
Dst = JITState.IRBuilder->CreateAdd(Dst, JITState.IRBuilder->getInt64(CTX->MemoryMapper.GetBaseOffset<uint64_t>(0)));
|
|
auto Type = Type::getIntNTy(*Con, Op->Size * 8);
|
|
Src = JITState.IRBuilder->CreateZExtOrTrunc(Src, Type);
|
|
Dst = JITState.IRBuilder->CreateIntToPtr(Dst, Type->getPointerTo());
|
|
CreateMemoryStore(Dst, Src);
|
|
break;
|
|
}
|
|
default:
|
|
LogMan::Msg::A("Unknown IR Op: %d(%s)", IROp->Op, FEXCore::IR::GetName(IROp->Op).data());
|
|
break;
|
|
}
|
|
}
|
|
|
|
void* FEXCore::CPU::LLVMJITCore::CompileCode(FEXCore::IR::IRListView<true> const *IR, FEXCore::Core::DebugData *DebugData) {
|
|
using namespace llvm;
|
|
|
|
JumpTargets.clear();
|
|
ForwardJumpTargets.clear();
|
|
JITCurrentState.Blocks.clear();
|
|
|
|
CurrentIR = IR;
|
|
|
|
#if DESTMAP_AS_MAP
|
|
DestMap.clear();
|
|
#else
|
|
uintptr_t ListSize = CurrentIR->GetListSize();
|
|
if (ListSize > DestMap.size()) {
|
|
DestMap.resize(std::max(DestMap.size() * 2, ListSize));
|
|
}
|
|
#endif
|
|
|
|
std::ostringstream FunctionName;
|
|
FunctionName << "Function_0x";
|
|
FunctionName << std::hex << ThreadState->State.State.rip;
|
|
|
|
auto FunctionModule = new llvm::Module("Module", *Con);
|
|
auto EngineBuilder = llvm::EngineBuilder(std::unique_ptr<llvm::Module>(FunctionModule));
|
|
EngineBuilder.setEngineKind(llvm::EngineKind::JIT);
|
|
EngineBuilder.setMCJITMemoryManager(std::unique_ptr<llvm::RTDyldMemoryManager>(JITState.MemManager));
|
|
|
|
auto Engine = EngineBuilder.create(LLVMTarget);
|
|
|
|
Type *i64 = Type::getInt64Ty(*Con);
|
|
auto FunctionType = FunctionType::get(Type::getVoidTy(*Con),
|
|
{
|
|
i64,
|
|
}, false);
|
|
Func = Function::Create(FunctionType,
|
|
Function::ExternalLinkage,
|
|
FunctionName.str(),
|
|
FunctionModule);
|
|
Func->setCallingConv(CallingConv::C);
|
|
|
|
{
|
|
auto Entry = BasicBlock::Create(*Con, "Entry", Func);
|
|
JITCurrentState.Blocks.emplace_back(Entry);
|
|
JITState.IRBuilder->SetInsertPoint(Entry);
|
|
JITCurrentState.CurrentBlock = Entry;
|
|
|
|
CreateGlobalVariables(Engine, FunctionModule);
|
|
|
|
auto Builder = JITState.IRBuilder;
|
|
|
|
// Let's create the exit block quick
|
|
JITCurrentState.ExitBlock = BasicBlock::Create(*Con, "ExitBlock", Func);
|
|
JITCurrentState.Blocks.emplace_back(JITCurrentState.ExitBlock);
|
|
|
|
JITState.IRBuilder->SetInsertPoint(JITCurrentState.ExitBlock);
|
|
Builder->CreateRetVoid();
|
|
|
|
JITState.IRBuilder->SetInsertPoint(Entry);
|
|
JITCurrentState.CurrentBlock = Entry;
|
|
JITCurrentState.Blocks.emplace_back(JITCurrentState.CurrentBlock);
|
|
JITCurrentState.CurrentBlockHasTerm = false;
|
|
|
|
uintptr_t ListBegin = CurrentIR->GetListData();
|
|
uintptr_t DataBegin = CurrentIR->GetData();
|
|
|
|
auto HeaderIterator = CurrentIR->begin();
|
|
IR::OrderedNodeWrapper *HeaderNodeWrapper = HeaderIterator();
|
|
IR::OrderedNode *HeaderNode = HeaderNodeWrapper->GetNode(ListBegin);
|
|
auto HeaderOp = HeaderNode->Op(DataBegin)->CW<FEXCore::IR::IROp_IRHeader>();
|
|
LogMan::Throw::A(HeaderOp->Header.Op == IR::OP_IRHEADER, "First op wasn't IRHeader");
|
|
|
|
IR::OrderedNode *BlockNode = HeaderOp->Blocks.GetNode(ListBegin);
|
|
|
|
while (1) {
|
|
using namespace FEXCore::IR;
|
|
auto BlockIROp = BlockNode->Op(DataBegin)->CW<FEXCore::IR::IROp_CodeBlock>();
|
|
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
|
|
|
|
// We grab these nodes this way so we can iterate easily
|
|
auto CodeBegin = CurrentIR->at(BlockIROp->Begin);
|
|
auto CodeLast = CurrentIR->at(BlockIROp->Last);
|
|
|
|
while (1) {
|
|
HandleIR(CurrentIR, &CodeBegin);
|
|
|
|
// CodeLast is inclusive. So we still need to dump the CodeLast op as well
|
|
if (CodeBegin == CodeLast) {
|
|
break;
|
|
}
|
|
++CodeBegin;
|
|
|
|
}
|
|
|
|
if (BlockIROp->Next.ID() == 0) {
|
|
break;
|
|
} else {
|
|
BlockNode = BlockIROp->Next.GetNode(ListBegin);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (auto &Block : JITCurrentState.Blocks) {
|
|
// If the block is empty then let is just jump to the exit block
|
|
if (Block->empty()) {
|
|
JITState.IRBuilder->SetInsertPoint(Block);
|
|
JITState.IRBuilder->CreateBr(JITCurrentState.ExitBlock);
|
|
}
|
|
}
|
|
|
|
llvm::ModulePassManager FPM;
|
|
llvm::ModuleAnalysisManager FAM;
|
|
llvm::PassBuilder passBuilder(LLVMTarget);
|
|
|
|
passBuilder.registerModuleAnalyses(FAM);
|
|
passBuilder.buildModuleSimplificationPipeline(
|
|
llvm::PassBuilder::OptimizationLevel::O3,
|
|
llvm::PassBuilder::ThinLTOPhase::None);
|
|
|
|
raw_ostream &Out = outs();
|
|
|
|
// if (CTX->Config.LLVM_PrinterPass)
|
|
{
|
|
FPM.addPass(PrintModulePass(Out));
|
|
}
|
|
|
|
if (CTX->Config.LLVM_IRValidation)
|
|
{
|
|
verifyModule(*FunctionModule, &Out);
|
|
}
|
|
|
|
FPM.run(*FunctionModule, FAM);
|
|
Engine->finalizeObject();
|
|
|
|
JITState.Functions.emplace_back(Engine);
|
|
|
|
DebugData->HostCodeSize = JITState.MemManager->GetLastCodeAllocation();
|
|
void *FunctionPtr = reinterpret_cast<void*>(Engine->getFunctionAddress(FunctionName.str()));
|
|
|
|
return FunctionPtr;
|
|
}
|
|
|
|
FEXCore::CPU::CPUBackend *CreateLLVMCore(FEXCore::Core::InternalThreadState *Thread) {
|
|
return new LLVMJITCore(Thread);
|
|
}
|
|
|
|
}
|