steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 14:00:16 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity

Arm64: Convert ARM Emitter over to new emitter

Browse Source 
Not yet complete. Missing a full SVE implementation and needs
testing/validation.
This commit is contained in:
Ryan Houdek committed 2023-01-04 05:30:01 -08:00
1 parent 7a6ef8821f
commit ffd9bb547d
18 files changed
+3208 -6296
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.cpp
Vendored
+3 -4
View File
@@ -1,7 +1,6 @@
#include "Interface/Core/ArchHelpers/Arm64.h"
#include "Interface/Core/ArchHelpers/MContext.h"
#include <aarch64/cpu-aarch64.h>
#include "Interface/Core/ArchHelpers/CodeEmitter/Buffer.h"
#include <FEXCore/Utils/EnumUtils.h>
#include <FEXCore/Utils/LogManager.h>
@@ -572,7 +571,7 @@ bool HandleAtomicVectorStore(void *_ucontext, void *_info, uint32_t Instr) {
PC[1] = STP;
PC[2] = DMB;
// Back up one instruction and have another go
vixl::aarch64::CPU::EnsureIAndDCacheCoherency(&PC[0], 16);
FEXCore::ARMEmitter::Buffer::ClearICache(&PC[0], 16);
return true;
}
}
@@ -2311,7 +2310,7 @@ bool HandleSIGBUS(bool ParanoidTSO, int Signal, void *info, void *ucontext) {
return false;
}
vixl::aarch64::CPU::EnsureIAndDCacheCoherency(&PC[-1], 16);
FEXCore::ARMEmitter::Buffer::ClearICache(&PC[-1], 16);
return true;
}
return false;
External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp
Vendored
+179 -186
View File
@@ -1,4 +1,5 @@
#include "Interface/Core/ArchHelpers/Arm64Emitter.h"
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include "Interface/Context/Context.h"
#include "Interface/HLE/Thunks/Thunks.h"
@@ -20,38 +21,24 @@ namespace FEXCore::CPU {
// We want vixl to not allocate a default buffer. Jit and dispatcher will manually create one.
Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size)
: vixl::aarch64::Assembler(size ? (byte*)FEXCore::Allocator::mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) : reinterpret_cast<byte*>(~0ULL),
size,
vixl::aarch64::PositionDependentCode)
: Emitter(size ? (uint8_t*)FEXCore::Allocator::mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) : nullptr, size)
, EmitterCTX {ctx} {
CPU.SetUp();
#ifdef VIXL_SIMULATOR
auto Features = vixl::CPUFeatures::All();
#else
auto Features = vixl::CPUFeatures::InferFromOS();
if (ctx->HostFeatures.SupportsAtomics) {
// Hypervisor can hide this on the c630?
Features.Combine(vixl::CPUFeatures::Feature::kLORegions);
}
#endif
SetCPUFeatures(Features);
}
Arm64Emitter::~Arm64Emitter() {
auto CodeBuffer = GetBuffer();
if (CodeBuffer->GetCapacity()) {
FEXCore::Allocator::munmap(CodeBuffer->GetStartAddress<void*>(), CodeBuffer->GetCapacity());
auto BufferSize = GetBufferSize();
if (BufferSize) {
FEXCore::Allocator::munmap(GetBufferBase(), BufferSize);
}
}
void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, bool NOPPad) {
bool Is64Bit = Reg.IsX();
void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, bool NOPPad) {
bool Is64Bit = s == ARMEmitter::Size::i64Bit;
int Segments = Is64Bit ? 4 : 2;
if (Is64Bit && ((~Constant)>> 16) == 0) {
movn(Reg, (~Constant) & 0xFFFF);
movn(s, Reg, (~Constant) & 0xFFFF);
if (NOPPad) {
nop(); nop(); nop();
@@ -98,17 +85,17 @@ void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant,
else {
// Need to use ADRP + ADD
adrp(Reg, AlignedOffset >> 12);
add(Reg, Reg, Constant & 0xFFF);
add(s, Reg, Reg, Constant & 0xFFF);
NumMoves = 2;
}
}
}
else {
movz(Reg, (Constant) & 0xFFFF, 0);
movz(s, Reg, (Constant) & 0xFFFF, 0);
for (int i = 1; i < Segments; ++i) {
uint16_t Part = (Constant >> (i * 16)) & 0xFFFF;
if (Part) {
movk(Reg, Part, i * 16);
movk(s, Reg, Part, i * 16);
++NumMoves;
}
}
@@ -124,140 +111,143 @@ void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant,
void Arm64Emitter::PushCalleeSavedRegisters() {
// We need to save pairs of registers
// We save r19-r30
MemOperand PairOffset(sp, -16, PreIndex);
const std::array<std::pair<vixl::aarch64::Register, vixl::aarch64::Register>, 6> CalleeSaved = {{
{x19, x20},
{x21, x22},
{x23, x24},
{x25, x26},
{x27, x28},
{x29, x30},
const std::array<std::pair<ARMEmitter::XRegister, ARMEmitter::XRegister>, 6> CalleeSaved = {{
{ARMEmitter::XReg::x19, ARMEmitter::XReg::x20},
{ARMEmitter::XReg::x21, ARMEmitter::XReg::x22},
{ARMEmitter::XReg::x23, ARMEmitter::XReg::x24},
{ARMEmitter::XReg::x25, ARMEmitter::XReg::x26},
{ARMEmitter::XReg::x27, ARMEmitter::XReg::x28},
{ARMEmitter::XReg::x29, ARMEmitter::XReg::x30},
}};
for (auto &RegPair : CalleeSaved) {
stp(RegPair.first, RegPair.second, PairOffset);
stp<ARMEmitter::IndexType::PRE>(RegPair.first, RegPair.second, ARMEmitter::Reg::rsp, -16);
}
// Additionally we need to store the lower 64bits of v8-v15
// Here's a fun thing, we can use two ST4 instructions to store everything
// We just need a single sub to sp before that
const std::array<
std::tuple<vixl::aarch64::VRegister,
vixl::aarch64::VRegister,
vixl::aarch64::VRegister,
vixl::aarch64::VRegister>, 2> FPRs = {{
{v8, v9, v10, v11},
{v12, v13, v14, v15},
std::tuple<ARMEmitter::DRegister,
ARMEmitter::DRegister,
ARMEmitter::DRegister,
ARMEmitter::DRegister>, 2> FPRs = {{
{ARMEmitter::DReg::d8, ARMEmitter::DReg::d9, ARMEmitter::DReg::d10, ARMEmitter::DReg::d11},
{ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15},
}};
uint32_t VectorSaveSize = sizeof(uint64_t) * 8;
sub(sp, sp, VectorSaveSize);
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, VectorSaveSize);
// SP supporting move
// We just saved x19 so it is safe
add(x19, sp, 0);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r19, ARMEmitter::Reg::rsp, 0);
MemOperand QuadOffset(x19, 32, PostIndex);
for (auto &RegQuad : FPRs) {
st4(std::get<0>(RegQuad).D(),
std::get<1>(RegQuad).D(),
std::get<2>(RegQuad).D(),
std::get<3>(RegQuad).D(),
st4(ARMEmitter::SubRegSize::i64Bit,
std::get<0>(RegQuad),
std::get<1>(RegQuad),
std::get<2>(RegQuad),
std::get<3>(RegQuad),
0,
QuadOffset);
ARMEmitter::Reg::r19,
32);
}
}
void Arm64Emitter::PopCalleeSavedRegisters() {
const std::array<
std::tuple<vixl::aarch64::VRegister,
vixl::aarch64::VRegister,
vixl::aarch64::VRegister,
vixl::aarch64::VRegister>, 2> FPRs = {{
{v12, v13, v14, v15},
{v8, v9, v10, v11},
std::tuple<ARMEmitter::DRegister,
ARMEmitter::DRegister,
ARMEmitter::DRegister,
ARMEmitter::DRegister>, 2> FPRs = {{
{ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15},
{ARMEmitter::DReg::d8, ARMEmitter::DReg::d9, ARMEmitter::DReg::d10, ARMEmitter::DReg::d11},
}};
MemOperand QuadOffset(sp, 32, PostIndex);
for (auto &RegQuad : FPRs) {
ld4(std::get<0>(RegQuad).D(),
std::get<1>(RegQuad).D(),
std::get<2>(RegQuad).D(),
std::get<3>(RegQuad).D(),
ld4(ARMEmitter::SubRegSize::i64Bit,
std::get<0>(RegQuad),
std::get<1>(RegQuad),
std::get<2>(RegQuad),
std::get<3>(RegQuad),
0,
QuadOffset);
ARMEmitter::Reg::rsp,
32);
}
MemOperand PairOffset(sp, 16, PostIndex);
const std::array<std::pair<vixl::aarch64::Register, vixl::aarch64::Register>, 6> CalleeSaved = {{
{x29, x30},
{x27, x28},
{x25, x26},
{x23, x24},
{x21, x22},
{x19, x20},
const std::array<std::pair<ARMEmitter::XRegister, ARMEmitter::XRegister>, 6> CalleeSaved = {{
{ARMEmitter::XReg::x29, ARMEmitter::XReg::x30},
{ARMEmitter::XReg::x27, ARMEmitter::XReg::x28},
{ARMEmitter::XReg::x25, ARMEmitter::XReg::x26},
{ARMEmitter::XReg::x23, ARMEmitter::XReg::x24},
{ARMEmitter::XReg::x21, ARMEmitter::XReg::x22},
{ARMEmitter::XReg::x19, ARMEmitter::XReg::x20},
}};
for (auto &RegPair : CalleeSaved) {
ldp(RegPair.first, RegPair.second, PairOffset);
ldp<ARMEmitter::IndexType::POST>(RegPair.first, RegPair.second, ARMEmitter::Reg::rsp, 16);
}
}
void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t GPRSpillMask, uint32_t FPRSpillMask) {
if (StaticRegisterAllocation()) {
for (size_t i = 0; i < SRA64.size(); i+=2) {
auto Reg1 = SRA64[i];
auto Reg2 = SRA64[i+1];
if (((1U << Reg1.GetCode()) & GPRSpillMask) &&
((1U << Reg2.GetCode()) & GPRSpillMask)) {
stp(Reg1, Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
}
else if (((1U << Reg1.GetCode()) & GPRSpillMask)) {
str(Reg1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
}
else if (((1U << Reg2.GetCode()) & GPRSpillMask)) {
str(Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1])));
}
if (!StaticRegisterAllocation()) {
return;
}
for (size_t i = 0; i < SRA64.size(); i+=2) {
auto Reg1 = SRA64[i];
auto Reg2 = SRA64[i+1];
if (((1U << Reg1.Idx()) & GPRSpillMask) &&
((1U << Reg2.Idx()) & GPRSpillMask)) {
stp<ARMEmitter::IndexType::OFFSET>(Reg1.X(), Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]));
}
else if (((1U << Reg1.Idx()) & GPRSpillMask)) {
str(Reg1.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]));
}
else if (((1U << Reg2.Idx()) & GPRSpillMask)) {
str(Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1]));
}
}
if (FPRs) {
if (EmitterCTX->HostFeatures.SupportsAVX) {
for (size_t i = 0; i < SRAFPR.size(); i++) {
const auto Reg = SRAFPR[i];
if (FPRs) {
if (EmitterCTX->HostFeatures.SupportsAVX) {
for (size_t i = 0; i < SRAFPR.size(); i++) {
const auto Reg = SRAFPR[i];
if (((1U << Reg.GetCode()) & FPRSpillMask) != 0) {
mov(TMP4, offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
st1b(Reg.Z().VnB(), PRED_TMP_32B, SVEMemOperand(STATE, TMP4));
}
if (((1U << Reg.Idx()) & FPRSpillMask) != 0) {
mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
st1b<ARMEmitter::SubRegSize::i8Bit>(Reg, PRED_TMP_32B, STATE.R(), TMP4.R());
}
} else {
if (GPRSpillMask && FPRSpillMask == ~0U) {
// Optimize the common case where we can spill four registers per instruction
auto TmpReg = SRA64[__builtin_ffs(GPRSpillMask)];
// Load the sse offset in to the temporary register
add(TmpReg, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0]));
for (size_t i = 0; i < SRAFPR.size(); i += 4) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
const auto Reg3 = SRAFPR[i + 2];
const auto Reg4 = SRAFPR[i + 3];
st1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(TmpReg, 64, PostIndex));
}
}
else {
for (size_t i = 0; i < SRAFPR.size(); i += 2) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
}
} else {
if (GPRSpillMask && FPRSpillMask == ~0U) {
// Optimize the common case where we can spill four registers per instruction
auto TmpReg = SRA64[__builtin_ffs(GPRSpillMask)];
if (((1U << Reg1.GetCode()) & FPRSpillMask) &&
((1U << Reg2.GetCode()) & FPRSpillMask)) {
stp(Reg1.Q(), Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])));
}
else if (((1U << Reg1.GetCode()) & FPRSpillMask)) {
str(Reg1.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])));
}
else if (((1U << Reg2.GetCode()) & FPRSpillMask)) {
str(Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0])));
}
// Load the sse offset in to the temporary register
add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0]));
for (size_t i = 0; i < SRAFPR.size(); i += 4) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
const auto Reg3 = SRAFPR[i + 2];
const auto Reg4 = SRAFPR[i + 3];
st1<ARMEmitter::SubRegSize::i64Bit>(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64);
}
}
else {
for (size_t i = 0; i < SRAFPR.size(); i += 2) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
if (((1U << Reg1.Idx()) & FPRSpillMask) &&
((1U << Reg2.Idx()) & FPRSpillMask)) {
stp<ARMEmitter::IndexType::OFFSET>(Reg1.Q(), Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]));
}
else if (((1U << Reg1.Idx()) & FPRSpillMask)) {
str(Reg1.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]));
}
else if (((1U << Reg2.Idx()) & FPRSpillMask)) {
str(Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0]));
}
}
}
@@ -266,77 +256,79 @@ void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t GPRSpillMask, uint32_t FP
}
void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRFillMask) {
if (StaticRegisterAllocation()) {
if (FPRs) {
if (EmitterCTX->HostFeatures.SupportsAVX) {
// Set up predicate registers.
// We don't bother spilling these in SpillStaticRegs,
// since all that matters is we restore them on a fill.
// It's not a concern if they get trounced by something else.
ptrue(PRED_TMP_16B.VnB(), SVE_VL16);
ptrue(PRED_TMP_32B.VnB(), SVE_VL32);
if (!StaticRegisterAllocation()) {
return;
}
for (size_t i = 0; i < SRAFPR.size(); i++) {
const auto Reg = SRAFPR[i];
if (FPRs) {
if (EmitterCTX->HostFeatures.SupportsAVX) {
// Set up predicate registers.
// We don't bother spilling these in SpillStaticRegs,
// since all that matters is we restore them on a fill.
// It's not a concern if they get trounced by something else.
ptrue<ARMEmitter::SubRegSize::i8Bit>(PRED_TMP_16B, ARMEmitter::PredicatePattern::SVE_VL16);
ptrue<ARMEmitter::SubRegSize::i8Bit>(PRED_TMP_32B, ARMEmitter::PredicatePattern::SVE_VL32);
if (((1U << Reg.GetCode()) & FPRFillMask) != 0) {
mov(TMP4, offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
ld1b(Reg.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(STATE, TMP4));
}
for (size_t i = 0; i < SRAFPR.size(); i++) {
const auto Reg = SRAFPR[i];
if (((1U << Reg.Idx()) & FPRFillMask) != 0) {
mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
ld1b<ARMEmitter::SubRegSize::i8Bit>(Reg, PRED_TMP_32B, STATE.R(), TMP4.R());
}
} else {
if (GPRFillMask && FPRFillMask == ~0U) {
// Optimize the common case where we can fill four registers per instruction.
// Use one of the filling static registers before we fill it.
auto TmpReg = SRA64[__builtin_ffs(GPRFillMask)];
// Load the sse offset in to the temporary register
add(TmpReg, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0]));
for (size_t i = 0; i < SRAFPR.size(); i += 4) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
const auto Reg3 = SRAFPR[i + 2];
const auto Reg4 = SRAFPR[i + 3];
ld1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(TmpReg, 64, PostIndex));
}
}
else {
for (size_t i = 0; i < SRAFPR.size(); i += 2) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
}
} else {
if (GPRFillMask && FPRFillMask == ~0U) {
// Optimize the common case where we can fill four registers per instruction.
// Use one of the filling static registers before we fill it.
auto TmpReg = SRA64[__builtin_ffs(GPRFillMask)];
if (((1U << Reg1.GetCode()) & FPRFillMask) &&
((1U << Reg2.GetCode()) & FPRFillMask)) {
ldp(Reg1.Q(), Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])));
}
else if (((1U << Reg1.GetCode()) & FPRFillMask)) {
ldr(Reg1.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])));
}
else if (((1U << Reg2.GetCode()) & FPRFillMask)) {
ldr(Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0])));
}
// Load the sse offset in to the temporary register
add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0]));
for (size_t i = 0; i < SRAFPR.size(); i += 4) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
const auto Reg3 = SRAFPR[i + 2];
const auto Reg4 = SRAFPR[i + 3];
ld1<ARMEmitter::SubRegSize::i64Bit>(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64);
}
}
else {
for (size_t i = 0; i < SRAFPR.size(); i += 2) {
const auto Reg1 = SRAFPR[i];
const auto Reg2 = SRAFPR[i + 1];
if (((1U << Reg1.Idx()) & FPRFillMask) &&
((1U << Reg2.Idx()) & FPRFillMask)) {
ldp<ARMEmitter::IndexType::OFFSET>(Reg1.Q(), Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]));
}
else if (((1U << Reg1.Idx()) & FPRFillMask)) {
ldr(Reg1.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]));
}
else if (((1U << Reg2.Idx()) & FPRFillMask)) {
ldr(Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0]));
}
}
}
}
}
for (size_t i = 0; i < SRA64.size(); i+=2) {
auto Reg1 = SRA64[i];
auto Reg2 = SRA64[i+1];
if (((1U << Reg1.GetCode()) & GPRFillMask) &&
((1U << Reg2.GetCode()) & GPRFillMask)) {
ldp(Reg1, Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
}
else if (((1U << Reg1.GetCode()) & GPRFillMask)) {
ldr(Reg1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
}
else if (((1U << Reg2.GetCode()) & GPRFillMask)) {
ldr(Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1])));
}
for (size_t i = 0; i < SRA64.size(); i+=2) {
auto Reg1 = SRA64[i];
auto Reg2 = SRA64[i+1];
if (((1U << Reg1.Idx()) & GPRFillMask) &&
((1U << Reg2.Idx()) & GPRFillMask)) {
ldp<ARMEmitter::IndexType::OFFSET>(Reg1.X(), Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]));
}
else if ((1U << Reg1.Idx()) & GPRFillMask) {
ldr(Reg1.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]));
}
else if ((1U << Reg2.Idx()) & GPRFillMask) {
ldr(Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1]));
}
}
}
void Arm64Emitter::PushDynamicRegsAndLR(aarch64::Register TmpReg) {
void Arm64Emitter::PushDynamicRegsAndLR(FEXCore::ARMEmitter::Register TmpReg) {
const auto CanUseSVE = EmitterCTX->HostFeatures.SupportsAVX;
const auto GPRSize = 1 * Core::CPUState::GPR_REG_SIZE;
const auto FPRRegSize = CanUseSVE ? Core::CPUState::XMM_AVX_REG_SIZE
@@ -344,10 +336,10 @@ void Arm64Emitter::PushDynamicRegsAndLR(aarch64::Register TmpReg) {
const auto FPRSize = RAFPR.size() * FPRRegSize;
const uint64_t SPOffset = AlignUp(GPRSize + FPRSize, 16);
sub(sp, sp, SPOffset);
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);
// rsp capable move
add(TmpReg, aarch64::sp, 0);
add(ARMEmitter::Size::i64Bit, TmpReg, ARMEmitter::Reg::rsp, 0);
if (CanUseSVE) {
for (size_t i = 0; i < RAFPR.size(); i += 4) {
@@ -355,20 +347,21 @@ void Arm64Emitter::PushDynamicRegsAndLR(aarch64::Register TmpReg) {
const auto Reg2 = RAFPR[i + 1];
const auto Reg3 = RAFPR[i + 2];
const auto Reg4 = RAFPR[i + 3];
st4b(Reg1.Z().VnB(), Reg2.Z().VnB(), Reg3.Z().VnB(), Reg4.Z().VnB(), PRED_TMP_32B, SVEMemOperand(TmpReg));
add(TmpReg, TmpReg, 32 * 4);
st4b(Reg1, Reg2, Reg3, Reg4, PRED_TMP_32B, TmpReg, 0);
add(ARMEmitter::Size::i64Bit, TmpReg, TmpReg, 32 * 4);
}
} else {
static_assert(RAFPR.size() % 4 == 0, "Needs to have multiple of 4 FPRs for RA");
for (size_t i = 0; i < RAFPR.size(); i += 4) {
const auto Reg1 = RAFPR[i];
const auto Reg2 = RAFPR[i + 1];
const auto Reg3 = RAFPR[i + 2];
const auto Reg4 = RAFPR[i + 3];
st1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(TmpReg, 64, PostIndex));
st1<ARMEmitter::SubRegSize::i64Bit>(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64);
}
}
str(aarch64::lr, MemOperand(TmpReg, 0));
str(ARMEmitter::XReg::lr, TmpReg, 0);
}
void Arm64Emitter::PopDynamicRegsAndLR() {
@@ -380,8 +373,8 @@ void Arm64Emitter::PopDynamicRegsAndLR() {
const auto Reg2 = RAFPR[i + 1];
const auto Reg3 = RAFPR[i + 2];
const auto Reg4 = RAFPR[i + 3];
ld4b(Reg1.Z().VnB(), Reg2.Z().VnB(), Reg3.Z().VnB(), Reg4.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(aarch64::sp));
add(aarch64::sp, aarch64::sp, 32 * 4);
ld4b(Reg1, Reg2, Reg3, Reg4, PRED_TMP_32B, ARMEmitter::Reg::rsp);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 32 * 4);
}
} else {
for (size_t i = 0; i < RAFPR.size(); i += 4) {
@@ -389,11 +382,11 @@ void Arm64Emitter::PopDynamicRegsAndLR() {
const auto Reg2 = RAFPR[i + 1];
const auto Reg3 = RAFPR[i + 2];
const auto Reg4 = RAFPR[i + 3];
ld1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(aarch64::sp, 64, PostIndex));
ld1<ARMEmitter::SubRegSize::i64Bit>(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), ARMEmitter::Reg::rsp, 64);
}
}
ldr(aarch64::lr, MemOperand(aarch64::sp, 16, PostIndex));
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
}
void Arm64Emitter::Align16B() {
External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h
Vendored
+55 -51
View File
@@ -1,5 +1,9 @@
#pragma once
#include "FEXCore/Utils/EnumUtils.h"
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/ArchHelpers/CodeEmitter/Registers.h"
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include "Interface/Core/ObjectCache/Relocations.h"
@@ -24,72 +28,70 @@
#include <utility>
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
// All but x29 are caller saved
const std::array<aarch64::Register, 16> SRA64 = {
x4, x5, x6, x7, x8, x9, x10, x11,
x12, x18, x17, x16, x15, x14, x13, x29
constexpr std::array<FEXCore::ARMEmitter::Register, 16> SRA64 = {
FEXCore::ARMEmitter::Reg::r4, FEXCore::ARMEmitter::Reg::r5, FEXCore::ARMEmitter::Reg::r6, FEXCore::ARMEmitter::Reg::r7, FEXCore::ARMEmitter::Reg::r8, FEXCore::ARMEmitter::Reg::r9, FEXCore::ARMEmitter::Reg::r10, FEXCore::ARMEmitter::Reg::r11,
FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r18, FEXCore::ARMEmitter::Reg::r17, FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r15, FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r13, FEXCore::ARMEmitter::Reg::r29
};
// All are callee saved
const std::array<aarch64::Register, 9> RA64 = {
x20, x21, x22, x23, x24, x25, x26, x27,
x19
constexpr std::array<FEXCore::ARMEmitter::Register, 9> RA64 = {
FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21, FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23, FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25, FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27,
FEXCore::ARMEmitter::Reg::r19
};
const std::array<std::pair<aarch64::Register, aarch64::Register>, 4> RA64Pair = {{
{x20, x21},
{x22, x23},
{x24, x25},
{x26, x27},
constexpr std::array<std::pair<FEXCore::ARMEmitter::Register, FEXCore::ARMEmitter::Register>, 4> RA64Pair = {{
{FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21},
{FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23},
{FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25},
{FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27},
}};
// All are caller saved
const std::array<aarch64::VRegister, 16> SRAFPR = {
v16, v17, v18, v19, v20, v21, v22, v23,
v24, v25, v26, v27, v28, v29, v30, v31
constexpr std::array<FEXCore::ARMEmitter::VRegister, 16> SRAFPR = {
FEXCore::ARMEmitter::VReg::v16, FEXCore::ARMEmitter::VReg::v17, FEXCore::ARMEmitter::VReg::v18, FEXCore::ARMEmitter::VReg::v19, FEXCore::ARMEmitter::VReg::v20, FEXCore::ARMEmitter::VReg::v21, FEXCore::ARMEmitter::VReg::v22, FEXCore::ARMEmitter::VReg::v23,
FEXCore::ARMEmitter::VReg::v24, FEXCore::ARMEmitter::VReg::v25, FEXCore::ARMEmitter::VReg::v26, FEXCore::ARMEmitter::VReg::v27, FEXCore::ARMEmitter::VReg::v28, FEXCore::ARMEmitter::VReg::v29, FEXCore::ARMEmitter::VReg::v30, FEXCore::ARMEmitter::VReg::v31
};
// v8..v15 = (lower 64bits) Callee saved
const std::array<aarch64::VRegister, 12> RAFPR = {
/*v0, v1, v2, v3,*/v4, v5, v6, v7, // v0 ~ v3 are used as temps
v8, v9, v10, v11, v12, v13, v14, v15
constexpr std::array<FEXCore::ARMEmitter::VRegister, 12> RAFPR = {
/*FEXCore::ARMEmitter::VReg::v0, FEXCore::ARMEmitter::VReg::v1, FEXCore::ARMEmitter::VReg::v2, FEXCore::ARMEmitter::VReg::v3,*/FEXCore::ARMEmitter::VReg::v4, FEXCore::ARMEmitter::VReg::v5, FEXCore::ARMEmitter::VReg::v6, FEXCore::ARMEmitter::VReg::v7, // FEXCore::ARMEmitter::VReg::v0 ~ FEXCore::ARMEmitter::VReg::v3 are used as temps
FEXCore::ARMEmitter::VReg::v8, FEXCore::ARMEmitter::VReg::v9, FEXCore::ARMEmitter::VReg::v10, FEXCore::ARMEmitter::VReg::v11, FEXCore::ARMEmitter::VReg::v12, FEXCore::ARMEmitter::VReg::v13, FEXCore::ARMEmitter::VReg::v14, FEXCore::ARMEmitter::VReg::v15
};
// Contains the address to the currently available CPU state
#define STATE x28
constexpr auto STATE = FEXCore::ARMEmitter::XReg::x28;
// GPR temporaries. Only x3 can be used across spill boundaries
// so if these ever need to change, be very careful about that.
#define TMP1 x0
#define TMP2 x1
#define TMP3 x2
#define TMP4 x3
constexpr auto TMP1 = FEXCore::ARMEmitter::XReg::x0;
constexpr auto TMP2 = FEXCore::ARMEmitter::XReg::x1;
constexpr auto TMP3 = FEXCore::ARMEmitter::XReg::x2;
constexpr auto TMP4 = FEXCore::ARMEmitter::XReg::x3;
// Vector temporaries
#define VTMP1 v0
#define VTMP2 v1
#define VTMP3 v2
#define VTMP4 v3
constexpr auto VTMP1 = FEXCore::ARMEmitter::VReg::v0;
constexpr auto VTMP2 = FEXCore::ARMEmitter::VReg::v1;
constexpr auto VTMP3 = FEXCore::ARMEmitter::VReg::v2;
constexpr auto VTMP4 = FEXCore::ARMEmitter::VReg::v3;
// Predicate register temporaries (used when AVX support is enabled)
// PRED_TMP_16B indicates a predicate register that indicates the first 16 bytes set to 1.
// PRED_TMP_32B indicates a predicate register that indicates the first 32 bytes set to 1.
#define PRED_TMP_16B p6
#define PRED_TMP_32B p7
constexpr FEXCore::ARMEmitter::PRegister PRED_TMP_16B = FEXCore::ARMEmitter::PReg::p6;
constexpr FEXCore::ARMEmitter::PRegister PRED_TMP_32B = FEXCore::ARMEmitter::PReg::p7;
// This class contains common emitter utility functions that can
// be used by both Arm64 JIT and ARM64 Dispatcher
class Arm64Emitter : public vixl::aarch64::Assembler {
class Arm64Emitter : public FEXCore::ARMEmitter::Emitter {
protected:
Arm64Emitter(FEXCore::Context::Context *ctx, size_t size);
~Arm64Emitter();
FEXCore::Context::Context *EmitterCTX;
vixl::aarch64::CPU CPU;
void LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, bool NOPPad = false);
void LoadConstant(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register Reg, uint64_t Constant, bool NOPPad = false);
// NOTE: These functions WILL clobber the register TMP4 if AVX support is enabled
// and FPRs are being spilled or filled. If only GPRs are spilled/filled, then
@@ -103,13 +105,14 @@ protected:
// We can't guarantee only the lower 64bits are used so flush everything
static constexpr uint32_t CALLER_FPR_MASK = ~0U;
void PushDynamicRegsAndLR(aarch64::Register TmpReg);
void PushDynamicRegsAndLR(FEXCore::ARMEmitter::Register TmpReg);
void PopDynamicRegsAndLR();
void PushCalleeSavedRegisters();
void PopCalleeSavedRegisters();
void Align16B();
#ifdef VIXL_SIMULATOR
// Generates a vixl simulator runtime call.
//
@@ -121,57 +124,58 @@ protected:
// 2) Simulator wrapper handler
// 3) Function to call
// 4) Style of the function call (Call versus tail-call)
template<typename R, typename... P>
void GenerateRuntimeCall(R (*Function)(P...)) {
uintptr_t SimulatorWrapperAddress = reinterpret_cast<uintptr_t>(
&(Simulator::RuntimeCallStructHelper<R, P...>::Wrapper));
&(vixl::aarch64::Simulator::RuntimeCallStructHelper<R, P...>::Wrapper));
uintptr_t FunctionAddress = reinterpret_cast<uintptr_t>(Function);
hlt(kRuntimeCallOpcode);
hlt(vixl::aarch64::kRuntimeCallOpcode);
// Simulator wrapper address pointer.
dc(SimulatorWrapperAddress);
dc64(SimulatorWrapperAddress);
// Runtime function address to call
dc(FunctionAddress);
dc64(FunctionAddress);
// Call type
dc32(kCallRuntime);
dc32(vixl::aarch64::kCallRuntime);
}
template<typename R, typename... P>
void GenerateIndirectRuntimeCall(vixl::aarch64::Register Reg) {
void GenerateIndirectRuntimeCall(ARMEmitter::Register Reg) {
uintptr_t SimulatorWrapperAddress = reinterpret_cast<uintptr_t>(
&(Simulator::RuntimeCallStructHelper<R, P...>::Wrapper));
&(vixl::aarch64::Simulator::RuntimeCallStructHelper<R, P...>::Wrapper));
hlt(kIndirectRuntimeCallOpcode);
hlt(vixl::aarch64::kIndirectRuntimeCallOpcode);
// Simulator wrapper address pointer.
dc(SimulatorWrapperAddress);
dc64(SimulatorWrapperAddress);
// Register that contains the function to call
dc(Reg.GetCode());
dc32(Reg.Idx());
// Call type
dc32(kCallRuntime);
dc32(vixl::aarch64::kCallRuntime);
}
template<>
void GenerateIndirectRuntimeCall<float, __uint128_t>(vixl::aarch64::Register Reg) {
void GenerateIndirectRuntimeCall<float, __uint128_t>(ARMEmitter::Register Reg) {
uintptr_t SimulatorWrapperAddress = reinterpret_cast<uintptr_t>(
&(Simulator::RuntimeCallStructHelper<float, __uint128_t>::Wrapper));
&(vixl::aarch64::Simulator::RuntimeCallStructHelper<float, __uint128_t>::Wrapper));
hlt(kIndirectRuntimeCallOpcode);
hlt(vixl::aarch64::kIndirectRuntimeCallOpcode);
// Simulator wrapper address pointer.
dc(SimulatorWrapperAddress);
dc64(SimulatorWrapperAddress);
// Register that contains the function to call
dc(Reg.GetCode());
dc32(Reg.Idx());
// Call type
dc32(kCallRuntime);
dc32(vixl::aarch64::kCallRuntime);
}
#endif
External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.cpp
Vendored
+177 -190
View File
@@ -1,3 +1,4 @@
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/LookupCache.h"
#include "Interface/Core/ArchHelpers/MContext.h"
@@ -30,14 +31,8 @@
#include <sys/syscall.h>
#include <unistd.h>
#define STATE_PTR(STATE_TYPE, FIELD) \
MemOperand(STATE, offsetof(FEXCore::Core::STATE_TYPE, FIELD))
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config)
@@ -46,16 +41,18 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
, Simulator {&Decoder}
#endif
{
#ifdef VIXL_DISASSEMBLER
const auto DisasmBegin = GetCursorAddress<const Instruction*>();
#endif
#ifdef VIXL_SIMULATOR
// Hardcode a 256-bit vector width if we are running in the simulator.
Simulator.SetVectorLengthInBits(256);
#endif
SetAllowAssembler(true);
EmitDispatcher();
}
void Arm64Dispatcher::EmitDispatcher() {
#ifdef VIXL_DISASSEMBLER
const auto DisasmBegin = GetCursorAddress<const vixl::aarch64::Instruction*>();
#endif
DispatchPtr = GetCursorAddress<AsmDispatch>();
@@ -67,9 +64,9 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
// Ptr();
// }
Literal l_CTX {reinterpret_cast<uintptr_t>(CTX)};
Literal l_Sleep {reinterpret_cast<uint64_t>(SleepThread)};
Literal l_CompileBlock {GetCompileBlockPtr()};
ARMEmitter::ForwardLabel l_CTX;
ARMEmitter::ForwardLabel l_Sleep;
ARMEmitter::ForwardLabel l_CompileBlock;
// Push all the register we need to save
PushCalleeSavedRegisters();
@@ -77,12 +74,12 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
// Push our memory base to the correct register
// Move our thread pointer to the correct register
// This is passed in to parameter 0 (x0)
mov(STATE, x0);
mov(STATE, ARMEmitter::XReg::x0);
// Save this stack pointer so we can cleanly shutdown the emulation with a long jump
// regardless of where we were in the stack
add(x0, sp, 0);
str(x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0);
str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));
AbsoluteLoopTopAddressFillSRA = GetCursorAddress<uint64_t>();
@@ -92,91 +89,97 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
// We want to ensure that we are 16 byte aligned at the top of this loop
Align16B();
aarch64::Label FullLookup{};
aarch64::Label CallBlock{};
aarch64::Label LoopTop{};
aarch64::Label ExitSpillSRA{};
aarch64::Label ThreadPauseHandler{};
ARMEmitter::BiDirectionalLabel FullLookup{};
ARMEmitter::BiDirectionalLabel CallBlock{};
ARMEmitter::BackwardLabel LoopTop{};
bind(&LoopTop);
AbsoluteLoopTopAddress = GetLabelAddress<uint64_t>(&LoopTop);
Bind(&LoopTop);
AbsoluteLoopTopAddress = GetCursorAddress<uint64_t>();
// Load in our RIP
// Don't modify x2 since it contains our RIP once the block doesn't exist
ldr(x2, STATE_PTR(CpuStateFrame, State.rip));
auto RipReg = x2;
auto RipReg = ARMEmitter::XReg::x2;
ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
// L1 Cache
ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x3, Shift::LSL, 4));
ldp(x3, x0, MemOperand(x0));
cmp(x0, RipReg);
b(&FullLookup, Condition::ne);
and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg.R(), LookupCache::L1_ENTRIES_MASK);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::r0, ARMEmitter::Reg::r3, ARMEmitter::ShiftType::LSL , 4);
ldp<ARMEmitter::IndexType::OFFSET>(ARMEmitter::XReg::x3, ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, 0);
cmp(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, RipReg.R());
b(ARMEmitter::Condition::CC_NE, &FullLookup);
br(x3);
br(ARMEmitter::Reg::r3);
// L1C check failed, do a full lookup
bind(&FullLookup);
Bind(&FullLookup);
// This is the block cache lookup routine
// It matches what is going on it LookupCache.h::FindBlock
ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer));
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer));
// Mask the address by the virtual address size so we can check for aliases
uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize;
if (std::popcount(VirtualMemorySize) == 1) {
and_(x3, RipReg, VirtualMemorySize - 1);
and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg.R(), VirtualMemorySize - 1);
}
else {
LoadConstant(x3, VirtualMemorySize);
and_(x3, RipReg, x3);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, VirtualMemorySize);
and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg.R(), ARMEmitter::Reg::r3);
}
aarch64::Label NoBlock;
#ifdef VIXL_SIMULATOR
// VIXL simulator can't run syscalls.
constexpr bool SignalSafeCompile = false;
#else
constexpr bool SignalSafeCompile = true;
#endif
ARMEmitter::ForwardLabel NoBlock;
{
// Offset the address and add to our page pointer
lsr(x1, x3, 12);
lsr(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::r3, 12);
// Load the pointer from the offset
ldr(x0, MemOperand(x0, x1, Shift::LSL, 3));
ldr(ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, ARMEmitter::Reg::r1, ARMEmitter::ExtendedType::LSL_64, 3);
// If page pointer is zero then we have no block
cbz(x0, &NoBlock);
cbz(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, &NoBlock);
// Steal the page offset
and_(x1, x3, 0x0FFF);
and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::r3, 0x0FFF);
// Shift the offset by the size of the block cache entry
add(x0, x0, Operand(x1, Shift::LSL, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry))));
add(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::ShiftType::LSL, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry)));
// Load the guest address first to ensure it maps to the address we are currently at
// This fixes aliasing problems
ldr(x1, MemOperand(x0, offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode)));
cmp(x1, RipReg);
b(&NoBlock, Condition::ne);
ldr(ARMEmitter::XReg::x1, ARMEmitter::Reg::r0, offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode));
cmp(ARMEmitter::XReg::x1, RipReg);
b(ARMEmitter::Condition::CC_NE, &NoBlock);
// Now load the actual host block to execute if we can
ldr(x3, MemOperand(x0, offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode)));
cbz(x3, &NoBlock);
ldr(ARMEmitter::XReg::x3, ARMEmitter::Reg::r0, offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode));
cbz(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, &NoBlock);
// If we've made it here then we have a real compiled block
{
// update L1 cache
ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
and_(x1, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x1, Shift::LSL, 4));
stp(x3, x2, MemOperand(x0));
and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, RipReg.R(), LookupCache::L1_ENTRIES_MASK);
add(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::ShiftType::LSL, 4);
stp<ARMEmitter::IndexType::OFFSET>(ARMEmitter::XReg::x3, ARMEmitter::XReg::x2, ARMEmitter::Reg::r0);
// Jump to the block
br(x3);
br(ARMEmitter::Reg::r3);
}
}
{
bind(&ExitSpillSRA);
ThreadStopHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
if (config.StaticRegisterAllocation)
SpillStaticRegs();
@@ -190,12 +193,6 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
ret();
}
#ifdef VIXL_SIMULATOR
// VIXL simulator can't run syscalls.
constexpr bool SignalSafeCompile = false;
#else
constexpr bool SignalSafeCompile = true;
#endif
{
ExitFunctionLinkerAddress = GetCursorAddress<uint64_t>();
if (config.StaticRegisterAllocation)
@@ -210,52 +207,52 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
// X3: Size of mask, sizeof(uint64_t)
// X8: Syscall
LoadConstant(x0, ~0ULL);
stp(x0, x0, MemOperand(sp, -16, PreIndex));
LoadConstant(x0, SIG_SETMASK);
add(x1, sp, 0);
add(x2, sp, 0);
LoadConstant(x3, 8);
LoadConstant(x8, SYS_rt_sigprocmask);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ~0ULL);
stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::Reg::rsp, -16);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::rsp, 0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask);
svc(0);
}
mov(x0, STATE);
mov(x1, lr);
mov(ARMEmitter::XReg::x0, STATE);
mov(ARMEmitter::XReg::x1, ARMEmitter::XReg::lr);
ldr(x2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uintptr_t, void *, void *>(x2);
GenerateIndirectRuntimeCall<uintptr_t, void *, void *>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
if (SignalSafeCompile) {
// Now restore the signal mask
// Living in the same location
mov(x4, x0);
LoadConstant(x0, SIG_SETMASK);
add(x1, sp, 0);
LoadConstant(x2, 0);
LoadConstant(x3, 8);
LoadConstant(x8, SYS_rt_sigprocmask);
mov(ARMEmitter::XReg::x4, ARMEmitter::XReg::x0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, 0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask);
svc(0);
// Bring stack back
add(sp, sp, 16);
mov(x0, x4);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16);
mov(ARMEmitter::XReg::x0, ARMEmitter::XReg::x4);
}
if (config.StaticRegisterAllocation)
FillStaticRegs();
br(x0);
br(ARMEmitter::Reg::r0);
}
// Need to create the block
{
bind(&NoBlock);
Bind(&NoBlock);
if (config.StaticRegisterAllocation)
SpillStaticRegs();
@@ -269,42 +266,42 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
// X3: Size of mask, sizeof(uint64_t)
// X8: Syscall
LoadConstant(x0, ~0ULL);
stp(x0, x2, MemOperand(sp, -16, PreIndex));
LoadConstant(x0, SIG_SETMASK);
add(x1, sp, 0);
add(x2, sp, 0);
LoadConstant(x3, 8);
LoadConstant(x8, SYS_rt_sigprocmask);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ~0ULL);
stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::x0, ARMEmitter::XReg::x2, ARMEmitter::Reg::rsp, -16);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::rsp, 0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask);
svc(0);
// Reload x2 to bring back RIP
ldr(x2, MemOperand(sp, 8, Offset));
ldr(ARMEmitter::XReg::x2, ARMEmitter::Reg::rsp, 8);
}
ldr(x0, &l_CTX);
mov(x1, STATE);
ldr(x3, &l_CompileBlock);
ldr(ARMEmitter::XReg::x0, &l_CTX);
mov(ARMEmitter::XReg::x1, STATE);
ldr(ARMEmitter::XReg::x3, &l_CompileBlock);
// X2 contains our guest RIP
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<void, void *, uint64_t, void *>(x3);
GenerateIndirectRuntimeCall<void, void *, uint64_t, void *>(ARMEmitter::Reg::r3);
#else
blr(x3); // { CTX, Frame, RIP}
blr(ARMEmitter::Reg::r3); // { CTX, Frame, RIP}
#endif
if (SignalSafeCompile) {
// Now restore the signal mask
// Living in the same location
LoadConstant(x0, SIG_SETMASK);
add(x1, sp, 0);
LoadConstant(x2, 0);
LoadConstant(x3, 8);
LoadConstant(x8, SYS_rt_sigprocmask);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, 0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask);
svc(0);
// Bring stack back
add(sp, sp, 16);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16);
}
if (config.StaticRegisterAllocation)
@@ -324,7 +321,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
{
// Guest SIGILL handler
// Needs to be distinct from the SignalHandlerReturnAddress
GuestSignal_SIGILL = GetCursorAddress<uint64_t>();
GuestSignal_SIGILL = GetCursorAddress<uint64_t>();
if (config.StaticRegisterAllocation)
SpillStaticRegs();
@@ -355,8 +352,8 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
// brk = SIGTRAP
// ??? = SIGSEGV
// Force a SIGSEGV by loading zero
LoadConstant(x1, 0);
ldr(x1, MemOperand(x1));
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, 0);
ldr(ARMEmitter::XReg::x1, ARMEmitter::Reg::r1);
}
{
@@ -364,19 +361,18 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
if (config.StaticRegisterAllocation)
SpillStaticRegs();
bind(&ThreadPauseHandler);
ThreadPauseHandlerAddress = GetCursorAddress<uint64_t>();
// We are pausing, this means the frontend should be waiting for this thread to idle
// We will have faulted and jumped to this location at this point
// Call our sleep handler
ldr(x0, &l_CTX);
mov(x1, STATE);
ldr(x2, &l_Sleep);
ldr(ARMEmitter::XReg::x0, &l_CTX);
mov(ARMEmitter::XReg::x1, STATE);
ldr(ARMEmitter::XReg::x2, &l_Sleep);
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<void, void *, void *>(x2);
GenerateIndirectRuntimeCall<void, void *, void *>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PauseReturnInstruction = GetCursorAddress<uint64_t>();
@@ -406,27 +402,27 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
PushCalleeSavedRegisters();
// First thing we need to move the thread state pointer back in to our register
mov(STATE, x0);
mov(STATE, ARMEmitter::XReg::x0);
// Make sure to adjust the refcounter so we don't clear the cache now
ldr(w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));
add(w2, w2, 1);
str(w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));
ldr(ARMEmitter::WReg::w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));
add(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 1);
str(ARMEmitter::WReg::w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));
// Now push the callback return trampoline to the guest stack
// Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host
LoadConstant(x0, CTX->X86CodeGen.CallbackReturn);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, CTX->X86CodeGen.CallbackReturn);
ldr(x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
sub(x2, x2, 16);
str(x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 16);
str(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
// Store the trampoline to the guest stack
// Guest stack is now correctly misaligned after a regular call instruction
str(x0, MemOperand(x2));
str(ARMEmitter::XReg::x0, ARMEmitter::Reg::r2, 0);
// Store RIP to the context state
str(x1, STATE_PTR(CpuStateFrame, State.rip));
str(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, State.rip));
// load static regs
if (config.StaticRegisterAllocation)
@@ -439,14 +435,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
{
LUDIVHandlerAddress = GetCursorAddress<uint64_t>();
PushDynamicRegsAndLR(x3);
PushDynamicRegsAndLR(ARMEmitter::Reg::r3);
SpillStaticRegs();
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV));
ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
#else
blr(x3);
blr(ARMEmitter::Reg::r3);
#endif
FillStaticRegs();
@@ -461,14 +457,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
{
LDIVHandlerAddress = GetCursorAddress<uint64_t>();
PushDynamicRegsAndLR(x3);
PushDynamicRegsAndLR(ARMEmitter::Reg::r3);
SpillStaticRegs();
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV));
ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
#else
blr(x3);
blr(ARMEmitter::Reg::r3);
#endif
FillStaticRegs();
@@ -483,14 +479,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
{
LUREMHandlerAddress = GetCursorAddress<uint64_t>();
PushDynamicRegsAndLR(x3);
PushDynamicRegsAndLR(ARMEmitter::Reg::r3);
SpillStaticRegs();
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM));
ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
#else
blr(x3);
blr(ARMEmitter::Reg::r3);
#endif
FillStaticRegs();
@@ -505,14 +501,15 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
{
LREMHandlerAddress = GetCursorAddress<uint64_t>();
PushDynamicRegsAndLR(x3);
PushDynamicRegsAndLR(ARMEmitter::Reg::r3);
SpillStaticRegs();
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM));
ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
#else
blr(x3);
blr(ARMEmitter::Reg::r3);
#endif
FillStaticRegs();
@@ -524,16 +521,16 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
ret();
}
place(&l_CTX);
place(&l_Sleep);
place(&l_CompileBlock);
Bind(&l_CTX);
dc64(reinterpret_cast<uintptr_t>(CTX));
Bind(&l_Sleep);
dc64(reinterpret_cast<uint64_t>(SleepThread));
Bind(&l_CompileBlock);
dc64(GetCompileBlockPtr());
FinalizeCode();
Start = reinterpret_cast<uint64_t>(DispatchPtr);
End = GetCursorAddress<uint64_t>();
vixl::aarch64::CPU::EnsureIAndDCacheCoherency(reinterpret_cast<void*>(DispatchPtr), End - reinterpret_cast<uint64_t>(DispatchPtr));
GetBuffer()->SetExecutable();
ClearICache(reinterpret_cast<void*>(DispatchPtr), End - reinterpret_cast<uint64_t>(DispatchPtr));
if (CTX->Config.BlockJITNaming()) {
std::string Name = "Dispatch_" + std::to_string(FHU::Syscalls::gettid());
@@ -542,8 +539,9 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
if (CTX->Config.GlobalJITNaming()) {
CTX->Symbols.RegisterJITSpace(reinterpret_cast<void*>(DispatchPtr), End - reinterpret_cast<uint64_t>(DispatchPtr));
}
#ifdef VIXL_DISASSEMBLER
const auto DisasmEnd = GetCursorAddress<const Instruction*>();
const auto DisasmEnd = GetCursorAddress<const vixl::aarch64::Instruction*>();
Disasm.DisassembleBuffer(DisasmBegin, DisasmEnd);
#endif
}
@@ -551,101 +549,90 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
#ifdef VIXL_SIMULATOR
void Arm64Dispatcher::ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) {
Simulator.WriteXRegister(0, reinterpret_cast<int64_t>(Frame));
Simulator.RunFrom(reinterpret_cast<Instruction const*>(DispatchPtr));
Simulator.RunFrom(reinterpret_cast<vixl::aarch64::Instruction const*>(DispatchPtr));
}
void Arm64Dispatcher::ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) {
Simulator.WriteXRegister(0, reinterpret_cast<int64_t>(Frame));
Simulator.WriteXRegister(1, RIP);
Simulator.RunFrom(reinterpret_cast<Instruction const*>(CallbackPtr));
Simulator.RunFrom(reinterpret_cast<vixl::aarch64::Instruction const*>(CallbackPtr));
}
#endif
// Used by GenerateGDBPauseCheck, GenerateInterpreterTrampoline, destination buffer is set before use
static thread_local vixl::aarch64::Assembler emit((uint8_t*)&emit, 1);
size_t Arm64Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) {
FEXCore::ARMEmitter::Emitter emit{CodeBuffer, MaxGDBPauseCheckSize};
*emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxGDBPauseCheckSize);
vixl::CodeBufferCheckScope scope(&emit, MaxGDBPauseCheckSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
aarch64::Label RunBlock;
ARMEmitter::ForwardLabel RunBlock;
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
// This happens when single stepping
static_assert(sizeof(FEXCore::Context::Context::Config.RunningMode) == 4, "This is expected to be size of 4");
emit.ldr(x0, STATE_PTR(CpuStateFrame, Thread)); // Get thread
emit.ldr(x0, MemOperand(x0, offsetof(FEXCore::Core::InternalThreadState, CTX))); // Get Context
emit.ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
emit.ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Thread));
emit.ldr(ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, offsetof(FEXCore::Core::InternalThreadState, CTX)); // Get Context
emit.ldr(ARMEmitter::WReg::w0, ARMEmitter::Reg::r0, offsetof(FEXCore::Context::Context, Config.RunningMode));
// If the value == 0 then we don't need to stop
emit.cbz(w0, &RunBlock);
emit.cbz(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, &RunBlock);
{
Literal l_GuestRIP {GuestRIP};
ARMEmitter::ForwardLabel l_GuestRIP;
// Make sure RIP is syncronized to the context
emit.ldr(x0, &l_GuestRIP);
emit.str(x0, STATE_PTR(CpuStateFrame, State.rip));
emit.ldr(ARMEmitter::XReg::x0, &l_GuestRIP);
emit.str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, State.rip));
// Stop the thread
emit.ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA));
emit.br(x0);
emit.place(&l_GuestRIP);
emit.ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA));
emit.br(ARMEmitter::Reg::r0);
emit.Bind(&l_GuestRIP);
emit.dc64(GuestRIP);
}
emit.bind(&RunBlock);
emit.FinalizeCode();
emit.Bind(&RunBlock);
auto UsedBytes = emit.GetBuffer()->GetCursorOffset();
vixl::aarch64::CPU::EnsureIAndDCacheCoherency(CodeBuffer, UsedBytes);
auto UsedBytes = emit.GetCursorOffset();
emit.ClearICache(CodeBuffer, UsedBytes);
return UsedBytes;
}
size_t Arm64Dispatcher::GenerateInterpreterTrampoline(uint8_t *CodeBuffer) {
LOGMAN_THROW_AA_FMT(!config.StaticRegisterAllocation, "GenerateInterpreterTrampoline dispatcher does not support SRA");
*emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxInterpreterTrampolineSize);
FEXCore::ARMEmitter::Emitter emit{CodeBuffer, MaxInterpreterTrampolineSize};
ARMEmitter::ForwardLabel InlineIRData;
vixl::CodeBufferCheckScope scope(&emit, MaxInterpreterTrampolineSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
emit.mov(ARMEmitter::XReg::x0, STATE);
emit.adr(ARMEmitter::Reg::r1, &InlineIRData);
aarch64::Label InlineIRData;
emit.ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.Interpreter.FragmentExecuter));
emit.blr(ARMEmitter::Reg::r3);
emit.mov(x0, STATE);
emit.adr(x1, &InlineIRData);
emit.ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.DispatcherLoopTop));
emit.br(ARMEmitter::Reg::r0);
emit.ldr(x3, STATE_PTR(CpuStateFrame, Pointers.Interpreter.FragmentExecuter));
emit.blr(x3);
emit.Bind(&InlineIRData);
emit.ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.DispatcherLoopTop));
emit.br(x0);
emit.bind(&InlineIRData);
emit.FinalizeCode();
auto UsedBytes = emit.GetBuffer()->GetCursorOffset();
vixl::aarch64::CPU::EnsureIAndDCacheCoherency(CodeBuffer, UsedBytes);
auto UsedBytes = emit.GetCursorOffset();
emit.ClearICache(CodeBuffer, UsedBytes);
return UsedBytes;
}
void Arm64Dispatcher::SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) {
for (size_t i = 0; i < SRA64.size(); i++) {
if (IgnoreMask & (1U << SRA64[i].GetCode())) {
if (IgnoreMask & (1U << SRA64[i].Idx())) {
// Skip this one, it's already spilled
continue;
}
Thread->CurrentFrame->State.gregs[i] = ArchHelpers::Context::GetArmReg(ucontext, SRA64[i].GetCode());
Thread->CurrentFrame->State.gregs[i] = ArchHelpers::Context::GetArmReg(ucontext, SRA64[i].Idx());
}
if (EmitterCTX->HostFeatures.SupportsAVX) {
for (size_t i = 0; i < SRAFPR.size(); i++) {
auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].GetCode());
auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].Idx());
memcpy(&Thread->CurrentFrame->State.xmm.avx.data[i][0], &FPR, sizeof(__uint128_t));
}
} else {
for (size_t i = 0; i < SRAFPR.size(); i++) {
auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].GetCode());
auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].Idx());
memcpy(&Thread->CurrentFrame->State.xmm.sse.data[i][0], &FPR, sizeof(__uint128_t));
}
}
External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h
Vendored
+5
View File
@@ -15,6 +15,9 @@ namespace FEXCore::Core {
struct InternalThreadState;
}
#define STATE_PTR(STATE_TYPE, FIELD) \
STATE.R(), offsetof(FEXCore::Core::STATE_TYPE, FIELD)
namespace FEXCore::CPU {
class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter {
@@ -29,6 +32,8 @@ class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter {
void ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) override;
#endif
void EmitDispatcher();
protected:
void SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) override;
External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
Vendored
+601 -661
View File
File diff suppressed because it is too large. Load diff
External/FEXCore/Source/Interface/Core/JIT/Arm64/Arm64Relocations.cpp
Vendored
+16 -16
View File
@@ -9,7 +9,7 @@ $end_info$
#include "Interface/HLE/Thunks/Thunks.h"
namespace FEXCore::CPU {
uint64_t Arm64JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
switch (Op) {
case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER:
@@ -22,18 +22,18 @@ uint64_t Arm64JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiter
return ~0ULL;
}
void Arm64JITCore::InsertNamedThunkRelocation(vixl::aarch64::Register Reg, const IR::SHA256Sum &Sum) {
void Arm64JITCore::InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR::SHA256Sum &Sum) {
Relocation MoveABI{};
MoveABI.NamedThunkMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE;
// Offset is the offset from the entrypoint of the block
auto CurrentCursor = GetCursorAddress<uint8_t *>();
MoveABI.NamedThunkMove.Offset = CurrentCursor - GuestEntry;
MoveABI.NamedThunkMove.Symbol = Sum;
MoveABI.NamedThunkMove.RegisterIndex = Reg.GetCode();
MoveABI.NamedThunkMove.RegisterIndex = Reg.Idx();
uint64_t Pointer = reinterpret_cast<uint64_t>(EmitterCTX->ThunkHandler->LookupThunk(Sum));
LoadConstant(Reg, Pointer, EmitterCTX->Config.CacheObjectCodeCompilation());
LoadConstant(ARMEmitter::Size::i64Bit, Reg, Pointer, EmitterCTX->Config.CacheObjectCodeCompilation());
Relocations.emplace_back(MoveABI);
}
@@ -41,7 +41,7 @@ Arm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXC
uint64_t Pointer = GetNamedSymbolLiteral(Op);
Arm64JITCore::NamedSymbolLiteralPair Lit {
.Lit = Literal(Pointer),
.Lit = Pointer,
.MoveABI = {
.NamedSymbolLiteral = {
.Header = {
@@ -60,20 +60,21 @@ void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit) {
auto CurrentCursor = GetCursorAddress<uint8_t *>();
Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - GuestEntry;
place(&Lit.Lit);
Bind(&Lit.Loc);
dc64(Lit.Lit);
Relocations.emplace_back(Lit.MoveABI);
}
void Arm64JITCore::InsertGuestRIPMove(vixl::aarch64::Register Reg, uint64_t Constant) {
void Arm64JITCore::InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant) {
Relocation MoveABI{};
MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE;
// Offset is the offset from the entrypoint of the block
auto CurrentCursor = GetCursorAddress<uint8_t *>();
MoveABI.GuestRIPMove.Offset = CurrentCursor - GuestEntry;
MoveABI.GuestRIPMove.GuestRIP = Constant;
MoveABI.GuestRIPMove.RegisterIndex = Reg.GetCode();
MoveABI.GuestRIPMove.RegisterIndex = Reg.Idx();
LoadConstant(Reg, Constant, EmitterCTX->Config.CacheObjectCodeCompilation());
LoadConstant(ARMEmitter::Size::i64Bit, Reg, Constant, EmitterCTX->Config.CacheObjectCodeCompilation());
Relocations.emplace_back(MoveABI);
}
@@ -87,11 +88,10 @@ bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uin
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
uint64_t Pointer = GetNamedSymbolLiteral(Reloc->NamedSymbolLiteral.Symbol);
// Relocation occurs at the cursorEntry + offset relative to that cursor
GetBuffer()->SetCursorOffset(CursorEntry + Reloc->NamedSymbolLiteral.Offset);
SetCursorOffset(CursorEntry + Reloc->NamedSymbolLiteral.Offset);
// Generate a literal so we can place it
Literal<uint64_t> Lit(Pointer);
place(&Lit);
dc64(Pointer);
DataIndex += sizeof(Reloc->NamedSymbolLiteral);
break;
@@ -103,8 +103,8 @@ bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uin
}
// Relocation occurs at the cursorEntry + offset relative to that cursor.
GetBuffer()->SetCursorOffset(CursorEntry + Reloc->NamedThunkMove.Offset);
LoadConstant(vixl::aarch64::XRegister(Reloc->NamedThunkMove.RegisterIndex), Pointer, true);
SetCursorOffset(CursorEntry + Reloc->NamedThunkMove.Offset);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc->NamedThunkMove.RegisterIndex), Pointer, true);
DataIndex += sizeof(Reloc->NamedThunkMove);
break;
}
@@ -117,8 +117,8 @@ bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uin
}
// Relocation occurs at the cursorEntry + offset relative to that cursor.
GetBuffer()->SetCursorOffset(CursorEntry + Reloc->GuestRIPMove.Offset);
LoadConstant(vixl::aarch64::XRegister(Reloc->GuestRIPMove.RegisterIndex), Pointer, true);
SetCursorOffset(CursorEntry + Reloc->GuestRIPMove.Offset);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc->GuestRIPMove.RegisterIndex), Pointer, true);
DataIndex += sizeof(Reloc->GuestRIPMove);
break;
}
External/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp
Vendored
+289 -770
View File
File diff suppressed because it is too large. Load diff
External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp
Vendored
+138 -179
View File
@@ -6,6 +6,7 @@ $end_info$
#include "Interface/Context/Context.h"
#include "FEXCore/IR/IR.h"
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/LookupCache.h"
#include "Interface/Core/JIT/Arm64/JITClass.h"
@@ -17,8 +18,6 @@ $end_info$
#include <Interface/HLE/Thunks/Thunks.h>
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(SignalReturn) {
@@ -27,12 +26,11 @@ DEF_OP(SignalReturn) {
// Now branch to our signal return helper
// This can't be a direct branch since the code needs to live at a constant location
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler)));
br(x0);
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler));
br(ARMEmitter::Reg::r0);
}
DEF_OP(CallbackReturn) {
// spill back to CTX
SpillStaticRegs();
@@ -41,14 +39,14 @@ DEF_OP(CallbackReturn) {
// We can now lower the ref counter again
ldr(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)));
sub(w2, w2, 1);
str(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)));
ldr(ARMEmitter::WReg::w2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter));
sub(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 1);
str(ARMEmitter::WReg::w2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter));
// We need to adjust an additional 8 bytes to get back to the original "misaligned" RSP state
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])));
add(x2, x2, 8);
str(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])));
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP]));
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 8);
str(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP]));
PopCalleeSavedRegisters();
@@ -59,39 +57,41 @@ DEF_OP(CallbackReturn) {
DEF_OP(ExitFunction) {
auto Op = IROp->C<IR::IROp_ExitFunction>();
Label FullLookup;
ResetStack();
aarch64::Register RipReg;
uint64_t NewRIP;
if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
Literal l_BranchHost{ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker};
Literal l_BranchGuest{NewRIP};
ARMEmitter::ForwardLabel l_BranchHost;
ARMEmitter::ForwardLabel l_BranchGuest;
ldr(x0, &l_BranchHost);
blr(x0);
ldr(ARMEmitter::XReg::x0, &l_BranchHost);
blr(ARMEmitter::Reg::r0);
Bind(&l_BranchHost);
dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
Bind(&l_BranchGuest);
dc64(NewRIP);
place(&l_BranchHost);
place(&l_BranchGuest);
} else {
RipReg = GetReg<RA_64>(Op->NewRIP.ID());
ARMEmitter::ForwardLabel FullLookup;
auto RipReg = GetReg(Op->NewRIP.ID());
// L1 Cache
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer)));
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer));
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x3, Shift::LSL, 4));
and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg, LookupCache::L1_ENTRIES_MASK);
add(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::XReg::x3, ARMEmitter::ShiftType::LSL, 4);
ldp(x1, x0, MemOperand(x0));
cmp(x0, RipReg);
b(&FullLookup, Condition::ne);
br(x1);
ldp<ARMEmitter::IndexType::OFFSET>(ARMEmitter::XReg::x1, ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, 0);
cmp(ARMEmitter::XReg::x0, RipReg.X());
b(ARMEmitter::Condition::CC_NE, &FullLookup);
br(ARMEmitter::Reg::r1);
bind(&FullLookup);
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop)));
str(RipReg, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
Bind(&FullLookup);
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
br(TMP1);
}
}
@@ -103,66 +103,65 @@ DEF_OP(Jump) {
PendingTargetLabel = &JumpTargets.try_emplace(Target).first->second;
}
#define GRCMP(Node) (Op->CompareSize == 4 ? GetReg<RA_32>(Node) : GetReg<RA_64>(Node))
#define GRFCMP(Node) (Op->CompareSize == 4 ? GetVReg(Node).S() : GetVReg(Node).D())
static Condition MapBranchCC(IR::CondClassType Cond) {
static ARMEmitter::Condition MapBranchCC(IR::CondClassType Cond) {
switch (Cond.Val) {
case FEXCore::IR::COND_EQ: return Condition::eq;
case FEXCore::IR::COND_NEQ: return Condition::ne;
case FEXCore::IR::COND_SGE: return Condition::ge;
case FEXCore::IR::COND_SLT: return Condition::lt;
case FEXCore::IR::COND_SGT: return Condition::gt;
case FEXCore::IR::COND_SLE: return Condition::le;
case FEXCore::IR::COND_UGE: return Condition::cs;
case FEXCore::IR::COND_ULT: return Condition::cc;
case FEXCore::IR::COND_UGT: return Condition::hi;
case FEXCore::IR::COND_ULE: return Condition::ls;
case FEXCore::IR::COND_FLU: return Condition::lt;
case FEXCore::IR::COND_FGE: return Condition::ge;
case FEXCore::IR::COND_FLEU:return Condition::le;
case FEXCore::IR::COND_FGT: return Condition::gt;
case FEXCore::IR::COND_FU: return Condition::vs;
case FEXCore::IR::COND_FNU: return Condition::vc;
case FEXCore::IR::COND_EQ: return ARMEmitter::Condition::CC_EQ;
case FEXCore::IR::COND_NEQ: return ARMEmitter::Condition::CC_NE;
case FEXCore::IR::COND_SGE: return ARMEmitter::Condition::CC_GE;
case FEXCore::IR::COND_SLT: return ARMEmitter::Condition::CC_LT;
case FEXCore::IR::COND_SGT: return ARMEmitter::Condition::CC_GT;
case FEXCore::IR::COND_SLE: return ARMEmitter::Condition::CC_LE;
case FEXCore::IR::COND_UGE: return ARMEmitter::Condition::CC_CS;
case FEXCore::IR::COND_ULT: return ARMEmitter::Condition::CC_CC;
case FEXCore::IR::COND_UGT: return ARMEmitter::Condition::CC_HI;
case FEXCore::IR::COND_ULE: return ARMEmitter::Condition::CC_LS;
case FEXCore::IR::COND_FLU: return ARMEmitter::Condition::CC_LT;
case FEXCore::IR::COND_FGE: return ARMEmitter::Condition::CC_GE;
case FEXCore::IR::COND_FLEU:return ARMEmitter::Condition::CC_LE;
case FEXCore::IR::COND_FGT: return ARMEmitter::Condition::CC_GT;
case FEXCore::IR::COND_FU: return ARMEmitter::Condition::CC_VS;
case FEXCore::IR::COND_FNU: return ARMEmitter::Condition::CC_VC;
case FEXCore::IR::COND_VS:
case FEXCore::IR::COND_VC:
case FEXCore::IR::COND_MI:
case FEXCore::IR::COND_PL:
default:
LOGMAN_MSG_A_FMT("Unsupported compare type");
return Condition::nv;
return ARMEmitter::Condition::CC_NV;
}
}
DEF_OP(CondJump) {
auto Op = IROp->C<IR::IROp_CondJump>();
Label *TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second;
auto TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second;
uint64_t Const;
const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
const auto Size = Op->CompareSize == 4 ? ARMEmitter::Size::i32Bit : ARMEmitter::Size::i64Bit;
const auto SubSize = ARMEmitter::ToVectorSizePair(Op->CompareSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i64Bit);
if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_EQ) {
LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
cbz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel);
cbz(Size, GetReg(Op->Cmp1.ID()), TrueTargetLabel);
} else if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_NEQ) {
LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
cbnz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel);
cbnz(Size, GetReg(Op->Cmp1.ID()), TrueTargetLabel);
} else {
if (IsGPR(Op->Cmp1.ID())) {
if (isConst) {
cmp(GRCMP(Op->Cmp1.ID()), Const);
cmp(Size, GetReg(Op->Cmp1.ID()), Const);
} else {
cmp(GRCMP(Op->Cmp1.ID()), GRCMP(Op->Cmp2.ID()));
cmp(Size, GetReg(Op->Cmp1.ID()), GetReg(Op->Cmp2.ID()));
}
} else if (IsFPR(Op->Cmp1.ID())) {
fcmp(GRFCMP(Op->Cmp1.ID()), GRFCMP(Op->Cmp2.ID()));
fcmp(SubSize.Scalar, GetVReg(Op->Cmp1.ID()), GetVReg(Op->Cmp2.ID()));
} else {
LOGMAN_MSG_A_FMT("CondJump: Expected GPR or FPR");
}
b(TrueTargetLabel, MapBranchCC(Op->Cond));
b(MapBranchCC(Op->Cond), TrueTargetLabel);
}
PendingTargetLabel = &JumpTargets.try_emplace(Op->FalseBlock.ID()).first->second;
@@ -187,23 +186,25 @@ DEF_OP(Syscall) {
}
uint64_t SPOffset = AlignUp(FEXCore::HLE::SyscallArguments::MAX_ARGS * 8, 16);
sub(sp, sp, SPOffset);
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS; ++i) {
if (Op->Header.Args[i].IsInvalid()) continue;
str(GetReg<RA_64>(Op->Header.Args[i].ID()), MemOperand(sp, i * 8));
str(GetReg(Op->Header.Args[i].ID()).X(), ARMEmitter::Reg::rsp, i * 8);
}
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj)));
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc)));
mov(x1, STATE);
mov(x2, sp);
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj));
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R());
// SP supporting move
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::rsp, 0);
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint64_t, void*, void*, void*>(x3);
GenerateIndirectRuntimeCall<uint64_t, void*, void*, void*>(ARMEmitter::Reg::r3);
#else
blr(x3);
blr(ARMEmitter::Reg::r3);
#endif
add(sp, sp, SPOffset);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);
if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) != FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY &&
(Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
@@ -219,7 +220,7 @@ DEF_OP(Syscall) {
if ((Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
// Move result to its destination register
mov(GetReg<RA_64>(Node), x0);
mov(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0);
}
}
@@ -236,8 +237,8 @@ DEF_OP(InlineSyscall) {
// X6: Arg6 - Doesn't exist in x86-64 land. RA INTERSECT
// One argument is removed from the SyscallArguments::MAX_ARGS since the first argument was syscall number
const static std::array<vixl::aarch64::Register, FEXCore::HLE::SyscallArguments::MAX_ARGS-1> RegArgs = {{
x0, x1, x2, x3, x4, x5
const static std::array<ARMEmitter::XRegister, FEXCore::HLE::SyscallArguments::MAX_ARGS-1> RegArgs = {{
ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::XReg::x2, ARMEmitter::XReg::x3, ARMEmitter::XReg::x4, ARMEmitter::XReg::x5
}};
bool Intersects{};
@@ -246,12 +247,12 @@ DEF_OP(InlineSyscall) {
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) {
if (Op->Header.Args[i].IsInvalid()) break;
auto Reg = GetReg<RA_64>(Op->Header.Args[i].ID());
if (Reg.GetCode() == x8.GetCode() ||
Reg.GetCode() == x4.GetCode() ||
Reg.GetCode() == x5.GetCode()) {
auto Reg = GetReg(Op->Header.Args[i].ID());
if (Reg.Idx() == ARMEmitter::Reg::r8.Idx() ||
Reg.Idx() == ARMEmitter::Reg::r4.Idx() ||
Reg.Idx() == ARMEmitter::Reg::r5.Idx()) {
SpillMask |= (1U << Reg.GetCode());
SpillMask |= (1U << Reg.Idx());
Intersects = true;
}
}
@@ -266,66 +267,31 @@ DEF_OP(InlineSyscall) {
// 16bit LoadConstant to be a single instruction
// We must always spill at least one register (x8) so this value always has a bit set
// This gives the signal handler a value to check to see if we are in a syscall at all
LoadConstant(x0, SpillMask & 0xFFFF);
str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)));
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SpillMask & 0xFFFF);
str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
// Now that we have claimed to be a syscall we can set up the arguments
const auto EmitSize = CTX->Config.Is64BitMode() ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
const auto EmitSubSize = CTX->Config.Is64BitMode() ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i32Bit;
if (Intersects) {
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) {
if (Op->Header.Args[i].IsInvalid()) break;
if (CTX->Config.Is64BitMode()) {
auto Reg = GetReg<RA_64>(Op->Header.Args[i].ID());
// In the case of intersection with x4, x5, or x8 then these are currently SRA
// for registers RAX, RBX, and RSI. Which have just been spilled
// Just load back from the context. Could be slightly smarter but this is fairly uncommon
if (Reg.GetCode() == x8.GetCode()) {
ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI])));
}
else if (Reg.GetCode() == x4.GetCode()) {
ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX])));
}
else if (Reg.GetCode() == x5.GetCode()) {
ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX])));
}
auto Reg = GetReg(Op->Header.Args[i].ID());
// In the case of intersection with x4, x5, or x8 then these are currently SRA
// for registers RAX, RBX, and RSI. Which have just been spilled
// Just load back from the context. Could be slightly smarter but this is fairly uncommon
if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r8.Idx()) {
ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI]));
}
}
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) {
if (Op->Header.Args[i].IsInvalid()) break;
if (CTX->Config.Is64BitMode()) {
auto Reg = GetReg<RA_64>(Op->Header.Args[i].ID());
// In the case of intersection with x4, x5, or x8 then these are currently SRA
// for registers RAX, RBX, and RSI. Which have just been spilled
// Just load back from the context. Could be slightly smarter but this is fairly uncommon
if (Reg.GetCode() == x8.GetCode()) {
ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI])));
}
else if (Reg.GetCode() == x4.GetCode()) {
ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX])));
}
else if (Reg.GetCode() == x5.GetCode()) {
ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX])));
}
else {
mov(RegArgs[i], Reg);
}
else if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r4.Idx()) {
ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX]));
}
else if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r5.Idx()) {
ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX]));
}
else {
auto Reg = GetReg<RA_32>(Op->Header.Args[i].ID());
if (Reg.GetCode() == w8.GetCode()) {
ldr(RegArgs[i].W(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI])));
}
else if (Reg.GetCode() == w4.GetCode()) {
ldr(RegArgs[i].W(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX])));
}
else if (Reg.GetCode() == w5.GetCode()) {
ldr(RegArgs[i].W(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX])));
}
else {
uxtw(RegArgs[i].W(), Reg);
}
mov(EmitSize, RegArgs[i].R(), Reg);
}
}
}
@@ -333,16 +299,11 @@ DEF_OP(InlineSyscall) {
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) {
if (Op->Header.Args[i].IsInvalid()) break;
if (CTX->Config.Is64BitMode()) {
mov(RegArgs[i], GetReg<RA_64>(Op->Header.Args[i].ID()));
}
else {
uxtw(RegArgs[i], GetReg<RA_64>(Op->Header.Args[i].ID()));
}
mov(EmitSize, RegArgs[i].R(), GetReg(Op->Header.Args[i].ID()));
}
}
LoadConstant(x8, Op->HostSyscallNumber);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, Op->HostSyscallNumber);
svc(0);
// On updated signal mask we can receive a signal RIGHT HERE
@@ -353,16 +314,11 @@ DEF_OP(InlineSyscall) {
// Now the registers we've spilled are back in their original host registers
// We can safely claim we are no longer in a syscall
str(xzr, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)));
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
// Result is now in x0
// Move result to its destination register
if (CTX->Config.Is64BitMode()) {
mov(GetReg<RA_64>(Node), x0);
}
else {
uxtw(GetReg<RA_64>(Node), x0);
}
mov(EmitSize, GetReg(Node), ARMEmitter::Reg::r0);
}
}
@@ -376,14 +332,14 @@ DEF_OP(Thunk) {
PushDynamicRegsAndLR(TMP1);
mov(x0, GetReg<RA_64>(Op->ArgPtr.ID()));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->ArgPtr.ID()));
auto thunkFn = ThreadState->CTX->ThunkHandler->LookupThunk(Op->ThunkNameHash);
LoadConstant(x2, (uintptr_t)thunkFn);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, (uintptr_t)thunkFn);
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<void, void*, void*>(x2);
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PopDynamicRegsAndLR();
@@ -397,43 +353,45 @@ DEF_OP(ValidateCode) {
int len = Op->CodeLength;
int idx = 0;
LoadConstant(GetReg<RA_64>(Node), 0);
LoadConstant(x0, Entry + Op->Offset);
LoadConstant(x1, 1);
LoadConstant(ARMEmitter::Size::i64Bit, GetReg(Node), 0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, Entry + Op->Offset);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, 1);
const auto Dst = GetReg(Node);
while (len >= 8)
{
ldr(x2, MemOperand(x0, idx));
LoadConstant(x3, *(const uint32_t *)(OldCode + idx));
cmp(x2, x3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
ldr(ARMEmitter::XReg::x2, ARMEmitter::Reg::r0, idx);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint32_t *)(OldCode + idx));
cmp(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3);
csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ);
len -= 8;
idx += 8;
}
while (len >= 4)
{
ldr(w2, MemOperand(x0, idx));
LoadConstant(w3, *(const uint32_t *)(OldCode + idx));
cmp(w2, w3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
ldr(ARMEmitter::WReg::w2, ARMEmitter::Reg::r0, idx);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint32_t *)(OldCode + idx));
cmp(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3);
csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ);
len -= 4;
idx += 4;
}
while (len >= 2)
{
ldrh(w2, MemOperand(x0, idx));
LoadConstant(w3, *(const uint16_t *)(OldCode + idx));
cmp(w2, w3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
ldrh(ARMEmitter::Reg::r2, ARMEmitter::Reg::r0, idx);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint16_t *)(OldCode + idx));
cmp(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3);
csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ);
len -= 2;
idx += 2;
}
while (len >= 1)
{
ldrb(w2, MemOperand(x0, idx));
LoadConstant(w3, *(const uint8_t *)(OldCode + idx));
cmp(w2, w3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
ldrb(ARMEmitter::Reg::r2, ARMEmitter::Reg::r0, idx);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint8_t *)(OldCode + idx));
cmp(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3);
csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ);
len -= 1;
idx += 1;
}
@@ -446,15 +404,15 @@ DEF_OP(ThreadRemoveCodeEntry) {
PushDynamicRegsAndLR(TMP1);
mov(x0, STATE);
LoadConstant(x1, Entry);
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Entry);
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT)));
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT));
SpillStaticRegs();
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<void, void*, void*>(x2);
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
FillStaticRegs();
@@ -471,24 +429,25 @@ DEF_OP(CPUID) {
// x0 = CPUID Handler
// x1 = CPUID Function
// x2 = CPUID Leaf
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)));
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction)));
mov(x1, GetReg<RA_64>(Op->Function.ID()));
mov(x2, GetReg<RA_64>(Op->Leaf.ID()));
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj));
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetReg(Op->Function.ID()));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, GetReg(Op->Leaf.ID()));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(x3);
GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
#else
blr(x3);
blr(ARMEmitter::Reg::r3);
#endif
FillStaticRegs();
PopDynamicRegsAndLR();
// Results are in x0, x1
// Results want to be in a i64v2 vector
auto Dst = GetRegPair<RA_64>(Node);
mov(Dst.first, x0);
mov(Dst.second, x1);
auto Dst = GetRegPair(Node);
mov(ARMEmitter::Size::i64Bit, Dst.first, ARMEmitter::Reg::r0);
mov(ARMEmitter::Size::i64Bit, Dst.second, ARMEmitter::Reg::r1);
}
#undef DEF_OP
External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp
Vendored
+104 -308
View File
@@ -4,12 +4,10 @@ tags: backend|arm64
$end_info$
*/
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/JIT/Arm64/JITClass.h"
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(VInsGPR) {
const auto Op = IROp->C<IR::IROp_VInsGPR>();
@@ -19,8 +17,16 @@ DEF_OP(VInsGPR) {
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2 || ElementSize == 1, "Unexpected {} size", __func__);
const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit;
const auto ElementsPer128Bit = 16 / ElementSize;
const auto Dst = GetVReg(Node);
const auto DestVector = GetVReg(Op->DestVector.ID());
const auto Src = GetReg(Op->Src.ID());
if (HostSupportsSVE && Is256Bit) {
const auto ElementSizeBits = ElementSize * 8;
@@ -47,99 +53,56 @@ DEF_OP(VInsGPR) {
if (InUpperLane) {
// Move the upper lane down for the insertion.
const auto CompactPred = p0;
not_(CompactPred.VnB(), PRED_TMP_32B.Zeroing(), PRED_TMP_16B.VnB());
compact(VTMP1.Z().VnD(), CompactPred, DestVector.Z().VnD());
const auto CompactPred = ARMEmitter::PReg::p0;
not_(CompactPred, PRED_TMP_32B.Zeroing(), PRED_TMP_16B);
compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, DestVector);
}
// Put data in place for destructive SPLICE below.
mov(Dst.Z().VnD(), DestVector.Z().VnD());
mov(Dst.Z(), DestVector.Z());
// Inserts the GPR value into the given V register.
// Also automatically adjusts the index in the case of using the
// moved upper lane.
const auto Insert = [&](const aarch64::VRegister& reg, int index) {
switch (ElementSize) {
case 1:
if (InUpperLane) {
index -= 16;
}
ins(reg.V16B(), index, GetReg<RA_32>(Op->Src.ID()));
break;
case 2:
if (InUpperLane) {
index -= 8;
}
ins(reg.V8H(), index, GetReg<RA_32>(Op->Src.ID()));
break;
case 4:
if (InUpperLane) {
index -= 4;
}
ins(reg.V4S(), index, GetReg<RA_32>(Op->Src.ID()));
break;
case 8:
if (InUpperLane) {
index -= 2;
}
ins(reg.V2D(), index, GetReg<RA_64>(Op->Src.ID()));
break;
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
const auto Insert = [&](const FEXCore::ARMEmitter::VRegister& reg, int index) {
if (InUpperLane) {
index -= ElementsPer128Bit;
}
ins(SubEmitSize, reg, index, Src);
};
if (InUpperLane) {
Insert(VTMP1, DestIdx);
splice(Dst.Z().VnD(), PRED_TMP_16B, Dst.Z().VnD(), VTMP1.Z().VnD());
splice<ARMEmitter::OpType::Destructive>(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP1.Z());
} else {
Insert(Dst, DestIdx);
splice(Dst.Z().VnD(), PRED_TMP_16B, Dst.Z().VnD(), DestVector.Z().VnD());
splice<ARMEmitter::OpType::Destructive>(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), DestVector.Z());
}
} else {
mov(Dst, DestVector);
switch (ElementSize) {
case 1: {
ins(Dst.V16B(), DestIdx, GetReg<RA_32>(Op->Src.ID()));
break;
}
case 2: {
ins(Dst.V8H(), DestIdx, GetReg<RA_32>(Op->Src.ID()));
break;
}
case 4: {
ins(Dst.V4S(), DestIdx, GetReg<RA_32>(Op->Src.ID()));
break;
}
case 8: {
ins(Dst.V2D(), DestIdx, GetReg<RA_64>(Op->Src.ID()));
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
}
mov(Dst.Q(), DestVector.Q());
ins(SubEmitSize, Dst, DestIdx, Src);
}
}
DEF_OP(VCastFromGPR) {
auto Op = IROp->C<IR::IROp_VCastFromGPR>();
auto Dst = GetVReg(Node);
auto Src = GetReg(Op->Src.ID());
switch (Op->Header.ElementSize) {
case 1:
uxtb(TMP1.W(), GetReg<RA_32>(Op->Src.ID()));
fmov(GetVReg(Node).S(), TMP1.W());
uxtb(ARMEmitter::Size::i32Bit, TMP1, Src);
fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1);
break;
case 2:
uxth(TMP1.W(), GetReg<RA_32>(Op->Src.ID()));
fmov(GetVReg(Node).S(), TMP1.W());
uxth(ARMEmitter::Size::i32Bit, TMP1, Src);
fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1);
break;
case 4:
fmov(GetVReg(Node).S(), GetReg<RA_32>(Op->Src.ID()).W());
fmov(ARMEmitter::Size::i32Bit, Dst.S(), Src);
break;
case 8:
fmov(GetVReg(Node).D(), GetReg<RA_64>(Op->Src.ID()).X());
fmov(ARMEmitter::Size::i64Bit, Dst.D(), Src);
break;
default: LOGMAN_MSG_A_FMT("Unknown castGPR element size: {}", Op->Header.ElementSize);
}
@@ -151,21 +114,24 @@ DEF_OP(Float_FromGPR_S) {
const uint16_t ElementSize = Op->Header.ElementSize;
const uint16_t Conv = (ElementSize << 8) | Op->SrcElementSize;
auto Dst = GetVReg(Node);
auto Src = GetReg(Op->Src.ID());
switch (Conv) {
case 0x0404: { // Float <- int32_t
scvtf(GetVReg(Node).S(), GetReg<RA_32>(Op->Src.ID()));
scvtf(ARMEmitter::Size::i32Bit, Dst.S(), Src);
break;
}
case 0x0408: { // Float <- int64_t
scvtf(GetVReg(Node).S(), GetReg<RA_64>(Op->Src.ID()));
scvtf(ARMEmitter::Size::i64Bit, Dst.S(), Src);
break;
}
case 0x0804: { // Double <- int32_t
scvtf(GetVReg(Node).D(), GetReg<RA_32>(Op->Src.ID()));
scvtf(ARMEmitter::Size::i32Bit, Dst.D(), Src);
break;
}
case 0x0808: { // Double <- int64_t
scvtf(GetVReg(Node).D(), GetReg<RA_64>(Op->Src.ID()));
scvtf(ARMEmitter::Size::i64Bit, Dst.D(), Src);
break;
}
default:
@@ -178,13 +144,17 @@ DEF_OP(Float_FromGPR_S) {
DEF_OP(Float_FToF) {
auto Op = IROp->C<IR::IROp_Float_FToF>();
const uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
auto Dst = GetVReg(Node);
auto Src = GetVReg(Op->Scalar.ID());
switch (Conv) {
case 0x0804: { // Double <- Float
fcvt(GetVReg(Node).D(), GetVReg(Op->Scalar.ID()).S());
fcvt(Dst.D(), Src.S());
break;
}
case 0x0408: { // Float <- Double
fcvt(GetVReg(Node).S(), GetVReg(Op->Scalar.ID()).D());
fcvt(Dst.S(), Src.D());
break;
}
default: LOGMAN_MSG_A_FMT("Unknown FCVT sizes: 0x{:x}", Conv);
@@ -198,41 +168,18 @@ DEF_OP(Vector_SToF) {
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__);
const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit;
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector.ID());
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B.Merging();
switch (ElementSize) {
case 2:
scvtf(Dst.Z().VnH(), Mask, Vector.Z().VnH());
break;
case 4:
scvtf(Dst.Z().VnS(), Mask, Vector.Z().VnS());
break;
case 8:
scvtf(Dst.Z().VnD(), Mask, Vector.Z().VnD());
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize);
break;
}
const auto Mask = PRED_TMP_32B;
scvtf(Dst.Z(), SubEmitSize, Mask.Merging(), Vector.Z(), SubEmitSize);
} else {
switch (ElementSize) {
case 2:
scvtf(Dst.V8H(), Vector.V8H());
break;
case 4:
scvtf(Dst.V4S(), Vector.V4S());
break;
case 8:
scvtf(Dst.V2D(), Vector.V2D());
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize);
break;
}
scvtf(SubEmitSize, Dst.Q(), Vector.Q());
}
}
@@ -243,41 +190,18 @@ DEF_OP(Vector_FToZS) {
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__);
const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit;
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector.ID());
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B.Merging();
switch (ElementSize) {
case 2:
fcvtzs(Dst.Z().VnH(), Mask, Vector.Z().VnH());
break;
case 4:
fcvtzs(Dst.Z().VnS(), Mask, Vector.Z().VnS());
break;
case 8:
fcvtzs(Dst.Z().VnD(), Mask, Vector.Z().VnD());
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize);
break;
}
const auto Mask = PRED_TMP_32B;
fcvtzs(Dst, SubEmitSize, Mask.Merging(), Vector, SubEmitSize);
} else {
switch (ElementSize) {
case 2:
fcvtzs(Dst.V8H(), Vector.V8H());
break;
case 4:
fcvtzs(Dst.V4S(), Vector.V4S());
break;
case 8:
fcvtzs(Dst.V2D(), Vector.V2D());
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize);
break;
}
fcvtzs(SubEmitSize, Dst.Q(), Vector.Q());
}
}
@@ -288,47 +212,23 @@ DEF_OP(Vector_FToS) {
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__);
const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit;
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector.ID());
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B.Merging();
switch (ElementSize) {
case 2:
frinti(Dst.Z().VnH(), Mask, Vector.Z().VnH());
fcvtzs(Dst.Z().VnH(), Mask, Dst.Z().VnH());
break;
case 4:
frinti(Dst.Z().VnS(), Mask, Vector.Z().VnS());
fcvtzs(Dst.Z().VnS(), Mask, Dst.Z().VnS());
break;
case 8:
frinti(Dst.Z().VnD(), Mask, Vector.Z().VnD());
fcvtzs(Dst.Z().VnD(), Mask, Dst.Z().VnD());
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize);
break;
}
const auto Mask = PRED_TMP_32B;
frinti(SubEmitSize, Dst, Mask.Merging(), Vector);
fcvtzs(Dst, SubEmitSize, Mask.Merging(), Dst, SubEmitSize);
} else {
switch (ElementSize) {
case 2:
frinti(Dst.V8H(), Vector.V8H());
fcvtzs(Dst.V8H(), Dst.V8H());
break;
case 4:
frinti(Dst.V4S(), Vector.V4S());
fcvtzs(Dst.V4S(), Dst.V4S());
break;
case 8:
frinti(Dst.V2D(), Vector.V2D());
fcvtzs(Dst.V2D(), Dst.V2D());
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize);
break;
}
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector.ID());
frinti(SubEmitSize, Dst.Q(), Vector.Q());
fcvtzs(SubEmitSize, Dst.Q(), Dst.Q());
}
}
@@ -340,6 +240,11 @@ DEF_OP(Vector_FToF) {
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Conv = (ElementSize << 8) | Op->SrcElementSize;
LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__);
const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit;
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector.ID());
@@ -361,23 +266,23 @@ DEF_OP(Vector_FToF) {
switch (Conv) {
case 0x0402: { // Float <- Half
zip1(Dst.Z().VnH(), Vector.Z().VnH(), Vector.Z().VnH());
fcvtlt(Dst.Z().VnS(), Mask, Dst.Z().VnH());
zip1(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Vector.Z(), Vector.Z());
fcvtlt(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Mask, Dst.Z());
break;
}
case 0x0804: { // Double <- Float
zip1(Dst.Z().VnS(), Vector.Z().VnS(), Vector.Z().VnS());
fcvtlt(Dst.Z().VnD(), Mask, Dst.Z().VnS());
zip1(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Vector.Z(), Vector.Z());
fcvtlt(FEXCore::ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Dst.Z());
break;
}
case 0x0204: { // Half <- Float
fcvtnt(Dst.Z().VnH(), Mask, Vector.Z().VnS());
uzp2(Dst.Z().VnH(), Dst.Z().VnH(), Dst.Z().VnH());
fcvtnt(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst, Mask, Vector);
uzp2(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Dst.Z(), Dst.Z());
break;
}
case 0x0408: { // Float <- Double
fcvtnt(Dst.Z().VnS(), Mask, Vector.Z().VnD());
uzp2(Dst.Z().VnS(), Dst.Z().VnS(), Dst.Z().VnS());
fcvtnt(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst, Mask, Vector);
uzp2(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Dst.Z(), Dst.Z());
break;
}
default:
@@ -386,20 +291,14 @@ DEF_OP(Vector_FToF) {
}
} else {
switch (Conv) {
case 0x0402: { // Float <- Half
fcvtl(Dst.V4S(), Vector.V4H());
break;
}
case 0x0402: // Float <- Half
case 0x0804: { // Double <- Float
fcvtl(Dst.V2D(), Vector.V2S());
break;
}
case 0x0204: { // Half <- Float
fcvtn(Dst.V4H(), Vector.V4S());
fcvtl(SubEmitSize, Dst.D(), Vector.D());
break;
}
case 0x0204: // Half <- Float
case 0x0408: { // Float <- Double
fcvtn(Dst.V2S(), Vector.V2D());
fcvtn(SubEmitSize, Dst.D(), Vector.D());
break;
}
default:
@@ -415,154 +314,51 @@ DEF_OP(Vector_FToI) {
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__);
const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit;
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector.ID());
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B.Merging();
switch (Op->Round) {
case FEXCore::IR::Round_Nearest.Val:
switch (ElementSize) {
case 2:
frintn(Dst.Z().VnH(), Mask, Vector.Z().VnH());
break;
case 4:
frintn(Dst.Z().VnS(), Mask, Vector.Z().VnS());
break;
case 8:
frintn(Dst.Z().VnD(), Mask, Vector.Z().VnD());
break;
}
frintn(SubEmitSize, Dst.Z(), Mask, Vector.Z());
break;
case FEXCore::IR::Round_Negative_Infinity.Val:
switch (ElementSize) {
case 2:
frintm(Dst.Z().VnH(), Mask, Vector.Z().VnH());
break;
case 4:
frintm(Dst.Z().VnS(), Mask, Vector.Z().VnS());
break;
case 8:
frintm(Dst.Z().VnD(), Mask, Vector.Z().VnD());
break;
}
frintm(SubEmitSize, Dst.Z(), Mask, Vector.Z());
break;
case FEXCore::IR::Round_Positive_Infinity.Val:
switch (ElementSize) {
case 2:
frintp(Dst.Z().VnH(), Mask, Vector.Z().VnH());
break;
case 4:
frintp(Dst.Z().VnS(), Mask, Vector.Z().VnS());
break;
case 8:
frintp(Dst.Z().VnD(), Mask, Vector.Z().VnD());
break;
}
frintp(SubEmitSize, Dst.Z(), Mask, Vector.Z());
break;
case FEXCore::IR::Round_Towards_Zero.Val:
switch (ElementSize) {
case 2:
frintz(Dst.Z().VnH(), Mask, Vector.Z().VnH());
break;
case 4:
frintz(Dst.Z().VnS(), Mask, Vector.Z().VnS());
break;
case 8:
frintz(Dst.Z().VnD(), Mask, Vector.Z().VnD());
break;
}
frintz(SubEmitSize, Dst.Z(), Mask, Vector.Z());
break;
case FEXCore::IR::Round_Host.Val:
switch (ElementSize) {
case 2:
frinti(Dst.Z().VnH(), Mask, Vector.Z().VnH());
break;
case 4:
frinti(Dst.Z().VnS(), Mask, Vector.Z().VnS());
break;
case 8:
frinti(Dst.Z().VnD(), Mask, Vector.Z().VnD());
break;
}
frinti(SubEmitSize, Dst.Z(), Mask, Vector.Z());
break;
}
} else {
switch (Op->Round) {
case FEXCore::IR::Round_Nearest.Val:
switch (ElementSize) {
case 2:
frintn(Dst.V8H(), Vector.V8H());
break;
case 4:
frintn(Dst.V4S(), Vector.V4S());
break;
case 8:
frintn(Dst.V2D(), Vector.V2D());
break;
}
frinti(SubEmitSize, Dst.Q(), Vector.Q());
break;
case FEXCore::IR::Round_Negative_Infinity.Val:
switch (ElementSize) {
case 2:
frintm(Dst.V8H(), Vector.V8H());
break;
case 4:
frintm(Dst.V4S(), Vector.V4S());
break;
case 8:
frintm(Dst.V2D(), Vector.V2D());
break;
}
frintm(SubEmitSize, Dst.Q(), Vector.Q());
break;
case FEXCore::IR::Round_Positive_Infinity.Val:
switch (ElementSize) {
case 2:
frintp(Dst.V8H(), Vector.V8H());
break;
case 4:
frintp(Dst.V4S(), Vector.V4S());
break;
case 8:
frintp(Dst.V2D(), Vector.V2D());
break;
}
frintp(SubEmitSize, Dst.Q(), Vector.Q());
break;
case FEXCore::IR::Round_Towards_Zero.Val:
switch (ElementSize) {
case 2:
frintz(Dst.V8H(), Vector.V8H());
break;
case 4:
frintz(Dst.V4S(), Vector.V4S());
break;
case 8:
frintz(Dst.V2D(), Vector.V2D());
break;
}
frintz(SubEmitSize, Dst.Q(), Vector.Q());
break;
case FEXCore::IR::Round_Host.Val:
switch (ElementSize) {
case 2:
frinti(Dst.V8H(), Vector.V8H());
break;
case 4:
frinti(Dst.V4S(), Vector.V4S());
break;
case 8:
frinti(Dst.V2D(), Vector.V2D());
break;
}
frinti(SubEmitSize, Dst.Q(), Vector.Q());
break;
}
}
External/FEXCore/Source/Interface/Core/JIT/Arm64/EncryptionOps.cpp
Vendored
+53 -47
View File
@@ -4,98 +4,104 @@ tags: backend|arm64
$end_info$
*/
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/JIT/Arm64/JITClass.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(AESImc) {
auto Op = IROp->C<IR::IROp_VAESImc>();
aesimc(GetVReg(Node).V16B(), GetVReg(Op->Vector.ID()).V16B());
aesimc(GetVReg(Node), GetVReg(Op->Vector.ID()));
}
DEF_OP(AESEnc) {
auto Op = IROp->C<IR::IROp_VAESEnc>();
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B());
aese(VTMP1.V16B(), VTMP2.V16B());
aesmc(VTMP1.V16B(), VTMP1.V16B());
eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B());
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aese(VTMP1, VTMP2);
aesmc(VTMP1, VTMP1);
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESEncLast) {
auto Op = IROp->C<IR::IROp_VAESEncLast>();
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B());
aese(VTMP1.V16B(), VTMP2.V16B());
eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B());
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aese(VTMP1, VTMP2);
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESDec) {
auto Op = IROp->C<IR::IROp_VAESDec>();
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B());
aesd(VTMP1.V16B(), VTMP2.V16B());
aesimc(VTMP1.V16B(), VTMP1.V16B());
eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B());
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aesd(VTMP1, VTMP2);
aesimc(VTMP1, VTMP1);
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESDecLast) {
auto Op = IROp->C<IR::IROp_VAESDecLast>();
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B());
aesd(VTMP1.V16B(), VTMP2.V16B());
eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B());
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aesd(VTMP1, VTMP2);
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESKeyGenAssist) {
auto Op = IROp->C<IR::IROp_VAESKeyGenAssist>();
aarch64::Literal ConstantLiteral (0x0C030609'0306090CULL, 0x040B0E01'0B0E0104ULL);
aarch64::Label PastConstant;
ARMEmitter::ForwardLabel Constant;
ARMEmitter::ForwardLabel PastConstant;
// Do a "regular" AESE step
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
mov(VTMP1.V16B(), GetVReg(Op->Src.ID()).V16B());
aese(VTMP1.V16B(), VTMP2.V16B());
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), GetVReg(Op->Src.ID()).Q());
aese(VTMP1, VTMP2);
// Do a table shuffle to undo ShiftRows
ldr(VTMP3, &ConstantLiteral);
ldr(VTMP3.Q(), &Constant);
// Now EOR in the RCON
if (Op->RCON) {
tbl(VTMP1.V16B(), VTMP1.V16B(), VTMP3.V16B());
tbl(VTMP1.Q(), VTMP1.Q(), VTMP3.Q());
LoadConstant(TMP1, static_cast<uint64_t>(Op->RCON) << 32);
dup(VTMP2.V2D(), TMP1);
eor(GetVReg(Node).V16B(), VTMP1.V16B(), VTMP2.V16B());
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Op->RCON) << 32);
dup(ARMEmitter::SubRegSize::i64Bit, VTMP2.Q(), TMP1);
eor(GetVReg(Node).Q(), VTMP1.Q(), VTMP2.Q());
}
else {
tbl(GetVReg(Node).V16B(), VTMP1.V16B(), VTMP3.V16B());
tbl(GetVReg(Node).Q(), VTMP1.Q(), VTMP3.Q());
}
b(&PastConstant);
place(&ConstantLiteral);
bind(&PastConstant);
Bind(&Constant);
dc64(0x040B0E01'0B0E0104ULL);
dc64(0x0C030609'0306090CULL);
Bind(&PastConstant);
}
DEF_OP(CRC32) {
auto Op = IROp->C<IR::IROp_CRC32>();
const auto Dst = GetReg(Node);
const auto Src1 = GetReg(Op->Src1.ID());
const auto Src2 = GetReg(Op->Src2.ID());
switch (Op->SrcSize) {
case 1:
crc32cb(GetReg<RA_32>(Node), GetReg<RA_32>(Op->Src1.ID()), GetReg<RA_32>(Op->Src2.ID()));
crc32cb(Dst.W(), Src1.W(), Src2.W());
break;
case 2:
crc32ch(GetReg<RA_32>(Node), GetReg<RA_32>(Op->Src1.ID()), GetReg<RA_32>(Op->Src2.ID()));
crc32ch(Dst.W(), Src1.W(), Src2.W());
break;
case 4:
crc32cw(GetReg<RA_32>(Node), GetReg<RA_32>(Op->Src1.ID()), GetReg<RA_32>(Op->Src2.ID()));
crc32cw(Dst.W(), Src1.W(), Src2.W());
break;
case 8:
crc32cx(GetReg<RA_32>(Node), GetReg<RA_32>(Op->Src1.ID()), GetReg<RA_64>(Op->Src2.ID()));
crc32cx(Dst, Src1, Src2);
break;
default: LOGMAN_MSG_A_FMT("Unknown CRC32 size: {}", Op->SrcSize);
}
@@ -104,24 +110,24 @@ DEF_OP(CRC32) {
DEF_OP(PCLMUL) {
auto Op = IROp->C<IR::IROp_PCLMUL>();
auto Dst = GetVReg(Node).Q();
auto Src1 = GetVReg(Op->Src1.ID()).V2D();
auto Src2 = GetVReg(Op->Src2.ID()).V2D();
auto Dst = GetVReg(Node);
auto Src1 = GetVReg(Op->Src1.ID());
auto Src2 = GetVReg(Op->Src2.ID());
switch (Op->Selector) {
case 0b00000000:
pmull(Dst, Src1, Src2);
pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), Src1.D(), Src2.D());
break;
case 0b00000001:
mov(VTMP1.V1D(), Src1, 1);
pmull(Dst, VTMP1.V2D(), Src2);
dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), Src1.Q(), 1);
pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), VTMP1.D(), Src2.D());
break;
case 0b00010000:
mov(VTMP1.V1D(), Src2, 1);
pmull(Dst, VTMP1.V2D(), Src1);
dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), Src2.Q(), 1);
pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), VTMP1.D(), Src1.D());
break;
case 0b00010001:
pmull2(Dst, Src1, Src2);
pmull2(ARMEmitter::SubRegSize::i128Bit, Dst.Q(), Src1.Q(), Src2.Q());
break;
default:
LOGMAN_MSG_A_FMT("Unknown PCLMUL selector: {}", Op->Selector);
External/FEXCore/Source/Interface/Core/JIT/Arm64/FlagOps.cpp
Vendored
+1 -4
View File
@@ -7,13 +7,10 @@ $end_info$
#include "Interface/Core/JIT/Arm64/JITClass.h"
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(GetHostFlag) {
auto Op = IROp->C<IR::IROp_GetHostFlag>();
ubfx(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Value.ID()), Op->Flag, 1);
ubfx(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(Op->Value.ID()), Op->Flag, 1);
}
#undef DEF_OP
External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
Vendored
+164 -145
View File
@@ -11,6 +11,7 @@ $end_info$
*/
#include "Interface/Context/Context.h"
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/LookupCache.h"
#include "Interface/Core/ArchHelpers/Arm64.h"
@@ -77,9 +78,6 @@ static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
FallbackInfo Info;
if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
@@ -93,13 +91,13 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
uxth(w0, GetReg<RA_32>(IROp->Args[0].ID()));
ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
const auto Src1 = GetReg(IROp->Args[0].ID());
uxth(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1);
ldr(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<void, uint16_t>(x1);
GenerateIndirectRuntimeCall<void, uint16_t>(ARMEmitter::Reg::r1);
#else
blr(x1);
blr(ARMEmitter::Reg::r1);
#endif
PopDynamicRegsAndLR();
@@ -112,22 +110,23 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
SpillStaticRegs();
PushDynamicRegsAndLR(TMP1);
fmov(v0.S(), GetVReg(IROp->Args[0].ID()).S()) ;
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
const auto Src1 = GetVReg(IROp->Args[0].ID());
fmov(ARMEmitter::SReg::s0, Src1.S());
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<__uint128_t, float>(x0);
GenerateIndirectRuntimeCall<__uint128_t, float>(ARMEmitter::Reg::r0);
#else
blr(x0);
blr(ARMEmitter::Reg::r0);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B());
ins(GetVReg(Node).V2D(), 0, x0);
ins(GetVReg(Node).V8H(), 4, w1);
const auto Dst = GetVReg(Node);
eor(Dst.Q(), Dst.Q(), Dst.Q());
ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0);
ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1);
}
break;
@@ -136,21 +135,23 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
mov(v0.D(), GetVReg(IROp->Args[0].ID()).D());
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
const auto Src1 = GetVReg(IROp->Args[0].ID());
mov(ARMEmitter::DReg::d0, Src1.D());
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<__uint128_t, double>(x0);
GenerateIndirectRuntimeCall<__uint128_t, double>(ARMEmitter::Reg::r0);
#else
blr(x0);
blr(ARMEmitter::Reg::r0);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B());
ins(GetVReg(Node).V2D(), 0, x0);
ins(GetVReg(Node).V8H(), 4, w1);
const auto Dst = GetVReg(Node);
eor(Dst.Q(), Dst.Q(), Dst.Q());
ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0);
ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1);
}
break;
@@ -160,26 +161,28 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
const auto Src1 = GetReg(IROp->Args[0].ID());
if (Info.ABI == FABI_F80_I16) {
uxth(w0, GetReg<RA_32>(IROp->Args[0].ID()));
uxth(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1);
}
else {
mov(w0, GetReg<RA_32>(IROp->Args[0].ID()));
mov(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1);
}
ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
ldr(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<__uint128_t, uint32_t>(x1);
GenerateIndirectRuntimeCall<__uint128_t, uint32_t>(ARMEmitter::Reg::r1);
#else
blr(x1);
blr(ARMEmitter::Reg::r1);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B());
ins(GetVReg(Node).V2D(), 0, x0);
ins(GetVReg(Node).V8H(), 4, w1);
const auto Dst = GetVReg(Node);
eor(Dst.Q(), Dst.Q(), Dst.Q());
ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0);
ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1);
}
break;
@@ -188,21 +191,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<float, uint64_t, uint64_t>(x2);
GenerateIndirectRuntimeCall<float, uint64_t, uint64_t>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
fmov(GetVReg(Node).S(), v0.S());
const auto Dst = GetVReg(Node);
fmov(Dst.S(), ARMEmitter::SReg::s0);
}
break;
@@ -211,21 +217,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<double, uint64_t, uint64_t>(x2);
GenerateIndirectRuntimeCall<double, uint64_t, uint64_t>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
mov(GetVReg(Node).D(), v0.D());
const auto Dst = GetVReg(Node);
mov(Dst.D(), ARMEmitter::DReg::d0);
}
break;
@@ -234,20 +243,22 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
mov(v0.D(), GetVReg(IROp->Args[0].ID()).D());
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
const auto Src1 = GetVReg(IROp->Args[0].ID());
mov(ARMEmitter::DReg::d0, Src1.D());
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<double, double>(x0);
GenerateIndirectRuntimeCall<double, double>(ARMEmitter::Reg::r0);
#else
blr(x0);
blr(ARMEmitter::Reg::r0);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
mov(GetVReg(Node).D(), v0.D());
const auto Dst = GetVReg(Node);
mov(Dst.D(), ARMEmitter::DReg::d0);
}
break;
@@ -256,21 +267,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
mov(v0.D(), GetVReg(IROp->Args[0].ID()).D());
mov(v1.D(), GetVReg(IROp->Args[1].ID()).D());
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
const auto Src1 = GetVReg(IROp->Args[0].ID());
const auto Src2 = GetVReg(IROp->Args[1].ID());
mov(ARMEmitter::DReg::d0, Src1.D());
mov(ARMEmitter::DReg::d1, Src2.D());
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<double, double, double>(x0);
GenerateIndirectRuntimeCall<double, double, double>(ARMEmitter::Reg::r0);
#else
blr(x0);
blr(ARMEmitter::Reg::r0);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
mov(GetVReg(Node).D(), v0.D());
const auto Dst = GetVReg(Node);
mov(Dst.D(), ARMEmitter::DReg::d0);
}
break;
@@ -279,21 +293,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint32_t, uint64_t, uint64_t>(x2);
GenerateIndirectRuntimeCall<uint32_t, uint64_t, uint64_t>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
uxth(GetReg<RA_64>(Node), x0);
const auto Dst = GetReg(Node);
uxth(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0);
}
break;
case FABI_I32_F80:{
@@ -301,21 +318,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint32_t, uint64_t, uint64_t>(x2);
GenerateIndirectRuntimeCall<uint32_t, uint64_t, uint64_t>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
mov(GetReg<RA_32>(Node), w0);
const auto Dst = GetReg(Node);
mov(ARMEmitter::Size::i32Bit, Dst, ARMEmitter::Reg::r0);
}
break;
case FABI_I64_F80:{
@@ -323,21 +343,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t>(x2);
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
mov(GetReg<RA_64>(Node), x0);
const auto Dst = GetReg(Node);
mov(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0);
}
break;
case FABI_I64_F80_F80:{
@@ -345,23 +368,27 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
const auto Src2 = GetVReg(IROp->Args[1].ID());
umov(x2, GetVReg(IROp->Args[1].ID()).V2D(), 0);
umov(w3, GetVReg(IROp->Args[1].ID()).V8H(), 4);
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r2, Src2, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r3, Src2, 4);
ldr(ARMEmitter::XReg::x4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t, uint64_t>(x4);
GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r4);
#else
blr(x4);
blr(ARMEmitter::Reg::r4);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
mov(GetReg<RA_64>(Node), x0);
const auto Dst = GetReg(Node);
mov(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0);
}
break;
case FABI_F80_F80:{
@@ -369,23 +396,26 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t>(x2);
GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t>(ARMEmitter::Reg::r2);
#else
blr(x2);
blr(ARMEmitter::Reg::r2);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B());
ins(GetVReg(Node).V2D(), 0, x0);
ins(GetVReg(Node).V8H(), 4, w1);
const auto Dst = GetVReg(Node);
eor(Dst.Q(), Dst.Q(), Dst.Q());
ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0);
ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1);
}
break;
case FABI_F80_F80_F80:{
@@ -393,29 +423,32 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
PushDynamicRegsAndLR(TMP1);
umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0);
umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4);
const auto Src1 = GetVReg(IROp->Args[0].ID());
const auto Src2 = GetVReg(IROp->Args[1].ID());
umov(x2, GetVReg(IROp->Args[1].ID()).V2D(), 0);
umov(w3, GetVReg(IROp->Args[1].ID()).V8H(), 4);
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r0, Src1, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r1, Src1, 4);
ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
umov<ARMEmitter::SubRegSize::i64Bit>(ARMEmitter::Reg::r2, Src2, 0);
umov<ARMEmitter::SubRegSize::i16Bit>(ARMEmitter::Reg::r3, Src2, 4);
ldr(ARMEmitter::XReg::x4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]));
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t, uint64_t>(x4);
GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r4);
#else
blr(x4);
blr(ARMEmitter::Reg::r4);
#endif
PopDynamicRegsAndLR();
FillStaticRegs();
eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B());
ins(GetVReg(Node).V2D(), 0, x0);
ins(GetVReg(Node).V8H(), 4, w1);
const auto Dst = GetVReg(Node);
eor(Dst.Q(), Dst.Q(), Dst.Q());
ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0);
ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1);
}
break;
case FABI_UNKNOWN:
default:
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
@@ -435,7 +468,6 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram
auto HostCode = Thread->LookupCache->FindBlock(GuestRip);
if (!HostCode) {
//fmt::print("ExitFunctionLink: Aborting, {:X} not in cache\n", GuestRip);
Frame->State.rip = GuestRip;
return Frame->Pointers.Common.DispatcherLoopTop;
}
@@ -444,25 +476,22 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram
auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker;
auto offset = HostCode/4 - branch/4;
if (IsInt26(offset)) {
if (vixl::IsInt26(offset)) {
// optimal case - can branch directly
// patch the code
vixl::aarch64::Assembler emit((uint8_t*)(branch), 24);
vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 24);
emit.b(offset);
emit.FinalizeCode();
vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24);
FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 24);
// Add de-linking handler
Context::Context::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [branch, LinkerAddress]{
vixl::aarch64::Assembler emit((uint8_t*)(branch), 24);
vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
Literal l_BranchHost{LinkerAddress};
emit.ldr(x0, &l_BranchHost);
emit.blr(x0);
emit.place(&l_BranchHost);
emit.FinalizeCode();
vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24);
FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 24);
FEXCore::ARMEmitter::ForwardLabel l_BranchHost;
emit.ldr(FEXCore::ARMEmitter::XReg::x0, &l_BranchHost);
emit.blr(FEXCore::ARMEmitter::Reg::r0);
emit.Bind(&l_BranchHost);
emit.dc64(LinkerAddress);
FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 24);
});
} else {
// fallback case - do a soft-er link by patching the pointer
@@ -488,10 +517,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
RAPass = Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA");
#if DEBUG
Decoder.AppendVisitor(&Disasm)
#endif
uint32_t NumUsedGPRs = NumGPRs;
uint32_t NumUsedGPRPairs = NumGPRPairs;
uint32_t UsedRegisterCount = RegisterCount;
@@ -559,7 +584,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
}
// Must be done after Dispatcher init
SetAllowAssembler(true);
ClearCache();
}
@@ -582,16 +606,15 @@ void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
void Arm64JITCore::EmitDetectionString() {
const char JITString[] = "FEXJIT::Arm64JITCore::";
auto Buffer = GetBuffer();
Buffer->EmitString(JITString);
Buffer->Align();
EmitString(JITString);
Align();
}
void Arm64JITCore::ClearCache() {
// Get the backing code buffer
auto CodeBuffer = GetEmptyCodeBuffer();
*GetBuffer() = vixl::CodeBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
SetBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
EmitDetectionString();
}
@@ -636,7 +659,6 @@ FEXCore::IR::RegisterClassType Arm64JITCore::GetRegClass(IR::NodeID Node) const
return FEXCore::IR::RegisterClassType {GetPhys(Node).Class};
}
bool Arm64JITCore::IsFPR(IR::NodeID Node) const {
auto Class = GetRegClass(Node);
@@ -656,7 +678,6 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
bool GDBEnabled) {
FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
using namespace aarch64;
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
@@ -665,12 +686,12 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
this->DebugData = DebugData;
#ifdef VIXL_DISASSEMBLER
const auto DisasmBegin = GetCursorAddress<const Instruction*>();
const auto DisasmBegin = GetCursorAddress<const vixl::aarch64::Instruction*>();
#endif
#ifndef NDEBUG
LoadConstant(x0, Entry);
#endif
#ifndef NDEBUG
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, Entry);
#endif
this->IR = IR;
@@ -704,7 +725,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
if (GDBEnabled) {
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(GuestEntry, Entry);
GetBuffer()->CursorForward(GDBSize);
CursorIncrement(GDBSize);
}
//LOGMAN_THROW_A_FMT(RAData->HasFullRA(), "Arm64 JIT only works with RA");
@@ -714,11 +735,11 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
if (SpillSlots) {
const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
if (IsImmAddSub(TotalSpillSlotsSize)) {
sub(sp, sp, TotalSpillSlotsSize);
if (vixl::aarch64::Assembler::IsImmAddSub(TotalSpillSlotsSize)) {
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
} else {
LoadConstant(x0, TotalSpillSlotsSize);
sub(sp, sp, x0);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, TotalSpillSlotsSize);
sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, ARMEmitter::XReg::x0, ARMEmitter::ExtendedType::LSL_64, 0);
}
}
@@ -743,7 +764,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
}
PendingTargetLabel = nullptr;
bind(&IsTarget->second);
Bind(&IsTarget->second);
}
for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
@@ -769,13 +790,11 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
}
PendingTargetLabel = nullptr;
FinalizeCode();
auto CodeEnd = GetCursorAddress<uint8_t *>();
CPU.EnsureIAndDCacheCoherency(GuestEntry, CodeEnd - GuestEntry);
ClearICache(GuestEntry, CodeEnd - GuestEntry);
#ifdef VIXL_DISASSEMBLER
const auto DisasmEnd = GetCursorAddress<const Instruction*>();
const auto DisasmEnd = GetCursorAddress<const vixl::aarch64::Instruction*>();
Disasm.DisassembleBuffer(DisasmBegin, DisasmEnd);
#endif
@@ -796,12 +815,12 @@ void Arm64JITCore::ResetStack() {
const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
if (IsImmAddSub(TotalSpillSlotsSize)) {
add(sp, sp, TotalSpillSlotsSize);
if (vixl::aarch64::Assembler::IsImmAddSub(TotalSpillSlotsSize)) {
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
} else {
// Too big to fit in a 12bit immediate
LoadConstant(x0, TotalSpillSlotsSize);
add(sp, sp, x0);
// Too big to fit in a 12bit immediate
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, TotalSpillSlotsSize);
add(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, ARMEmitter::XReg::x0, ARMEmitter::ExtendedType::LSL_64, 0);
}
}
External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
Vendored
+25 -44
View File
@@ -8,6 +8,7 @@ $end_info$
#include <FEXCore/IR/RegisterAllocationData.h>
#include "Interface/Core/ArchHelpers/Arm64Emitter.h"
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include <aarch64/assembler-aarch64.h>
@@ -28,9 +29,6 @@ namespace FEXCore::Core {
}
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
class Arm64JITCore final : public CPUBackend, public Arm64Emitter {
public:
explicit Arm64JITCore(FEXCore::Context::Context *ctx,
@@ -58,12 +56,12 @@ private:
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
const bool HostSupportsSVE{};
Label *PendingTargetLabel;
ARMEmitter::BiDirectionalLabel *PendingTargetLabel;
FEXCore::Context::Context *CTX;
FEXCore::IR::IRListView const *IR;
uint64_t Entry;
std::map<IR::NodeID, aarch64::Label> JumpTargets;
std::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> JumpTargets;
/**
* @name Register Allocation
@@ -86,22 +84,8 @@ private:
constexpr static uint8_t RA_64 = 1;
constexpr static uint8_t RA_FPR = 2;
template<uint8_t RAType>
[[nodiscard]] aarch64::Register GetReg(IR::NodeID Node) const;
template<uint8_t RAType>
[[nodiscard]] std::pair<aarch64::Register, aarch64::Register> GetRegPair(IR::NodeID Node) const;
[[nodiscard]] FEXCore::IR::RegisterClassType GetRegClass(IR::NodeID Node) const;
template<>
[[nodiscard]] aarch64::Register GetReg<Arm64JITCore::RA_32>(IR::NodeID Node) const {
return GetReg<Arm64JITCore::RA_64>(Node).W();
}
template<>
[[nodiscard]] aarch64::Register GetReg<Arm64JITCore::RA_64>(IR::NodeID Node) const {
auto Reg = GetPhys(Node);
[[nodiscard]] FEXCore::ARMEmitter::Register GetReg(IR::NodeID Node) const {
const auto Reg = GetPhys(Node);
LOGMAN_THROW_AA_FMT(Reg.Class == IR::GPRFixedClass.Val || Reg.Class == IR::GPRClass.Val, "Unexpected Class: {}", Reg.Class);
@@ -114,22 +98,8 @@ private:
FEX_UNREACHABLE;
}
template<>
[[nodiscard]] std::pair<aarch64::Register, aarch64::Register> GetRegPair<Arm64JITCore::RA_32>(IR::NodeID Node) const {
uint32_t Reg = GetPhys(Node).Reg;
auto Pair = RA64Pair[Reg];
return std::make_pair(Pair.first.W(), Pair.second.W());
}
template<>
[[nodiscard]] std::pair<aarch64::Register, aarch64::Register> GetRegPair<Arm64JITCore::RA_64>(IR::NodeID Node) const {
uint32_t Reg = GetPhys(Node).Reg;
return RA64Pair[Reg];
}
[[nodiscard]] aarch64::VRegister GetVReg(IR::NodeID Node) const {
auto Reg = GetPhys(Node);
[[nodiscard]] FEXCore::ARMEmitter::VRegister GetVReg(IR::NodeID Node) const {
const auto Reg = GetPhys(Node);
LOGMAN_THROW_AA_FMT(Reg.Class == IR::FPRFixedClass.Val || Reg.Class == IR::FPRClass.Val, "Unexpected Class: {}", Reg.Class);
@@ -142,6 +112,16 @@ private:
FEX_UNREACHABLE;
}
[[nodiscard]] std::pair<FEXCore::ARMEmitter::Register, FEXCore::ARMEmitter::Register> GetRegPair(IR::NodeID Node) const {
const auto Reg = GetPhys(Node);
LOGMAN_THROW_AA_FMT(Reg.Class == IR::GPRPairClass.Val, "Unexpected Class: {}", Reg.Class);
return RA64Pair[Reg.Reg];
}
[[nodiscard]] FEXCore::IR::RegisterClassType GetRegClass(IR::NodeID Node) const;
[[nodiscard]] IR::PhysicalRegister GetPhys(IR::NodeID Node) const {
auto PhyReg = RAData->GetNodeRegister(Node);
@@ -153,8 +133,8 @@ private:
[[nodiscard]] bool IsFPR(IR::NodeID Node) const;
[[nodiscard]] bool IsGPR(IR::NodeID Node) const;
[[nodiscard]] MemOperand GenerateMemOperand(uint8_t AccessSize,
aarch64::Register Base,
[[nodiscard]] FEXCore::ARMEmitter::ExtendedMemOperand GenerateMemOperand(uint8_t AccessSize,
FEXCore::ARMEmitter::Register Base,
IR::OrderedNodeWrapper Offset,
IR::MemOffsetType OffsetType,
uint8_t OffsetScale);
@@ -164,8 +144,8 @@ private:
//
// TMP1 is safe to use again once this memory operand is used with its
// equivalent loads or stores that this was called for.
[[nodiscard]] SVEMemOperand GenerateSVEMemOperand(uint8_t AccessSize,
aarch64::Register Base,
[[nodiscard]] FEXCore::ARMEmitter::SVEMemOperand GenerateSVEMemOperand(uint8_t AccessSize,
FEXCore::ARMEmitter::Register Base,
IR::OrderedNodeWrapper Offset,
IR::MemOffsetType OffsetType,
uint8_t OffsetScale);
@@ -203,7 +183,8 @@ private:
* @brief A literal pair relocation object for named symbol literals
*/
struct NamedSymbolLiteralPair {
Literal<uint64_t> Lit;
ARMEmitter::ForwardLabel Loc;
uint64_t Lit;
Relocation MoveABI{};
};
@@ -213,7 +194,7 @@ private:
* @param Reg - The GPR to move the thunk handler in to
* @param Sum - The hash of the thunk
*/
void InsertNamedThunkRelocation(vixl::aarch64::Register Reg, const IR::SHA256Sum &Sum);
void InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR::SHA256Sum &Sum);
/**
* @brief Inserts a guest GPR move relocation
@@ -221,7 +202,7 @@ private:
* @param Reg - The GPR to move the guest RIP in to
* @param Constant - The guest RIP that will be relocated
*/
void InsertGuestRIPMove(vixl::aarch64::Register Reg, uint64_t Constant);
void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant);
/**
* @brief Inserts a named symbol as a literal in memory
External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp
Vendored
+355 -345
View File
File diff suppressed because it is too large. Load diff
External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp
Vendored
+62 -65
View File
@@ -5,12 +5,11 @@ $end_info$
*/
#include <syscall.h>
#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h"
#include "Interface/Core/JIT/Arm64/JITClass.h"
#include "FEXCore/Debug/InternalThreadState.h"
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(GuestOpcode) {
@@ -23,13 +22,13 @@ DEF_OP(Fence) {
auto Op = IROp->C<IR::IROp_Fence>();
switch (Op->Fence) {
case IR::Fence_Load.Val:
dmb(FullSystem, BarrierReads);
dmb(FEXCore::ARMEmitter::BarrierScope::LD);
break;
case IR::Fence_LoadStore.Val:
dmb(FullSystem, BarrierAll);
dmb(FEXCore::ARMEmitter::BarrierScope::SY);
break;
case IR::Fence_Store.Val:
dmb(FullSystem, BarrierWrites);
dmb(FEXCore::ARMEmitter::BarrierScope::ST);
break;
default: LOGMAN_MSG_A_FMT("Unknown Fence: {}", Op->Fence); break;
}
@@ -52,88 +51,88 @@ DEF_OP(Break) {
uint64_t Constant{};
memcpy(&Constant, &State, sizeof(State));
LoadConstant(x1, Constant);
str(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData)));
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Constant);
str(ARMEmitter::XReg::x1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
switch (Op->Reason.Signal) {
case SIGILL:
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGILL)));
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGILL));
br(TMP1);
break;
case SIGTRAP:
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)));
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
br(TMP1);
break;
case SIGSEGV:
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGSEGV)));
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGSEGV));
br(TMP1);
break;
default:
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)));
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
br(TMP1);
break;
}
}
DEF_OP(GetRoundingMode) {
auto Dst = GetReg<RA_64>(Node);
mrs(Dst, FPCR);
lsr(Dst, Dst, 22);
auto Dst = GetReg(Node);
mrs(Dst, ARMEmitter::SystemRegister::FPCR);
lsr(ARMEmitter::Size::i64Bit, Dst, Dst, 22);
// FTZ is already in the correct location
// Rounding mode is different
and_(TMP1, Dst, 0b11);
and_(ARMEmitter::Size::i64Bit, TMP1, Dst, 0b11);
cmp(TMP1, 1);
LoadConstant(TMP3, IR::ROUND_MODE_POSITIVE_INFINITY);
csel(TMP2, TMP3, xzr, vixl::aarch64::Condition::eq);
cmp(ARMEmitter::Size::i64Bit, TMP1, 1);
LoadConstant(ARMEmitter::Size::i64Bit, TMP3, IR::ROUND_MODE_POSITIVE_INFINITY);
csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, ARMEmitter::Reg::zr, ARMEmitter::Condition::CC_EQ);
cmp(TMP1, 2);
LoadConstant(TMP3, IR::ROUND_MODE_NEGATIVE_INFINITY);
csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq);
cmp(ARMEmitter::Size::i64Bit, TMP1, 2);
LoadConstant(ARMEmitter::Size::i64Bit, TMP3, IR::ROUND_MODE_NEGATIVE_INFINITY);
csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ);
cmp(TMP1, 3);
LoadConstant(TMP3, IR::ROUND_MODE_TOWARDS_ZERO);
csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq);
cmp(ARMEmitter::Size::i64Bit, TMP1, 3);
LoadConstant(ARMEmitter::Size::i64Bit, TMP3, IR::ROUND_MODE_TOWARDS_ZERO);
csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ);
orr(Dst, Dst, TMP2);
orr(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2.R());
bfi(Dst, TMP2, 0, 2);
bfi(ARMEmitter::Size::i64Bit, Dst, TMP2, 0, 2);
}
DEF_OP(SetRoundingMode) {
auto Op = IROp->C<IR::IROp_SetRoundingMode>();
auto Src = GetReg<RA_64>(Op->RoundMode.ID());
auto Src = GetReg(Op->RoundMode.ID());
// Setup the rounding flags correctly
and_(TMP1, Src, 0b11);
and_(ARMEmitter::Size::i64Bit, TMP1, Src, 0b11);
cmp(TMP1, IR::ROUND_MODE_POSITIVE_INFINITY);
LoadConstant(TMP3, 1);
csel(TMP2, TMP3, xzr, vixl::aarch64::Condition::eq);
cmp(ARMEmitter::Size::i64Bit, TMP1, IR::ROUND_MODE_POSITIVE_INFINITY);
LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 1);
csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, ARMEmitter::Reg::zr, ARMEmitter::Condition::CC_EQ);
cmp(TMP1, IR::ROUND_MODE_NEGATIVE_INFINITY);
LoadConstant(TMP3, 2);
csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq);
cmp(ARMEmitter::Size::i64Bit, TMP1, IR::ROUND_MODE_NEGATIVE_INFINITY);
LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 2);
csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ);
cmp(TMP1, IR::ROUND_MODE_TOWARDS_ZERO);
LoadConstant(TMP3, 3);
csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq);
cmp(ARMEmitter::Size::i64Bit, TMP1, IR::ROUND_MODE_TOWARDS_ZERO);
LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 3);
csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ);
mrs(TMP1, FPCR);
mrs(TMP1, ARMEmitter::SystemRegister::FPCR);
// vixl simulator doesn't support anything beyond ties-to-even rounding
#ifndef VIXL_SIMULATOR
// Insert the rounding flags
bfi(TMP1, TMP2, 22, 2);
bfi(ARMEmitter::Size::i64Bit, TMP1, TMP2, 22, 2);
#endif
// Insert the FTZ flag
lsr(TMP2, Src, 2);
bfi(TMP1, TMP2, 24, 1);
lsr(ARMEmitter::Size::i64Bit, TMP2, Src, 2);
bfi(ARMEmitter::Size::i64Bit, TMP1, TMP2, 24, 1);
// Now save the new FPCR
msr(FPCR, TMP1);
msr(ARMEmitter::SystemRegister::FPCR, TMP1);
}
DEF_OP(Print) {
@@ -143,17 +142,16 @@ DEF_OP(Print) {
SpillStaticRegs();
if (IsGPR(Op->Value.ID())) {
mov(x0, GetReg<RA_64>(Op->Value.ID()));
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue)));
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->Value.ID()));
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue));
}
else {
fmov(x0, GetVReg(Op->Value.ID()).V1D());
// Bug in vixl that source vector needs to b V1D rather than V2D?
fmov(x1, GetVReg(Op->Value.ID()).V1D(), 1);
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue)));
fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetVReg(Op->Value.ID()), false);
fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetVReg(Op->Value.ID()), true);
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue));
}
blr(x3);
blr(ARMEmitter::Reg::r3);
FillStaticRegs();
PopDynamicRegsAndLR();
@@ -173,27 +171,27 @@ DEF_OP(ProcessorID) {
// 16bit LoadConstant to be a single instruction
// We must always spill at least one register (x8) so this value always has a bit set
// This gives the signal handler a value to check to see if we are in a syscall at all
LoadConstant(x0, SpillMask & 0xFFFF);
str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)));
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SpillMask & 0xFFFF);
str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
// Allocate some temporary space for storing the uint32_t CPU and Node IDs
sub(sp, sp, 16);
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16);
// Load the getcpu syscall number
LoadConstant(x8, SYS_getcpu);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_getcpu);
// CPU pointer in x0
add(x0, sp, 0);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0);
// Node in x1
add(x1, sp, 4);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 4);
svc(0);
// On updated signal mask we can receive a signal RIGHT HERE
// Load the values returned by the kernel
ldp(w0, w1, MemOperand(sp));
ldp<ARMEmitter::IndexType::OFFSET>(ARMEmitter::WReg::w0, ARMEmitter::WReg::w1, ARMEmitter::Reg::rsp);
// Deallocate stack space
sub(sp, sp, 16);
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16);
// Now that we are done in the syscall we need to carefully peel back the state
// First unspill the registers from before
@@ -201,14 +199,13 @@ DEF_OP(ProcessorID) {
// Now the registers we've spilled are back in their original host registers
// We can safely claim we are no longer in a syscall
str(xzr, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)));
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
// Now store the result in the destination in the expected format
// uint32_t Res = (node << 12) | cpu;
// CPU is in w0
// Node is in w1
orr(GetReg<RA_64>(Node), x0, Operand(x1, LSL, 12));
orr(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0, ARMEmitter::Reg::r1, ARMEmitter::ShiftType::LSL, 12);
}
DEF_OP(RDRAND) {
@@ -216,21 +213,21 @@ DEF_OP(RDRAND) {
// Results are in x0, x1
// Results want to be in a i64v2 vector
auto Dst = GetRegPair<RA_64>(Node);
auto Dst = GetRegPair(Node);
if (Op->GetReseeded) {
mrs(Dst.first, RNDRRS);
mrs(Dst.first, ARMEmitter::SystemRegister::RNDRRS);
}
else {
mrs(Dst.first, RNDR);
mrs(Dst.first, ARMEmitter::SystemRegister::RNDR);
}
// If the rng number is valid then NZCV is 0b0000, otherwise NZCV is 0b0100
cset(Dst.second, Condition::ne);
cset(ARMEmitter::Size::i64Bit, Dst.second, ARMEmitter::Condition::CC_NE);
}
DEF_OP(Yield) {
hint(SystemHint::YIELD);
yield();
}
#undef DEF_OP
External/FEXCore/Source/Interface/Core/JIT/Arm64/MoveOps.cpp
Vendored
+21 -48
View File
@@ -7,64 +7,37 @@ $end_info$
#include "Interface/Core/JIT/Arm64/JITClass.h"
namespace FEXCore::CPU {
using namespace vixl;
using namespace vixl::aarch64;
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(ExtractElementPair) {
auto Op = IROp->C<IR::IROp_ExtractElementPair>();
switch (Op->Header.Size) {
case 4: {
auto Src = GetRegPair<RA_32>(Op->Pair.ID());
std::array<aarch64::Register, 2> Regs = {Src.first, Src.second};
mov (GetReg<RA_32>(Node), Regs[Op->Element]);
break;
}
case 8: {
auto Src = GetRegPair<RA_64>(Op->Pair.ID());
std::array<aarch64::Register, 2> Regs = {Src.first, Src.second};
mov (GetReg<RA_64>(Node), Regs[Op->Element]);
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Size"); break;
}
LOGMAN_THROW_AA_FMT(Op->Header.Size == 4 || Op->Header.Size == 8, "Invalid size");
const auto EmitSize = Op->Header.Size == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
const auto Src = GetRegPair(Op->Pair.ID());
const std::array<ARMEmitter::Register, 2> Regs = {Src.first, Src.second};
mov(EmitSize, GetReg(Node), Regs[Op->Element]);
}
DEF_OP(CreateElementPair) {
auto Op = IROp->C<IR::IROp_CreateElementPair>();
std::pair<aarch64::Register, aarch64::Register> Dst;
aarch64::Register RegFirst;
aarch64::Register RegSecond;
aarch64::Register RegTmp;
LOGMAN_THROW_AA_FMT(IROp->ElementSize == 4 || IROp->ElementSize == 8, "Invalid size");
std::pair<ARMEmitter::Register, ARMEmitter::Register> Dst = GetRegPair(Node);
ARMEmitter::Register RegFirst = GetReg(Op->Lower.ID());
ARMEmitter::Register RegSecond = GetReg(Op->Upper.ID());
ARMEmitter::Register RegTmp = TMP1.R();
switch (IROp->ElementSize) {
case 4: {
Dst = GetRegPair<RA_32>(Node);
RegFirst = GetReg<RA_32>(Op->Lower.ID());
RegSecond = GetReg<RA_32>(Op->Upper.ID());
RegTmp = w0;
break;
}
case 8: {
Dst = GetRegPair<RA_64>(Node);
RegFirst = GetReg<RA_64>(Op->Lower.ID());
RegSecond = GetReg<RA_64>(Op->Upper.ID());
RegTmp = x0;
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Size"); break;
}
const auto EmitSize = IROp->ElementSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
if (Dst.first.GetCode() != RegSecond.GetCode()) {
mov(Dst.first, RegFirst);
mov(Dst.second, RegSecond);
} else if (Dst.second.GetCode() != RegFirst.GetCode()) {
mov(Dst.second, RegSecond);
mov(Dst.first, RegFirst);
if (Dst.first.Idx() != RegSecond.Idx()) {
mov(EmitSize, Dst.first, RegFirst);
mov(EmitSize, Dst.second, RegSecond);
} else if (Dst.second.Idx() != RegFirst.Idx()) {
mov(EmitSize, Dst.second, RegSecond);
mov(EmitSize, Dst.first, RegFirst);
} else {
mov(RegTmp, RegFirst);
mov(Dst.second, RegSecond);
mov(Dst.first, RegTmp);
mov(EmitSize, RegTmp, RegFirst);
mov(EmitSize, Dst.second, RegSecond);
mov(EmitSize, Dst.first, RegTmp);
}
}
External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp
Vendored
+960 -3233
View File
File diff suppressed because it is too large. Load diff