From ffd9bb547d2cc727376c891323ddfec9a3b42cc9 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 23 Dec 2022 22:14:30 -0800 Subject: [PATCH] Arm64: Convert ARM Emitter over to new emitter Not yet complete. Missing a full SVE implementation and needs testing/validation. --- .../Interface/Core/ArchHelpers/Arm64.cpp | 7 +- .../Core/ArchHelpers/Arm64Emitter.cpp | 365 +- .../Interface/Core/ArchHelpers/Arm64Emitter.h | 106 +- .../Core/Dispatcher/Arm64Dispatcher.cpp | 367 +- .../Core/Dispatcher/Arm64Dispatcher.h | 5 + .../Interface/Core/JIT/Arm64/ALUOps.cpp | 1262 +++-- .../Core/JIT/Arm64/Arm64Relocations.cpp | 32 +- .../Interface/Core/JIT/Arm64/AtomicOps.cpp | 1059 ++--- .../Interface/Core/JIT/Arm64/BranchOps.cpp | 317 +- .../Core/JIT/Arm64/ConversionOps.cpp | 412 +- .../Core/JIT/Arm64/EncryptionOps.cpp | 100 +- .../Interface/Core/JIT/Arm64/FlagOps.cpp | 5 +- .../Source/Interface/Core/JIT/Arm64/JIT.cpp | 309 +- .../Interface/Core/JIT/Arm64/JITClass.h | 69 +- .../Interface/Core/JIT/Arm64/MemoryOps.cpp | 700 +-- .../Interface/Core/JIT/Arm64/MiscOps.cpp | 127 +- .../Interface/Core/JIT/Arm64/MoveOps.cpp | 69 +- .../Interface/Core/JIT/Arm64/VectorOps.cpp | 4193 ++++------------- 18 files changed, 3208 insertions(+), 6296 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.cpp b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.cpp index 841455073..ea21e07f6 100644 --- a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.cpp +++ b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.cpp @@ -1,7 +1,6 @@ #include "Interface/Core/ArchHelpers/Arm64.h" #include "Interface/Core/ArchHelpers/MContext.h" - -#include +#include "Interface/Core/ArchHelpers/CodeEmitter/Buffer.h" #include #include @@ -572,7 +571,7 @@ bool HandleAtomicVectorStore(void *_ucontext, void *_info, uint32_t Instr) { PC[1] = STP; PC[2] = DMB; // Back up one instruction and have another go - vixl::aarch64::CPU::EnsureIAndDCacheCoherency(&PC[0], 16); + FEXCore::ARMEmitter::Buffer::ClearICache(&PC[0], 16); return true; } } @@ -2311,7 +2310,7 @@ bool HandleSIGBUS(bool ParanoidTSO, int Signal, void *info, void *ucontext) { return false; } - vixl::aarch64::CPU::EnsureIAndDCacheCoherency(&PC[-1], 16); + FEXCore::ARMEmitter::Buffer::ClearICache(&PC[-1], 16); return true; } return false; diff --git a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp index eb751dcab..5aa24e539 100644 --- a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp +++ b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp @@ -1,4 +1,5 @@ #include "Interface/Core/ArchHelpers/Arm64Emitter.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/Dispatcher/Dispatcher.h" #include "Interface/Context/Context.h" #include "Interface/HLE/Thunks/Thunks.h" @@ -20,38 +21,24 @@ namespace FEXCore::CPU { // We want vixl to not allocate a default buffer. Jit and dispatcher will manually create one. Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size) - : vixl::aarch64::Assembler(size ? (byte*)FEXCore::Allocator::mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) : reinterpret_cast(~0ULL), - size, - vixl::aarch64::PositionDependentCode) + : Emitter(size ? (uint8_t*)FEXCore::Allocator::mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) : nullptr, size) , EmitterCTX {ctx} { CPU.SetUp(); - -#ifdef VIXL_SIMULATOR - auto Features = vixl::CPUFeatures::All(); -#else - auto Features = vixl::CPUFeatures::InferFromOS(); - if (ctx->HostFeatures.SupportsAtomics) { - // Hypervisor can hide this on the c630? - Features.Combine(vixl::CPUFeatures::Feature::kLORegions); - } -#endif - - SetCPUFeatures(Features); } Arm64Emitter::~Arm64Emitter() { - auto CodeBuffer = GetBuffer(); - if (CodeBuffer->GetCapacity()) { - FEXCore::Allocator::munmap(CodeBuffer->GetStartAddress(), CodeBuffer->GetCapacity()); + auto BufferSize = GetBufferSize(); + if (BufferSize) { + FEXCore::Allocator::munmap(GetBufferBase(), BufferSize); } } -void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, bool NOPPad) { - bool Is64Bit = Reg.IsX(); +void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, bool NOPPad) { + bool Is64Bit = s == ARMEmitter::Size::i64Bit; int Segments = Is64Bit ? 4 : 2; if (Is64Bit && ((~Constant)>> 16) == 0) { - movn(Reg, (~Constant) & 0xFFFF); + movn(s, Reg, (~Constant) & 0xFFFF); if (NOPPad) { nop(); nop(); nop(); @@ -98,17 +85,17 @@ void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, else { // Need to use ADRP + ADD adrp(Reg, AlignedOffset >> 12); - add(Reg, Reg, Constant & 0xFFF); + add(s, Reg, Reg, Constant & 0xFFF); NumMoves = 2; } } } else { - movz(Reg, (Constant) & 0xFFFF, 0); + movz(s, Reg, (Constant) & 0xFFFF, 0); for (int i = 1; i < Segments; ++i) { uint16_t Part = (Constant >> (i * 16)) & 0xFFFF; if (Part) { - movk(Reg, Part, i * 16); + movk(s, Reg, Part, i * 16); ++NumMoves; } } @@ -124,140 +111,143 @@ void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, void Arm64Emitter::PushCalleeSavedRegisters() { // We need to save pairs of registers // We save r19-r30 - MemOperand PairOffset(sp, -16, PreIndex); - const std::array, 6> CalleeSaved = {{ - {x19, x20}, - {x21, x22}, - {x23, x24}, - {x25, x26}, - {x27, x28}, - {x29, x30}, + const std::array, 6> CalleeSaved = {{ + {ARMEmitter::XReg::x19, ARMEmitter::XReg::x20}, + {ARMEmitter::XReg::x21, ARMEmitter::XReg::x22}, + {ARMEmitter::XReg::x23, ARMEmitter::XReg::x24}, + {ARMEmitter::XReg::x25, ARMEmitter::XReg::x26}, + {ARMEmitter::XReg::x27, ARMEmitter::XReg::x28}, + {ARMEmitter::XReg::x29, ARMEmitter::XReg::x30}, }}; for (auto &RegPair : CalleeSaved) { - stp(RegPair.first, RegPair.second, PairOffset); + stp(RegPair.first, RegPair.second, ARMEmitter::Reg::rsp, -16); } // Additionally we need to store the lower 64bits of v8-v15 // Here's a fun thing, we can use two ST4 instructions to store everything // We just need a single sub to sp before that const std::array< - std::tuple, 2> FPRs = {{ - {v8, v9, v10, v11}, - {v12, v13, v14, v15}, + std::tuple, 2> FPRs = {{ + {ARMEmitter::DReg::d8, ARMEmitter::DReg::d9, ARMEmitter::DReg::d10, ARMEmitter::DReg::d11}, + {ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15}, }}; uint32_t VectorSaveSize = sizeof(uint64_t) * 8; - sub(sp, sp, VectorSaveSize); + sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, VectorSaveSize); // SP supporting move // We just saved x19 so it is safe - add(x19, sp, 0); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r19, ARMEmitter::Reg::rsp, 0); - MemOperand QuadOffset(x19, 32, PostIndex); for (auto &RegQuad : FPRs) { - st4(std::get<0>(RegQuad).D(), - std::get<1>(RegQuad).D(), - std::get<2>(RegQuad).D(), - std::get<3>(RegQuad).D(), + st4(ARMEmitter::SubRegSize::i64Bit, + std::get<0>(RegQuad), + std::get<1>(RegQuad), + std::get<2>(RegQuad), + std::get<3>(RegQuad), 0, - QuadOffset); + ARMEmitter::Reg::r19, + 32); } } void Arm64Emitter::PopCalleeSavedRegisters() { const std::array< - std::tuple, 2> FPRs = {{ - {v12, v13, v14, v15}, - {v8, v9, v10, v11}, + std::tuple, 2> FPRs = {{ + {ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15}, + {ARMEmitter::DReg::d8, ARMEmitter::DReg::d9, ARMEmitter::DReg::d10, ARMEmitter::DReg::d11}, }}; - MemOperand QuadOffset(sp, 32, PostIndex); for (auto &RegQuad : FPRs) { - ld4(std::get<0>(RegQuad).D(), - std::get<1>(RegQuad).D(), - std::get<2>(RegQuad).D(), - std::get<3>(RegQuad).D(), + ld4(ARMEmitter::SubRegSize::i64Bit, + std::get<0>(RegQuad), + std::get<1>(RegQuad), + std::get<2>(RegQuad), + std::get<3>(RegQuad), 0, - QuadOffset); + ARMEmitter::Reg::rsp, + 32); } - MemOperand PairOffset(sp, 16, PostIndex); - const std::array, 6> CalleeSaved = {{ - {x29, x30}, - {x27, x28}, - {x25, x26}, - {x23, x24}, - {x21, x22}, - {x19, x20}, + const std::array, 6> CalleeSaved = {{ + {ARMEmitter::XReg::x29, ARMEmitter::XReg::x30}, + {ARMEmitter::XReg::x27, ARMEmitter::XReg::x28}, + {ARMEmitter::XReg::x25, ARMEmitter::XReg::x26}, + {ARMEmitter::XReg::x23, ARMEmitter::XReg::x24}, + {ARMEmitter::XReg::x21, ARMEmitter::XReg::x22}, + {ARMEmitter::XReg::x19, ARMEmitter::XReg::x20}, }}; for (auto &RegPair : CalleeSaved) { - ldp(RegPair.first, RegPair.second, PairOffset); + ldp(RegPair.first, RegPair.second, ARMEmitter::Reg::rsp, 16); } } void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t GPRSpillMask, uint32_t FPRSpillMask) { - if (StaticRegisterAllocation()) { - for (size_t i = 0; i < SRA64.size(); i+=2) { - auto Reg1 = SRA64[i]; - auto Reg2 = SRA64[i+1]; - if (((1U << Reg1.GetCode()) & GPRSpillMask) && - ((1U << Reg2.GetCode()) & GPRSpillMask)) { - stp(Reg1, Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]))); - } - else if (((1U << Reg1.GetCode()) & GPRSpillMask)) { - str(Reg1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]))); - } - else if (((1U << Reg2.GetCode()) & GPRSpillMask)) { - str(Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1]))); - } + if (!StaticRegisterAllocation()) { + return; + } + + for (size_t i = 0; i < SRA64.size(); i+=2) { + auto Reg1 = SRA64[i]; + auto Reg2 = SRA64[i+1]; + if (((1U << Reg1.Idx()) & GPRSpillMask) && + ((1U << Reg2.Idx()) & GPRSpillMask)) { + stp(Reg1.X(), Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])); } + else if (((1U << Reg1.Idx()) & GPRSpillMask)) { + str(Reg1.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])); + } + else if (((1U << Reg2.Idx()) & GPRSpillMask)) { + str(Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1])); + } + } - if (FPRs) { - if (EmitterCTX->HostFeatures.SupportsAVX) { - for (size_t i = 0; i < SRAFPR.size(); i++) { - const auto Reg = SRAFPR[i]; + if (FPRs) { + if (EmitterCTX->HostFeatures.SupportsAVX) { + for (size_t i = 0; i < SRAFPR.size(); i++) { + const auto Reg = SRAFPR[i]; - if (((1U << Reg.GetCode()) & FPRSpillMask) != 0) { - mov(TMP4, offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0])); - st1b(Reg.Z().VnB(), PRED_TMP_32B, SVEMemOperand(STATE, TMP4)); - } + if (((1U << Reg.Idx()) & FPRSpillMask) != 0) { + mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0])); + st1b(Reg, PRED_TMP_32B, STATE.R(), TMP4.R()); } - } else { - if (GPRSpillMask && FPRSpillMask == ~0U) { - // Optimize the common case where we can spill four registers per instruction - auto TmpReg = SRA64[__builtin_ffs(GPRSpillMask)]; - // Load the sse offset in to the temporary register - add(TmpReg, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0])); - for (size_t i = 0; i < SRAFPR.size(); i += 4) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; - const auto Reg3 = SRAFPR[i + 2]; - const auto Reg4 = SRAFPR[i + 3]; - st1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(TmpReg, 64, PostIndex)); - } - } - else { - for (size_t i = 0; i < SRAFPR.size(); i += 2) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; + } + } else { + if (GPRSpillMask && FPRSpillMask == ~0U) { + // Optimize the common case where we can spill four registers per instruction + auto TmpReg = SRA64[__builtin_ffs(GPRSpillMask)]; - if (((1U << Reg1.GetCode()) & FPRSpillMask) && - ((1U << Reg2.GetCode()) & FPRSpillMask)) { - stp(Reg1.Q(), Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]))); - } - else if (((1U << Reg1.GetCode()) & FPRSpillMask)) { - str(Reg1.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]))); - } - else if (((1U << Reg2.GetCode()) & FPRSpillMask)) { - str(Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0]))); - } + // Load the sse offset in to the temporary register + add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0])); + for (size_t i = 0; i < SRAFPR.size(); i += 4) { + const auto Reg1 = SRAFPR[i]; + const auto Reg2 = SRAFPR[i + 1]; + const auto Reg3 = SRAFPR[i + 2]; + const auto Reg4 = SRAFPR[i + 3]; + st1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64); + } + } + else { + for (size_t i = 0; i < SRAFPR.size(); i += 2) { + const auto Reg1 = SRAFPR[i]; + const auto Reg2 = SRAFPR[i + 1]; + + if (((1U << Reg1.Idx()) & FPRSpillMask) && + ((1U << Reg2.Idx()) & FPRSpillMask)) { + stp(Reg1.Q(), Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])); + } + else if (((1U << Reg1.Idx()) & FPRSpillMask)) { + str(Reg1.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])); + } + else if (((1U << Reg2.Idx()) & FPRSpillMask)) { + str(Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0])); } } } @@ -266,77 +256,79 @@ void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t GPRSpillMask, uint32_t FP } void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRFillMask) { - if (StaticRegisterAllocation()) { - if (FPRs) { - if (EmitterCTX->HostFeatures.SupportsAVX) { - // Set up predicate registers. - // We don't bother spilling these in SpillStaticRegs, - // since all that matters is we restore them on a fill. - // It's not a concern if they get trounced by something else. - ptrue(PRED_TMP_16B.VnB(), SVE_VL16); - ptrue(PRED_TMP_32B.VnB(), SVE_VL32); + if (!StaticRegisterAllocation()) { + return; + } - for (size_t i = 0; i < SRAFPR.size(); i++) { - const auto Reg = SRAFPR[i]; + if (FPRs) { + if (EmitterCTX->HostFeatures.SupportsAVX) { + // Set up predicate registers. + // We don't bother spilling these in SpillStaticRegs, + // since all that matters is we restore them on a fill. + // It's not a concern if they get trounced by something else. + ptrue(PRED_TMP_16B, ARMEmitter::PredicatePattern::SVE_VL16); + ptrue(PRED_TMP_32B, ARMEmitter::PredicatePattern::SVE_VL32); - if (((1U << Reg.GetCode()) & FPRFillMask) != 0) { - mov(TMP4, offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0])); - ld1b(Reg.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(STATE, TMP4)); - } + for (size_t i = 0; i < SRAFPR.size(); i++) { + const auto Reg = SRAFPR[i]; + if (((1U << Reg.Idx()) & FPRFillMask) != 0) { + mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0])); + ld1b(Reg, PRED_TMP_32B, STATE.R(), TMP4.R()); } - } else { - if (GPRFillMask && FPRFillMask == ~0U) { - // Optimize the common case where we can fill four registers per instruction. - // Use one of the filling static registers before we fill it. - auto TmpReg = SRA64[__builtin_ffs(GPRFillMask)]; - // Load the sse offset in to the temporary register - add(TmpReg, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0])); - for (size_t i = 0; i < SRAFPR.size(); i += 4) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; - const auto Reg3 = SRAFPR[i + 2]; - const auto Reg4 = SRAFPR[i + 3]; - ld1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(TmpReg, 64, PostIndex)); - } - } - else { - for (size_t i = 0; i < SRAFPR.size(); i += 2) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; + } + } else { + if (GPRFillMask && FPRFillMask == ~0U) { + // Optimize the common case where we can fill four registers per instruction. + // Use one of the filling static registers before we fill it. + auto TmpReg = SRA64[__builtin_ffs(GPRFillMask)]; - if (((1U << Reg1.GetCode()) & FPRFillMask) && - ((1U << Reg2.GetCode()) & FPRFillMask)) { - ldp(Reg1.Q(), Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]))); - } - else if (((1U << Reg1.GetCode()) & FPRFillMask)) { - ldr(Reg1.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0]))); - } - else if (((1U << Reg2.GetCode()) & FPRFillMask)) { - ldr(Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0]))); - } + // Load the sse offset in to the temporary register + add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0])); + for (size_t i = 0; i < SRAFPR.size(); i += 4) { + const auto Reg1 = SRAFPR[i]; + const auto Reg2 = SRAFPR[i + 1]; + const auto Reg3 = SRAFPR[i + 2]; + const auto Reg4 = SRAFPR[i + 3]; + ld1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64); + } + } + else { + for (size_t i = 0; i < SRAFPR.size(); i += 2) { + const auto Reg1 = SRAFPR[i]; + const auto Reg2 = SRAFPR[i + 1]; + + if (((1U << Reg1.Idx()) & FPRFillMask) && + ((1U << Reg2.Idx()) & FPRFillMask)) { + ldp(Reg1.Q(), Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])); + } + else if (((1U << Reg1.Idx()) & FPRFillMask)) { + ldr(Reg1.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i][0])); + } + else if (((1U << Reg2.Idx()) & FPRFillMask)) { + ldr(Reg2.Q(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[i+1][0])); } } } } + } - for (size_t i = 0; i < SRA64.size(); i+=2) { - auto Reg1 = SRA64[i]; - auto Reg2 = SRA64[i+1]; - if (((1U << Reg1.GetCode()) & GPRFillMask) && - ((1U << Reg2.GetCode()) & GPRFillMask)) { - ldp(Reg1, Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]))); - } - else if (((1U << Reg1.GetCode()) & GPRFillMask)) { - ldr(Reg1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i]))); - } - else if (((1U << Reg2.GetCode()) & GPRFillMask)) { - ldr(Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1]))); - } + for (size_t i = 0; i < SRA64.size(); i+=2) { + auto Reg1 = SRA64[i]; + auto Reg2 = SRA64[i+1]; + if (((1U << Reg1.Idx()) & GPRFillMask) && + ((1U << Reg2.Idx()) & GPRFillMask)) { + ldp(Reg1.X(), Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])); + } + else if ((1U << Reg1.Idx()) & GPRFillMask) { + ldr(Reg1.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])); + } + else if ((1U << Reg2.Idx()) & GPRFillMask) { + ldr(Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1])); } } } -void Arm64Emitter::PushDynamicRegsAndLR(aarch64::Register TmpReg) { +void Arm64Emitter::PushDynamicRegsAndLR(FEXCore::ARMEmitter::Register TmpReg) { const auto CanUseSVE = EmitterCTX->HostFeatures.SupportsAVX; const auto GPRSize = 1 * Core::CPUState::GPR_REG_SIZE; const auto FPRRegSize = CanUseSVE ? Core::CPUState::XMM_AVX_REG_SIZE @@ -344,10 +336,10 @@ void Arm64Emitter::PushDynamicRegsAndLR(aarch64::Register TmpReg) { const auto FPRSize = RAFPR.size() * FPRRegSize; const uint64_t SPOffset = AlignUp(GPRSize + FPRSize, 16); - sub(sp, sp, SPOffset); + sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset); // rsp capable move - add(TmpReg, aarch64::sp, 0); + add(ARMEmitter::Size::i64Bit, TmpReg, ARMEmitter::Reg::rsp, 0); if (CanUseSVE) { for (size_t i = 0; i < RAFPR.size(); i += 4) { @@ -355,20 +347,21 @@ void Arm64Emitter::PushDynamicRegsAndLR(aarch64::Register TmpReg) { const auto Reg2 = RAFPR[i + 1]; const auto Reg3 = RAFPR[i + 2]; const auto Reg4 = RAFPR[i + 3]; - st4b(Reg1.Z().VnB(), Reg2.Z().VnB(), Reg3.Z().VnB(), Reg4.Z().VnB(), PRED_TMP_32B, SVEMemOperand(TmpReg)); - add(TmpReg, TmpReg, 32 * 4); + st4b(Reg1, Reg2, Reg3, Reg4, PRED_TMP_32B, TmpReg, 0); + add(ARMEmitter::Size::i64Bit, TmpReg, TmpReg, 32 * 4); } } else { + static_assert(RAFPR.size() % 4 == 0, "Needs to have multiple of 4 FPRs for RA"); for (size_t i = 0; i < RAFPR.size(); i += 4) { const auto Reg1 = RAFPR[i]; const auto Reg2 = RAFPR[i + 1]; const auto Reg3 = RAFPR[i + 2]; const auto Reg4 = RAFPR[i + 3]; - st1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(TmpReg, 64, PostIndex)); + st1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64); } } - str(aarch64::lr, MemOperand(TmpReg, 0)); + str(ARMEmitter::XReg::lr, TmpReg, 0); } void Arm64Emitter::PopDynamicRegsAndLR() { @@ -380,8 +373,8 @@ void Arm64Emitter::PopDynamicRegsAndLR() { const auto Reg2 = RAFPR[i + 1]; const auto Reg3 = RAFPR[i + 2]; const auto Reg4 = RAFPR[i + 3]; - ld4b(Reg1.Z().VnB(), Reg2.Z().VnB(), Reg3.Z().VnB(), Reg4.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(aarch64::sp)); - add(aarch64::sp, aarch64::sp, 32 * 4); + ld4b(Reg1, Reg2, Reg3, Reg4, PRED_TMP_32B, ARMEmitter::Reg::rsp); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 32 * 4); } } else { for (size_t i = 0; i < RAFPR.size(); i += 4) { @@ -389,11 +382,11 @@ void Arm64Emitter::PopDynamicRegsAndLR() { const auto Reg2 = RAFPR[i + 1]; const auto Reg3 = RAFPR[i + 2]; const auto Reg4 = RAFPR[i + 3]; - ld1(Reg1.V2D(), Reg2.V2D(), Reg3.V2D(), Reg4.V2D(), MemOperand(aarch64::sp, 64, PostIndex)); + ld1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), ARMEmitter::Reg::rsp, 64); } } - ldr(aarch64::lr, MemOperand(aarch64::sp, 16, PostIndex)); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); } void Arm64Emitter::Align16B() { diff --git a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h index 2fe99cebc..32c8c0eba 100644 --- a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h +++ b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h @@ -1,5 +1,9 @@ #pragma once +#include "FEXCore/Utils/EnumUtils.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Registers.h" + #include "Interface/Core/Dispatcher/Dispatcher.h" #include "Interface/Core/ObjectCache/Relocations.h" @@ -24,72 +28,70 @@ #include namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; - // All but x29 are caller saved -const std::array SRA64 = { - x4, x5, x6, x7, x8, x9, x10, x11, - x12, x18, x17, x16, x15, x14, x13, x29 +constexpr std::array SRA64 = { + FEXCore::ARMEmitter::Reg::r4, FEXCore::ARMEmitter::Reg::r5, FEXCore::ARMEmitter::Reg::r6, FEXCore::ARMEmitter::Reg::r7, FEXCore::ARMEmitter::Reg::r8, FEXCore::ARMEmitter::Reg::r9, FEXCore::ARMEmitter::Reg::r10, FEXCore::ARMEmitter::Reg::r11, + FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r18, FEXCore::ARMEmitter::Reg::r17, FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r15, FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r13, FEXCore::ARMEmitter::Reg::r29 }; // All are callee saved -const std::array RA64 = { - x20, x21, x22, x23, x24, x25, x26, x27, - x19 +constexpr std::array RA64 = { + FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21, FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23, FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25, FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27, + FEXCore::ARMEmitter::Reg::r19 }; -const std::array, 4> RA64Pair = {{ - {x20, x21}, - {x22, x23}, - {x24, x25}, - {x26, x27}, +constexpr std::array, 4> RA64Pair = {{ + {FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21}, + {FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23}, + {FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25}, + {FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27}, }}; // All are caller saved -const std::array SRAFPR = { - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31 +constexpr std::array SRAFPR = { + FEXCore::ARMEmitter::VReg::v16, FEXCore::ARMEmitter::VReg::v17, FEXCore::ARMEmitter::VReg::v18, FEXCore::ARMEmitter::VReg::v19, FEXCore::ARMEmitter::VReg::v20, FEXCore::ARMEmitter::VReg::v21, FEXCore::ARMEmitter::VReg::v22, FEXCore::ARMEmitter::VReg::v23, + FEXCore::ARMEmitter::VReg::v24, FEXCore::ARMEmitter::VReg::v25, FEXCore::ARMEmitter::VReg::v26, FEXCore::ARMEmitter::VReg::v27, FEXCore::ARMEmitter::VReg::v28, FEXCore::ARMEmitter::VReg::v29, FEXCore::ARMEmitter::VReg::v30, FEXCore::ARMEmitter::VReg::v31 }; // v8..v15 = (lower 64bits) Callee saved -const std::array RAFPR = { -/*v0, v1, v2, v3,*/v4, v5, v6, v7, // v0 ~ v3 are used as temps - v8, v9, v10, v11, v12, v13, v14, v15 +constexpr std::array RAFPR = { +/*FEXCore::ARMEmitter::VReg::v0, FEXCore::ARMEmitter::VReg::v1, FEXCore::ARMEmitter::VReg::v2, FEXCore::ARMEmitter::VReg::v3,*/FEXCore::ARMEmitter::VReg::v4, FEXCore::ARMEmitter::VReg::v5, FEXCore::ARMEmitter::VReg::v6, FEXCore::ARMEmitter::VReg::v7, // FEXCore::ARMEmitter::VReg::v0 ~ FEXCore::ARMEmitter::VReg::v3 are used as temps + FEXCore::ARMEmitter::VReg::v8, FEXCore::ARMEmitter::VReg::v9, FEXCore::ARMEmitter::VReg::v10, FEXCore::ARMEmitter::VReg::v11, FEXCore::ARMEmitter::VReg::v12, FEXCore::ARMEmitter::VReg::v13, FEXCore::ARMEmitter::VReg::v14, FEXCore::ARMEmitter::VReg::v15 }; // Contains the address to the currently available CPU state -#define STATE x28 +constexpr auto STATE = FEXCore::ARMEmitter::XReg::x28; // GPR temporaries. Only x3 can be used across spill boundaries // so if these ever need to change, be very careful about that. -#define TMP1 x0 -#define TMP2 x1 -#define TMP3 x2 -#define TMP4 x3 +constexpr auto TMP1 = FEXCore::ARMEmitter::XReg::x0; +constexpr auto TMP2 = FEXCore::ARMEmitter::XReg::x1; +constexpr auto TMP3 = FEXCore::ARMEmitter::XReg::x2; +constexpr auto TMP4 = FEXCore::ARMEmitter::XReg::x3; // Vector temporaries -#define VTMP1 v0 -#define VTMP2 v1 -#define VTMP3 v2 -#define VTMP4 v3 +constexpr auto VTMP1 = FEXCore::ARMEmitter::VReg::v0; +constexpr auto VTMP2 = FEXCore::ARMEmitter::VReg::v1; +constexpr auto VTMP3 = FEXCore::ARMEmitter::VReg::v2; +constexpr auto VTMP4 = FEXCore::ARMEmitter::VReg::v3; // Predicate register temporaries (used when AVX support is enabled) // PRED_TMP_16B indicates a predicate register that indicates the first 16 bytes set to 1. // PRED_TMP_32B indicates a predicate register that indicates the first 32 bytes set to 1. -#define PRED_TMP_16B p6 -#define PRED_TMP_32B p7 +constexpr FEXCore::ARMEmitter::PRegister PRED_TMP_16B = FEXCore::ARMEmitter::PReg::p6; +constexpr FEXCore::ARMEmitter::PRegister PRED_TMP_32B = FEXCore::ARMEmitter::PReg::p7; // This class contains common emitter utility functions that can // be used by both Arm64 JIT and ARM64 Dispatcher -class Arm64Emitter : public vixl::aarch64::Assembler { +class Arm64Emitter : public FEXCore::ARMEmitter::Emitter { protected: Arm64Emitter(FEXCore::Context::Context *ctx, size_t size); ~Arm64Emitter(); FEXCore::Context::Context *EmitterCTX; vixl::aarch64::CPU CPU; - void LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, bool NOPPad = false); + void LoadConstant(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register Reg, uint64_t Constant, bool NOPPad = false); + // NOTE: These functions WILL clobber the register TMP4 if AVX support is enabled // and FPRs are being spilled or filled. If only GPRs are spilled/filled, then @@ -103,13 +105,14 @@ protected: // We can't guarantee only the lower 64bits are used so flush everything static constexpr uint32_t CALLER_FPR_MASK = ~0U; - void PushDynamicRegsAndLR(aarch64::Register TmpReg); + void PushDynamicRegsAndLR(FEXCore::ARMEmitter::Register TmpReg); void PopDynamicRegsAndLR(); void PushCalleeSavedRegisters(); void PopCalleeSavedRegisters(); void Align16B(); + #ifdef VIXL_SIMULATOR // Generates a vixl simulator runtime call. // @@ -121,57 +124,58 @@ protected: // 2) Simulator wrapper handler // 3) Function to call // 4) Style of the function call (Call versus tail-call) + template void GenerateRuntimeCall(R (*Function)(P...)) { uintptr_t SimulatorWrapperAddress = reinterpret_cast( - &(Simulator::RuntimeCallStructHelper::Wrapper)); + &(vixl::aarch64::Simulator::RuntimeCallStructHelper::Wrapper)); uintptr_t FunctionAddress = reinterpret_cast(Function); - hlt(kRuntimeCallOpcode); + hlt(vixl::aarch64::kRuntimeCallOpcode); // Simulator wrapper address pointer. - dc(SimulatorWrapperAddress); + dc64(SimulatorWrapperAddress); // Runtime function address to call - dc(FunctionAddress); + dc64(FunctionAddress); // Call type - dc32(kCallRuntime); + dc32(vixl::aarch64::kCallRuntime); } template - void GenerateIndirectRuntimeCall(vixl::aarch64::Register Reg) { + void GenerateIndirectRuntimeCall(ARMEmitter::Register Reg) { uintptr_t SimulatorWrapperAddress = reinterpret_cast( - &(Simulator::RuntimeCallStructHelper::Wrapper)); + &(vixl::aarch64::Simulator::RuntimeCallStructHelper::Wrapper)); - hlt(kIndirectRuntimeCallOpcode); + hlt(vixl::aarch64::kIndirectRuntimeCallOpcode); // Simulator wrapper address pointer. - dc(SimulatorWrapperAddress); + dc64(SimulatorWrapperAddress); // Register that contains the function to call - dc(Reg.GetCode()); + dc32(Reg.Idx()); // Call type - dc32(kCallRuntime); + dc32(vixl::aarch64::kCallRuntime); } template<> - void GenerateIndirectRuntimeCall(vixl::aarch64::Register Reg) { + void GenerateIndirectRuntimeCall(ARMEmitter::Register Reg) { uintptr_t SimulatorWrapperAddress = reinterpret_cast( - &(Simulator::RuntimeCallStructHelper::Wrapper)); + &(vixl::aarch64::Simulator::RuntimeCallStructHelper::Wrapper)); - hlt(kIndirectRuntimeCallOpcode); + hlt(vixl::aarch64::kIndirectRuntimeCallOpcode); // Simulator wrapper address pointer. - dc(SimulatorWrapperAddress); + dc64(SimulatorWrapperAddress); // Register that contains the function to call - dc(Reg.GetCode()); + dc32(Reg.Idx()); // Call type - dc32(kCallRuntime); + dc32(vixl::aarch64::kCallRuntime); } #endif diff --git a/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.cpp b/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.cpp index 2b220ed76..5d0be4db0 100644 --- a/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.cpp +++ b/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.cpp @@ -1,3 +1,4 @@ +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/LookupCache.h" #include "Interface/Core/ArchHelpers/MContext.h" @@ -30,14 +31,8 @@ #include #include -#define STATE_PTR(STATE_TYPE, FIELD) \ - MemOperand(STATE, offsetof(FEXCore::Core::STATE_TYPE, FIELD)) - namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; - constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096; Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config) @@ -46,16 +41,18 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche , Simulator {&Decoder} #endif { -#ifdef VIXL_DISASSEMBLER - const auto DisasmBegin = GetCursorAddress(); -#endif - #ifdef VIXL_SIMULATOR // Hardcode a 256-bit vector width if we are running in the simulator. Simulator.SetVectorLengthInBits(256); #endif - SetAllowAssembler(true); + EmitDispatcher(); +} + +void Arm64Dispatcher::EmitDispatcher() { +#ifdef VIXL_DISASSEMBLER + const auto DisasmBegin = GetCursorAddress(); +#endif DispatchPtr = GetCursorAddress(); @@ -67,9 +64,9 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche // Ptr(); // } - Literal l_CTX {reinterpret_cast(CTX)}; - Literal l_Sleep {reinterpret_cast(SleepThread)}; - Literal l_CompileBlock {GetCompileBlockPtr()}; + ARMEmitter::ForwardLabel l_CTX; + ARMEmitter::ForwardLabel l_Sleep; + ARMEmitter::ForwardLabel l_CompileBlock; // Push all the register we need to save PushCalleeSavedRegisters(); @@ -77,12 +74,12 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche // Push our memory base to the correct register // Move our thread pointer to the correct register // This is passed in to parameter 0 (x0) - mov(STATE, x0); + mov(STATE, ARMEmitter::XReg::x0); // Save this stack pointer so we can cleanly shutdown the emulation with a long jump // regardless of where we were in the stack - add(x0, sp, 0); - str(x0, STATE_PTR(CpuStateFrame, ReturningStackLocation)); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0); + str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, ReturningStackLocation)); AbsoluteLoopTopAddressFillSRA = GetCursorAddress(); @@ -92,91 +89,97 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche // We want to ensure that we are 16 byte aligned at the top of this loop Align16B(); - aarch64::Label FullLookup{}; - aarch64::Label CallBlock{}; - aarch64::Label LoopTop{}; - aarch64::Label ExitSpillSRA{}; - aarch64::Label ThreadPauseHandler{}; + ARMEmitter::BiDirectionalLabel FullLookup{}; + ARMEmitter::BiDirectionalLabel CallBlock{}; + ARMEmitter::BackwardLabel LoopTop{}; - bind(&LoopTop); - AbsoluteLoopTopAddress = GetLabelAddress(&LoopTop); + Bind(&LoopTop); + AbsoluteLoopTopAddress = GetCursorAddress(); // Load in our RIP // Don't modify x2 since it contains our RIP once the block doesn't exist - ldr(x2, STATE_PTR(CpuStateFrame, State.rip)); - auto RipReg = x2; + + auto RipReg = ARMEmitter::XReg::x2; + ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip)); // L1 Cache - ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer)); + ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer)); - and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK); - add(x0, x0, Operand(x3, Shift::LSL, 4)); - ldp(x3, x0, MemOperand(x0)); - cmp(x0, RipReg); - b(&FullLookup, Condition::ne); + and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg.R(), LookupCache::L1_ENTRIES_MASK); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::r0, ARMEmitter::Reg::r3, ARMEmitter::ShiftType::LSL , 4); + ldp(ARMEmitter::XReg::x3, ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, 0); + cmp(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, RipReg.R()); + b(ARMEmitter::Condition::CC_NE, &FullLookup); - br(x3); + br(ARMEmitter::Reg::r3); // L1C check failed, do a full lookup - bind(&FullLookup); + Bind(&FullLookup); // This is the block cache lookup routine // It matches what is going on it LookupCache.h::FindBlock - ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer)); + ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer)); // Mask the address by the virtual address size so we can check for aliases uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize; if (std::popcount(VirtualMemorySize) == 1) { - and_(x3, RipReg, VirtualMemorySize - 1); + and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg.R(), VirtualMemorySize - 1); } else { - LoadConstant(x3, VirtualMemorySize); - and_(x3, RipReg, x3); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, VirtualMemorySize); + and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg.R(), ARMEmitter::Reg::r3); } - aarch64::Label NoBlock; +#ifdef VIXL_SIMULATOR + // VIXL simulator can't run syscalls. + constexpr bool SignalSafeCompile = false; +#else + constexpr bool SignalSafeCompile = true; +#endif + + ARMEmitter::ForwardLabel NoBlock; + { // Offset the address and add to our page pointer - lsr(x1, x3, 12); + lsr(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::r3, 12); // Load the pointer from the offset - ldr(x0, MemOperand(x0, x1, Shift::LSL, 3)); + ldr(ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, ARMEmitter::Reg::r1, ARMEmitter::ExtendedType::LSL_64, 3); // If page pointer is zero then we have no block - cbz(x0, &NoBlock); + cbz(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, &NoBlock); // Steal the page offset - and_(x1, x3, 0x0FFF); + and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::r3, 0x0FFF); // Shift the offset by the size of the block cache entry - add(x0, x0, Operand(x1, Shift::LSL, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry)))); + add(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::ShiftType::LSL, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry))); // Load the guest address first to ensure it maps to the address we are currently at // This fixes aliasing problems - ldr(x1, MemOperand(x0, offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode))); - cmp(x1, RipReg); - b(&NoBlock, Condition::ne); + ldr(ARMEmitter::XReg::x1, ARMEmitter::Reg::r0, offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode)); + cmp(ARMEmitter::XReg::x1, RipReg); + b(ARMEmitter::Condition::CC_NE, &NoBlock); // Now load the actual host block to execute if we can - ldr(x3, MemOperand(x0, offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode))); - cbz(x3, &NoBlock); + ldr(ARMEmitter::XReg::x3, ARMEmitter::Reg::r0, offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode)); + cbz(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, &NoBlock); // If we've made it here then we have a real compiled block { // update L1 cache - ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer)); + ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer)); - and_(x1, RipReg, LookupCache::L1_ENTRIES_MASK); - add(x0, x0, Operand(x1, Shift::LSL, 4)); - stp(x3, x2, MemOperand(x0)); + and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, RipReg.R(), LookupCache::L1_ENTRIES_MASK); + add(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::ShiftType::LSL, 4); + stp(ARMEmitter::XReg::x3, ARMEmitter::XReg::x2, ARMEmitter::Reg::r0); // Jump to the block - br(x3); + br(ARMEmitter::Reg::r3); } } { - bind(&ExitSpillSRA); ThreadStopHandlerAddressSpillSRA = GetCursorAddress(); if (config.StaticRegisterAllocation) SpillStaticRegs(); @@ -190,12 +193,6 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche ret(); } -#ifdef VIXL_SIMULATOR - // VIXL simulator can't run syscalls. - constexpr bool SignalSafeCompile = false; -#else - constexpr bool SignalSafeCompile = true; -#endif { ExitFunctionLinkerAddress = GetCursorAddress(); if (config.StaticRegisterAllocation) @@ -210,52 +207,52 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche // X3: Size of mask, sizeof(uint64_t) // X8: Syscall - LoadConstant(x0, ~0ULL); - stp(x0, x0, MemOperand(sp, -16, PreIndex)); - LoadConstant(x0, SIG_SETMASK); - add(x1, sp, 0); - add(x2, sp, 0); - LoadConstant(x3, 8); - LoadConstant(x8, SYS_rt_sigprocmask); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ~0ULL); + stp(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::Reg::rsp, -16); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::rsp, 0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask); svc(0); } - mov(x0, STATE); - mov(x1, lr); + mov(ARMEmitter::XReg::x0, STATE); + mov(ARMEmitter::XReg::x1, ARMEmitter::XReg::lr); - ldr(x2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink)); + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink)); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif if (SignalSafeCompile) { // Now restore the signal mask // Living in the same location - mov(x4, x0); - LoadConstant(x0, SIG_SETMASK); - add(x1, sp, 0); - LoadConstant(x2, 0); - LoadConstant(x3, 8); - LoadConstant(x8, SYS_rt_sigprocmask); + mov(ARMEmitter::XReg::x4, ARMEmitter::XReg::x0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, 0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask); svc(0); // Bring stack back - add(sp, sp, 16); - - mov(x0, x4); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16); + mov(ARMEmitter::XReg::x0, ARMEmitter::XReg::x4); } if (config.StaticRegisterAllocation) FillStaticRegs(); - br(x0); + + br(ARMEmitter::Reg::r0); } // Need to create the block { - bind(&NoBlock); + Bind(&NoBlock); if (config.StaticRegisterAllocation) SpillStaticRegs(); @@ -269,42 +266,42 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche // X3: Size of mask, sizeof(uint64_t) // X8: Syscall - LoadConstant(x0, ~0ULL); - stp(x0, x2, MemOperand(sp, -16, PreIndex)); - LoadConstant(x0, SIG_SETMASK); - add(x1, sp, 0); - add(x2, sp, 0); - LoadConstant(x3, 8); - LoadConstant(x8, SYS_rt_sigprocmask); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ~0ULL); + stp(ARMEmitter::XReg::x0, ARMEmitter::XReg::x2, ARMEmitter::Reg::rsp, -16); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::rsp, 0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask); svc(0); // Reload x2 to bring back RIP - ldr(x2, MemOperand(sp, 8, Offset)); + ldr(ARMEmitter::XReg::x2, ARMEmitter::Reg::rsp, 8); } - ldr(x0, &l_CTX); - mov(x1, STATE); - ldr(x3, &l_CompileBlock); + ldr(ARMEmitter::XReg::x0, &l_CTX); + mov(ARMEmitter::XReg::x1, STATE); + ldr(ARMEmitter::XReg::x3, &l_CompileBlock); // X2 contains our guest RIP #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x3); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); #else - blr(x3); // { CTX, Frame, RIP} + blr(ARMEmitter::Reg::r3); // { CTX, Frame, RIP} #endif if (SignalSafeCompile) { // Now restore the signal mask // Living in the same location - LoadConstant(x0, SIG_SETMASK); - add(x1, sp, 0); - LoadConstant(x2, 0); - LoadConstant(x3, 8); - LoadConstant(x8, SYS_rt_sigprocmask); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SIG_SETMASK); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, 0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, 8); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_rt_sigprocmask); svc(0); // Bring stack back - add(sp, sp, 16); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16); } if (config.StaticRegisterAllocation) @@ -324,7 +321,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche { // Guest SIGILL handler // Needs to be distinct from the SignalHandlerReturnAddress - GuestSignal_SIGILL = GetCursorAddress(); + GuestSignal_SIGILL = GetCursorAddress(); if (config.StaticRegisterAllocation) SpillStaticRegs(); @@ -355,8 +352,8 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche // brk = SIGTRAP // ??? = SIGSEGV // Force a SIGSEGV by loading zero - LoadConstant(x1, 0); - ldr(x1, MemOperand(x1)); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, 0); + ldr(ARMEmitter::XReg::x1, ARMEmitter::Reg::r1); } { @@ -364,19 +361,18 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche if (config.StaticRegisterAllocation) SpillStaticRegs(); - bind(&ThreadPauseHandler); ThreadPauseHandlerAddress = GetCursorAddress(); // We are pausing, this means the frontend should be waiting for this thread to idle // We will have faulted and jumped to this location at this point // Call our sleep handler - ldr(x0, &l_CTX); - mov(x1, STATE); - ldr(x2, &l_Sleep); + ldr(ARMEmitter::XReg::x0, &l_CTX); + mov(ARMEmitter::XReg::x1, STATE); + ldr(ARMEmitter::XReg::x2, &l_Sleep); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PauseReturnInstruction = GetCursorAddress(); @@ -406,27 +402,27 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche PushCalleeSavedRegisters(); // First thing we need to move the thread state pointer back in to our register - mov(STATE, x0); + mov(STATE, ARMEmitter::XReg::x0); // Make sure to adjust the refcounter so we don't clear the cache now - ldr(w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter)); - add(w2, w2, 1); - str(w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter)); + ldr(ARMEmitter::WReg::w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter)); + add(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 1); + str(ARMEmitter::WReg::w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter)); // Now push the callback return trampoline to the guest stack // Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host - LoadConstant(x0, CTX->X86CodeGen.CallbackReturn); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, CTX->X86CodeGen.CallbackReturn); - ldr(x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP])); - sub(x2, x2, 16); - str(x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP])); + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP])); + sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 16); + str(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP])); // Store the trampoline to the guest stack // Guest stack is now correctly misaligned after a regular call instruction - str(x0, MemOperand(x2)); + str(ARMEmitter::XReg::x0, ARMEmitter::Reg::r2, 0); // Store RIP to the context state - str(x1, STATE_PTR(CpuStateFrame, State.rip)); + str(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, State.rip)); // load static regs if (config.StaticRegisterAllocation) @@ -439,14 +435,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche { LUDIVHandlerAddress = GetCursorAddress(); - PushDynamicRegsAndLR(x3); + PushDynamicRegsAndLR(ARMEmitter::Reg::r3); SpillStaticRegs(); - ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV)); + ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV)); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x3); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); #else - blr(x3); + blr(ARMEmitter::Reg::r3); #endif FillStaticRegs(); @@ -461,14 +457,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche { LDIVHandlerAddress = GetCursorAddress(); - PushDynamicRegsAndLR(x3); + PushDynamicRegsAndLR(ARMEmitter::Reg::r3); SpillStaticRegs(); - ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV)); + ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV)); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x3); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); #else - blr(x3); + blr(ARMEmitter::Reg::r3); #endif FillStaticRegs(); @@ -483,14 +479,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche { LUREMHandlerAddress = GetCursorAddress(); - PushDynamicRegsAndLR(x3); + PushDynamicRegsAndLR(ARMEmitter::Reg::r3); SpillStaticRegs(); - ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM)); + ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM)); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x3); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); #else - blr(x3); + blr(ARMEmitter::Reg::r3); #endif FillStaticRegs(); @@ -505,14 +501,15 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche { LREMHandlerAddress = GetCursorAddress(); - PushDynamicRegsAndLR(x3); + PushDynamicRegsAndLR(ARMEmitter::Reg::r3); SpillStaticRegs(); - ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM)); + ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM)); + #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x3); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); #else - blr(x3); + blr(ARMEmitter::Reg::r3); #endif FillStaticRegs(); @@ -524,16 +521,16 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche ret(); } - place(&l_CTX); - place(&l_Sleep); - place(&l_CompileBlock); + Bind(&l_CTX); + dc64(reinterpret_cast(CTX)); + Bind(&l_Sleep); + dc64(reinterpret_cast(SleepThread)); + Bind(&l_CompileBlock); + dc64(GetCompileBlockPtr()); - - FinalizeCode(); Start = reinterpret_cast(DispatchPtr); End = GetCursorAddress(); - vixl::aarch64::CPU::EnsureIAndDCacheCoherency(reinterpret_cast(DispatchPtr), End - reinterpret_cast(DispatchPtr)); - GetBuffer()->SetExecutable(); + ClearICache(reinterpret_cast(DispatchPtr), End - reinterpret_cast(DispatchPtr)); if (CTX->Config.BlockJITNaming()) { std::string Name = "Dispatch_" + std::to_string(FHU::Syscalls::gettid()); @@ -542,8 +539,9 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche if (CTX->Config.GlobalJITNaming()) { CTX->Symbols.RegisterJITSpace(reinterpret_cast(DispatchPtr), End - reinterpret_cast(DispatchPtr)); } + #ifdef VIXL_DISASSEMBLER - const auto DisasmEnd = GetCursorAddress(); + const auto DisasmEnd = GetCursorAddress(); Disasm.DisassembleBuffer(DisasmBegin, DisasmEnd); #endif } @@ -551,101 +549,90 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche #ifdef VIXL_SIMULATOR void Arm64Dispatcher::ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) { Simulator.WriteXRegister(0, reinterpret_cast(Frame)); - Simulator.RunFrom(reinterpret_cast(DispatchPtr)); + Simulator.RunFrom(reinterpret_cast(DispatchPtr)); } void Arm64Dispatcher::ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) { Simulator.WriteXRegister(0, reinterpret_cast(Frame)); Simulator.WriteXRegister(1, RIP); - Simulator.RunFrom(reinterpret_cast(CallbackPtr)); + Simulator.RunFrom(reinterpret_cast(CallbackPtr)); } #endif -// Used by GenerateGDBPauseCheck, GenerateInterpreterTrampoline, destination buffer is set before use -static thread_local vixl::aarch64::Assembler emit((uint8_t*)&emit, 1); - size_t Arm64Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) { + FEXCore::ARMEmitter::Emitter emit{CodeBuffer, MaxGDBPauseCheckSize}; - *emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxGDBPauseCheckSize); - - vixl::CodeBufferCheckScope scope(&emit, MaxGDBPauseCheckSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert); - - aarch64::Label RunBlock; + ARMEmitter::ForwardLabel RunBlock; // If we have a gdb server running then run in a less efficient mode that checks if we need to exit // This happens when single stepping static_assert(sizeof(FEXCore::Context::Context::Config.RunningMode) == 4, "This is expected to be size of 4"); - emit.ldr(x0, STATE_PTR(CpuStateFrame, Thread)); // Get thread - emit.ldr(x0, MemOperand(x0, offsetof(FEXCore::Core::InternalThreadState, CTX))); // Get Context - emit.ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode))); + emit.ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Thread)); + emit.ldr(ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, offsetof(FEXCore::Core::InternalThreadState, CTX)); // Get Context + emit.ldr(ARMEmitter::WReg::w0, ARMEmitter::Reg::r0, offsetof(FEXCore::Context::Context, Config.RunningMode)); // If the value == 0 then we don't need to stop - emit.cbz(w0, &RunBlock); + emit.cbz(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, &RunBlock); { - Literal l_GuestRIP {GuestRIP}; + ARMEmitter::ForwardLabel l_GuestRIP; // Make sure RIP is syncronized to the context - emit.ldr(x0, &l_GuestRIP); - emit.str(x0, STATE_PTR(CpuStateFrame, State.rip)); + emit.ldr(ARMEmitter::XReg::x0, &l_GuestRIP); + emit.str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, State.rip)); // Stop the thread - emit.ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA)); - emit.br(x0); - emit.place(&l_GuestRIP); + emit.ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA)); + emit.br(ARMEmitter::Reg::r0); + emit.Bind(&l_GuestRIP); + emit.dc64(GuestRIP); } - emit.bind(&RunBlock); - emit.FinalizeCode(); + emit.Bind(&RunBlock); - auto UsedBytes = emit.GetBuffer()->GetCursorOffset(); - vixl::aarch64::CPU::EnsureIAndDCacheCoherency(CodeBuffer, UsedBytes); + auto UsedBytes = emit.GetCursorOffset(); + emit.ClearICache(CodeBuffer, UsedBytes); return UsedBytes; } size_t Arm64Dispatcher::GenerateInterpreterTrampoline(uint8_t *CodeBuffer) { LOGMAN_THROW_AA_FMT(!config.StaticRegisterAllocation, "GenerateInterpreterTrampoline dispatcher does not support SRA"); - *emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxInterpreterTrampolineSize); + FEXCore::ARMEmitter::Emitter emit{CodeBuffer, MaxInterpreterTrampolineSize}; + ARMEmitter::ForwardLabel InlineIRData; - vixl::CodeBufferCheckScope scope(&emit, MaxInterpreterTrampolineSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert); + emit.mov(ARMEmitter::XReg::x0, STATE); + emit.adr(ARMEmitter::Reg::r1, &InlineIRData); - aarch64::Label InlineIRData; + emit.ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.Interpreter.FragmentExecuter)); + emit.blr(ARMEmitter::Reg::r3); - emit.mov(x0, STATE); - emit.adr(x1, &InlineIRData); + emit.ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.DispatcherLoopTop)); + emit.br(ARMEmitter::Reg::r0); - emit.ldr(x3, STATE_PTR(CpuStateFrame, Pointers.Interpreter.FragmentExecuter)); - emit.blr(x3); + emit.Bind(&InlineIRData); - emit.ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.DispatcherLoopTop)); - emit.br(x0); - - emit.bind(&InlineIRData); - - emit.FinalizeCode(); - - auto UsedBytes = emit.GetBuffer()->GetCursorOffset(); - vixl::aarch64::CPU::EnsureIAndDCacheCoherency(CodeBuffer, UsedBytes); + auto UsedBytes = emit.GetCursorOffset(); + emit.ClearICache(CodeBuffer, UsedBytes); return UsedBytes; } void Arm64Dispatcher::SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) { for (size_t i = 0; i < SRA64.size(); i++) { - if (IgnoreMask & (1U << SRA64[i].GetCode())) { + if (IgnoreMask & (1U << SRA64[i].Idx())) { // Skip this one, it's already spilled continue; } - Thread->CurrentFrame->State.gregs[i] = ArchHelpers::Context::GetArmReg(ucontext, SRA64[i].GetCode()); + Thread->CurrentFrame->State.gregs[i] = ArchHelpers::Context::GetArmReg(ucontext, SRA64[i].Idx()); } if (EmitterCTX->HostFeatures.SupportsAVX) { for (size_t i = 0; i < SRAFPR.size(); i++) { - auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].GetCode()); + auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].Idx()); memcpy(&Thread->CurrentFrame->State.xmm.avx.data[i][0], &FPR, sizeof(__uint128_t)); } } else { for (size_t i = 0; i < SRAFPR.size(); i++) { - auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].GetCode()); + auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].Idx()); memcpy(&Thread->CurrentFrame->State.xmm.sse.data[i][0], &FPR, sizeof(__uint128_t)); } } diff --git a/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h b/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h index 8dcbc6945..fc52a541c 100644 --- a/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h +++ b/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h @@ -15,6 +15,9 @@ namespace FEXCore::Core { struct InternalThreadState; } +#define STATE_PTR(STATE_TYPE, FIELD) \ + STATE.R(), offsetof(FEXCore::Core::STATE_TYPE, FIELD) + namespace FEXCore::CPU { class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter { @@ -29,6 +32,8 @@ class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter { void ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) override; #endif + void EmitDispatcher(); + protected: void SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) override; diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp index 41b86f023..14f9ee9b7 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp @@ -4,6 +4,8 @@ tags: backend|arm64 $end_info$ */ +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Registers.h" #include "Interface/Core/JIT/Arm64/JITClass.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" @@ -12,18 +14,16 @@ namespace FEXCore::CPU { #define GRD(Node) (IROp->Size <= 4 ? GetDst(Node) : GetDst(Node)) #define GRS(Node) (IROp->Size <= 4 ? GetReg(Node) : GetReg(Node)) -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(TruncElementPair) { auto Op = IROp->C(); switch (IROp->Size) { case 4: { - auto Dst = GetRegPair(Node); - auto Src = GetRegPair(Op->Pair.ID()); - mov(Dst.first, Src.first); - mov(Dst.second, Src.second); + auto Dst = GetRegPair(Node); + auto Src = GetRegPair(Op->Pair.ID()); + mov(ARMEmitter::Size::i32Bit, Dst.first, Src.first); + mov(ARMEmitter::Size::i32Bit, Dst.second, Src.second); break; } default: @@ -34,22 +34,22 @@ DEF_OP(TruncElementPair) { DEF_OP(Constant) { auto Op = IROp->C(); - auto Dst = GetReg(Node); - LoadConstant(Dst, Op->Constant); + auto Dst = GetReg(Node); + LoadConstant(ARMEmitter::Size::i64Bit, Dst, Op->Constant); } DEF_OP(EntrypointOffset) { auto Op = IROp->C(); auto Constant = Entry + Op->Offset; - auto Dst = GetReg(Node); + auto Dst = GetReg(Node); uint64_t Mask = ~0ULL; uint8_t OpSize = IROp->Size; if (OpSize == 4) { Mask = 0xFFFF'FFFFULL; } - LoadConstant(Dst, Constant & Mask); + LoadConstant(ARMEmitter::Size::i64Bit, Dst, Constant & Mask); } DEF_OP(InlineConstant) { @@ -62,9 +62,9 @@ DEF_OP(InlineEntrypointOffset) { DEF_OP(CycleCounter) { #ifdef DEBUG_CYCLES - movz(GetReg(Node), 0); + movz(ARMEmitter::Size::i64Bit, GetReg(Node), 0); #else - mrs(GetReg(Node), CNTVCT_EL0); + mrs(GetReg(Node), ARMEmitter::SystemRegister::CNTVCT_EL0); #endif } @@ -72,27 +72,14 @@ DEF_OP(Add) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - switch (OpSize) { - case 4: - add(GetReg(Node), GetReg(Op->Src1.ID()), Const); - break; - case 8: - add(GetReg(Node), GetReg(Op->Src1.ID()), Const); - break; - default: LOGMAN_MSG_A_FMT("Unsupported Add size: {}", OpSize); - } + add(EmitSize, GetReg(Node), GetReg(Op->Src1.ID()), Const); } else { - switch (OpSize) { - case 4: - add(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - case 8: - add(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unsupported Add size: {}", OpSize); - } + add(EmitSize, GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); } } @@ -100,73 +87,45 @@ DEF_OP(Sub) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - switch (OpSize) { - case 4: - case 8: - sub(GRS(Node), GRS(Op->Src1.ID()), Const); - break; - default: LOGMAN_MSG_A_FMT("Unsupported Sub size: {}", OpSize); - } + sub(EmitSize, GetReg(Node), GetReg(Op->Src1.ID()), Const); } else { - switch (OpSize) { - case 4: - sub(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - case 8: - sub(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unsupported Sub size: {}", OpSize); - } + sub(EmitSize, GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); } - } DEF_OP(Neg) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 4: - neg(GetReg(Node), GetReg(Op->Src.ID())); - break; - case 8: - neg(GetReg(Node), GetReg(Op->Src.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unsupported Neg size: {}", OpSize); - } + + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + neg(EmitSize, GetReg(Node), GetReg(Op->Src.ID())); } DEF_OP(Mul) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - auto Dst = GetReg(Node); - switch (OpSize) { - case 4: - mul(Dst.W(), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - case 8: - mul(Dst, GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown Mul size: {}", OpSize); - } + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + mul(EmitSize, GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); } DEF_OP(UMul) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - auto Dst = GetReg(Node); - switch (OpSize) { - case 4: - mul(Dst.W(), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - case 8: - mul(Dst, GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown UMul size: {}", OpSize); - } + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + mul(EmitSize, GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); } DEF_OP(Div) { @@ -175,38 +134,29 @@ DEF_OP(Div) { // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64 const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 1: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); - sxtb(w2, Dividend); - sxtb(w3, Divisor); - sdiv(GetReg(Node), w2, w3); - break; - } - case 2: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); - sxth(w2, Dividend); - sxth(w3, Divisor); + const auto Dst = GetReg(Node); + auto Src1 = GetReg(Op->Src1.ID()); + auto Src2 = GetReg(Op->Src2.ID()); - sdiv(GetReg(Node), w2, w3); - break; - } - case 4: { - sdiv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - case 8: { - sdiv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - default: { - LOGMAN_MSG_A_FMT("Unknown DIV Size: {}", OpSize); - break; - } + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + if (OpSize == 1) { + sxtb(EmitSize, TMP1, Src1); + sxtb(EmitSize, TMP2, Src2); + + Src1 = TMP1; + Src2 = TMP2; } + else if (OpSize == 2) { + sxth(EmitSize, TMP1, Src1); + sxth(EmitSize, TMP2, Src2); + + Src1 = TMP1; + Src2 = TMP2; + } + + sdiv(EmitSize, Dst, Src1, Src2); } DEF_OP(UDiv) { @@ -215,238 +165,264 @@ DEF_OP(UDiv) { // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64 const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 1: { - udiv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - case 2: { - udiv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - case 4: { - udiv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - case 8: { - udiv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - default: { - LOGMAN_MSG_A_FMT("Unknown UDIV Size: {}", OpSize); - break; - } + + const auto Dst = GetReg(Node); + auto Src1 = GetReg(Op->Src1.ID()); + auto Src2 = GetReg(Op->Src2.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + if (OpSize == 1) { + uxtb(EmitSize, TMP1, Src1); + uxtb(EmitSize, TMP2, Src2); + + Src1 = TMP1; + Src2 = TMP2; } + else if (OpSize == 2) { + uxth(EmitSize, TMP1, Src1); + uxth(EmitSize, TMP2, Src2); + + Src1 = TMP1; + Src2 = TMP2; + } + + udiv(EmitSize, Dst, Src1, Src2); } DEF_OP(Rem) { auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 1: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); - sxtb(w2, Dividend); - sxtb(w3, Divisor); + const uint8_t OpSize = IROp->Size; - sdiv(TMP1.W(), w2, w3); - msub(GetReg(Node), TMP1.W(), w3, w2); - break; - } - case 2: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); + const auto Dst = GetReg(Node); + auto Src1 = GetReg(Op->Src1.ID()); + auto Src2 = GetReg(Op->Src2.ID()); - sxth(w2, Dividend); - sxth(w3, Divisor); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; - sdiv(TMP1.W(), w2, w3); - msub(GetReg(Node), TMP1.W(), w3, w2); - break; - } - case 4: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); + if (OpSize == 1) { + sxtb(EmitSize, TMP1, Src1); + sxtb(EmitSize, TMP2, Src2); - sdiv(TMP1.W(), Dividend, Divisor); - msub(GetReg(Node), TMP1, Divisor, Dividend); - break; - } - case 8: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); - - sdiv(TMP1, Dividend, Divisor); - msub(GetReg(Node), TMP1, Divisor, Dividend); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown REM Size: {}", OpSize); break; + Src1 = TMP1; + Src2 = TMP2; } + else if (OpSize == 2) { + sxth(EmitSize, TMP1, Src1); + sxth(EmitSize, TMP2, Src2); + + Src1 = TMP1; + Src2 = TMP2; + } + + sdiv(EmitSize, TMP1, Src1, Src2); + msub(EmitSize, Dst, TMP1, Src2, Src1); } DEF_OP(URem) { auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 1: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); + const uint8_t OpSize = IROp->Size; - udiv(TMP1.W(), Dividend, Divisor); - msub(GetReg(Node), TMP1, Divisor, Dividend); - break; - } - case 2: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); + const auto Dst = GetReg(Node); + auto Src1 = GetReg(Op->Src1.ID()); + auto Src2 = GetReg(Op->Src2.ID()); - udiv(TMP1.W(), Dividend, Divisor); - msub(GetReg(Node), TMP1, Divisor, Dividend); - break; - } - case 4: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + if (OpSize == 1) { + uxtb(EmitSize, TMP1, Src1); + uxtb(EmitSize, TMP2, Src2); - udiv(TMP1.W(), Dividend, Divisor); - msub(GetReg(Node), TMP1, Divisor, Dividend); - break; - } - case 8: { - auto Dividend = GetReg(Op->Src1.ID()); - auto Divisor = GetReg(Op->Src2.ID()); - - udiv(TMP1, Dividend, Divisor); - msub(GetReg(Node), TMP1, Divisor, Dividend); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown UREM Size: {}", OpSize); break; + Src1 = TMP1; + Src2 = TMP2; } + else if (OpSize == 2) { + uxth(EmitSize, TMP1, Src1); + uxth(EmitSize, TMP2, Src2); + + Src1 = TMP1; + Src2 = TMP2; + } + + udiv(EmitSize, TMP3, Src1, Src2); + msub(EmitSize, Dst, TMP3, Src2, Src1); } DEF_OP(MulH) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 4: - sxtw(TMP1, GetReg(Op->Src1.ID())); - sxtw(TMP2, GetReg(Op->Src2.ID())); - mul(TMP1, TMP1, TMP2); - ubfx(GetReg(Node), TMP1, 32, 32); - break; - case 8: - smulh(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown Sext size: {}", OpSize); + + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + const auto Src2 = GetReg(Op->Src2.ID()); + + if (OpSize == 4) { + sxtw(TMP1, Src1); + sxtw(TMP2, Src2); + mul(ARMEmitter::Size::i32Bit, Dst, TMP1, TMP2); + ubfx(ARMEmitter::Size::i32Bit, Dst, Dst, 32, 32); + } + else { + smulh(Dst, Src1, Src2); } } DEF_OP(UMulH) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 4: - uxtw(TMP1, GetReg(Op->Src1.ID())); - uxtw(TMP2, GetReg(Op->Src2.ID())); - mul(TMP1, TMP1, TMP2); - ubfx(GetReg(Node), TMP1, 32, 32); - break; - case 8: - umulh(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown Sext size: {}", OpSize); + + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + const auto Src2 = GetReg(Op->Src2.ID()); + + if (OpSize == 4) { + uxtw(ARMEmitter::Size::i64Bit, TMP1, Src1); + uxtw(ARMEmitter::Size::i64Bit, TMP2, Src2); + mul(ARMEmitter::Size::i64Bit, Dst, TMP1, TMP2); + ubfx(ARMEmitter::Size::i64Bit, Dst, Dst, 32, 32); + } + else { + umulh(Dst, Src1, Src2); } } DEF_OP(Or) { auto Op = IROp->C(); + const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - orr(GRS(Node), GRS(Op->Src1.ID()), Const); + orr(EmitSize, Dst, Src1, Const); } else { - orr(GRS(Node), GRS(Op->Src1.ID()), GRS(Op->Src2.ID())); + const auto Src2 = GetReg(Op->Src2.ID()); + orr(EmitSize, Dst, Src1, Src2); } } DEF_OP(And) { auto Op = IROp->C(); + const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - and_(GRS(Node), GRS(Op->Src1.ID()), Const); + and_(EmitSize, Dst, Src1, Const); } else { - and_(GRS(Node), GRS(Op->Src1.ID()), GRS(Op->Src2.ID())); + const auto Src2 = GetReg(Op->Src2.ID()); + and_(EmitSize, Dst, Src1, Src2); } } DEF_OP(Andn) { auto Op = IROp->C(); - const auto& Lhs = Op->Src1; - const auto& Rhs = Op->Src2; - uint64_t Const{}; + const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; - if (IsInlineConstant(Rhs, &Const)) { - bic(GRS(Node), GRS(Lhs.ID()), Const); + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + + uint64_t Const; + if (IsInlineConstant(Op->Src2, &Const)) { + bic(EmitSize, Dst, Src1, Const); } else { - bic(GRS(Node), GRS(Lhs.ID()), GRS(Rhs.ID())); + const auto Src2 = GetReg(Op->Src2.ID()); + bic(EmitSize, Dst, Src1, Src2); } } DEF_OP(Xor) { auto Op = IROp->C(); + const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - eor(GRS(Node), GRS(Op->Src1.ID()), Const); + eor(EmitSize, Dst, Src1, Const); } else { - eor(GRS(Node), GRS(Op->Src1.ID()), GRS(Op->Src2.ID())); + const auto Src2 = GetReg(Op->Src2.ID()); + eor(EmitSize, Dst, Src1, Src2); } } DEF_OP(Lshl) { auto Op = IROp->C(); + const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - lsl(GRS(Node), GRS(Op->Src1.ID()), (unsigned int)Const); + lsl(EmitSize, Dst, Src1, Const); } else { - lslv(GRS(Node), GRS(Op->Src1.ID()), GRS(Op->Src2.ID())); + const auto Src2 = GetReg(Op->Src2.ID()); + lslv(EmitSize, Dst, Src1, Src2); } } DEF_OP(Lshr) { auto Op = IROp->C(); + + const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - lsr(GRS(Node), GRS(Op->Src1.ID()), (unsigned int)Const); + lsr(EmitSize, Dst, Src1, Const); } else { - lsrv(GRS(Node), GRS(Op->Src1.ID()), GRS(Op->Src2.ID())); + const auto Src2 = GetReg(Op->Src2.ID()); + lsrv(EmitSize, Dst, Src1, Src2); } } DEF_OP(Ashr) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { if (OpSize >= 4) { - asr(GRS(Node), GRS(Op->Src1.ID()), (unsigned int)Const); + asr(EmitSize, Dst, Src1, (unsigned int)Const); } else { - sbfx(TMP1.X(), GetReg(Op->Src1.ID()), 0, OpSize * 8); - asr(GetReg(Node), TMP1.X(), (unsigned int)Const); - ubfx(GetReg(Node),GetReg(Node), 0, OpSize * 8); + sbfx(EmitSize, TMP1, Src1, 0, OpSize * 8); + asr(EmitSize, Dst, TMP1, (unsigned int)Const); + ubfx(EmitSize, Dst, Dst, 0, OpSize * 8); } } else { + const auto Src2 = GetReg(Op->Src2.ID()); if (OpSize >= 4) { - asrv(GRS(Node), GRS(Op->Src1.ID()), GRS(Op->Src2.ID())); + asrv(EmitSize, Dst, Src1, Src2); } else { - sbfx(TMP1.X(), GetReg(Op->Src1.ID()), 0, OpSize * 8); - asrv(GetReg(Node), TMP1.X(), GetReg(Op->Src2.ID())); - ubfx(GetReg(Node),GetReg(Node), 0, OpSize * 8); + sbfx(EmitSize, TMP1, Src1, 0, OpSize * 8); + asrv(EmitSize, Dst, TMP1, Src2); + ubfx(EmitSize, Dst, Dst, 0, OpSize * 8); } } } @@ -455,33 +431,18 @@ DEF_OP(Ror) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + uint64_t Const; if (IsInlineConstant(Op->Src2, &Const)) { - switch (OpSize) { - case 4: { - ror(GetReg(Node), GetReg(Op->Src1.ID()), (unsigned int)Const); - break; - } - case 8: { - ror(GetReg(Node), GetReg(Op->Src1.ID()), (unsigned int)Const); - break; - } - - default: LOGMAN_MSG_A_FMT("Unhandled ROR size: {}", OpSize); - } + ror(EmitSize, Dst, Src1, Const); } else { - switch (OpSize) { - case 4: { - rorv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - case 8: { - rorv(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); - break; - } - - default: LOGMAN_MSG_A_FMT("Unhandled ROR size: {}", OpSize); - } + const auto Src2 = GetReg(Op->Src2.ID()); + rorv(EmitSize, Dst, Src1, Src2); } } @@ -489,197 +450,205 @@ DEF_OP(Extr) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 4: { - extr(GetReg(Node), GetReg(Op->Upper.ID()), GetReg(Op->Lower.ID()), Op->LSB); - break; - } - case 8: { - extr(GetReg(Node), GetReg(Op->Upper.ID()), GetReg(Op->Lower.ID()), Op->LSB); - break; - } + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; - default: LOGMAN_MSG_A_FMT("Unhandled EXTR size: {}", OpSize); - } + const auto Dst = GetReg(Node); + const auto Upper = GetReg(Op->Upper.ID()); + const auto Lower = GetReg(Op->Lower.ID()); + + extr(EmitSize, Dst, Upper, Lower, Op->LSB); } DEF_OP(PDep) { auto Op = IROp->C(); const auto OpSize = IROp->Size; - const Register Input = GRS(Op->Input.ID()); - const Register Mask = GRS(Op->Mask.ID()); - const Register Dest = GRS(Node); + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; - const Register ShiftedBitReg = OpSize <= 4 ? TMP1.W() : TMP1; - const Register BitReg = OpSize <= 4 ? TMP2.W() : TMP2; - const Register SubMaskReg = OpSize <= 4 ? TMP3.W() : TMP3; - const Register IndexReg = OpSize <= 4 ? TMP4.W() : TMP4; - const Register SizedZero = OpSize <= 4 ? Register{wzr} : Register{xzr}; + const auto Input = GetReg(Op->Input.ID()); + const auto Mask = GetReg(Op->Mask.ID()); + const auto Dest = GetReg(Node); - const Register InputReg = OpSize <= 4 ? SRA64[0].W() : SRA64[0]; - const Register MaskReg = OpSize <= 4 ? SRA64[1].W() : SRA64[1]; - const Register DestReg = OpSize <= 4 ? SRA64[2].W() : SRA64[2]; - const auto SpillCode = 1U << InputReg.GetCode() | - 1U << MaskReg.GetCode() | - 1U << DestReg.GetCode(); + const auto ShiftedBitReg = TMP1.R(); + const auto BitReg = TMP2.R(); + const auto SubMaskReg = TMP3.R(); + const auto IndexReg = TMP4.R(); + const auto ZeroReg = ARMEmitter::Reg::zr; - aarch64::Label EarlyExit; - aarch64::Label NextBit; - aarch64::Label Done; + const auto InputReg = SRA64[0]; + const auto MaskReg = SRA64[1]; + const auto DestReg = SRA64[2]; - cbz(Mask, &EarlyExit); - mov(IndexReg, SizedZero); + const auto SpillCode = 1U << InputReg.Idx() | + 1U << MaskReg.Idx() | + 1U << DestReg.Idx(); + + ARMEmitter::ForwardLabel EarlyExit; + ARMEmitter::BackwardLabel NextBit; + ARMEmitter::ForwardLabel Done; + cbz(EmitSize, Mask, &EarlyExit); + mov(EmitSize, IndexReg, ZeroReg); // We sadly need to spill regs for this for the time being // TODO: Remove when scratch registers can be allocated // explicitly. SpillStaticRegs(false, SpillCode); - mov(InputReg, Input); - mov(MaskReg, Mask); - mov(DestReg, SizedZero); + + + mov(EmitSize, InputReg, Input); + mov(EmitSize, MaskReg, Mask); + mov(EmitSize, DestReg, ZeroReg); // Main loop - bind(&NextBit); - rbit(ShiftedBitReg, MaskReg); - clz(ShiftedBitReg, ShiftedBitReg); - lsrv(BitReg, InputReg, IndexReg); - and_(BitReg, BitReg, 1); - sub(SubMaskReg, MaskReg, 1); - add(IndexReg, IndexReg, 1); - ands(MaskReg, MaskReg, SubMaskReg); - lslv(ShiftedBitReg, BitReg, ShiftedBitReg); - orr(DestReg, DestReg, ShiftedBitReg); - b(&NextBit, Condition::ne); + Bind(&NextBit); + rbit(EmitSize, ShiftedBitReg, MaskReg); + clz(EmitSize, ShiftedBitReg, ShiftedBitReg); + lsrv(EmitSize, BitReg, InputReg, IndexReg); + and_(EmitSize, BitReg, BitReg, 1); + sub(EmitSize, SubMaskReg, MaskReg, 1); + add(EmitSize, IndexReg, IndexReg, 1); + ands(EmitSize, MaskReg, MaskReg, SubMaskReg); + lslv(EmitSize, ShiftedBitReg, BitReg, ShiftedBitReg); + orr(EmitSize, DestReg, DestReg, ShiftedBitReg); + b(ARMEmitter::Condition::CC_NE, &NextBit); // Store result in a temp so it doesn't get clobbered. // and restore it after the re-fill below. - mov(IndexReg, DestReg); + mov(EmitSize, IndexReg, DestReg); // Restore our registers before leaving // TODO: Also remove along with above TODO. FillStaticRegs(false, SpillCode); - mov(Dest, IndexReg); + mov(EmitSize, Dest, IndexReg); b(&Done); // Early exit - bind(&EarlyExit); - mov(Dest, SizedZero); + Bind(&EarlyExit); + mov(EmitSize, Dest, ZeroReg); // All done with nothing to do. - bind(&Done); + Bind(&Done); } DEF_OP(PExt) { auto Op = IROp->C(); const auto OpSize = IROp->Size; - const Register Input = GRS(Op->Input.ID()); - const Register Mask = GRS(Op->Mask.ID()); - const Register Dest = GRS(Node); + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; - const Register MaskReg = OpSize <= 4 ? TMP1.W() : TMP1; - const Register BitReg = OpSize <= 4 ? TMP2.W() : TMP2; - const Register SubMaskReg = OpSize <= 4 ? TMP3.W() : TMP3; - const Register Offset = OpSize <= 4 ? TMP4.W() : TMP4; - const Register SizedZero = OpSize <= 4 ? Register{wzr} : Register{xzr}; + const auto Input = GetReg(Op->Input.ID()); + const auto Mask = GetReg(Op->Mask.ID()); + const auto Dest = GetReg(Node); - aarch64::Label EarlyExit; - aarch64::Label NextBit; - aarch64::Label Done; + const auto MaskReg = TMP1; + const auto BitReg = TMP2; + const auto SubMaskReg = TMP3; + const auto Offset = TMP4; + const auto ZeroReg = ARMEmitter::Reg::zr; - cbz(Mask, &EarlyExit); - mov(MaskReg, Mask); - mov(Offset, SizedZero); + ARMEmitter::ForwardLabel EarlyExit; + ARMEmitter::BackwardLabel NextBit; + ARMEmitter::ForwardLabel Done; + + cbz(EmitSize, Mask, &EarlyExit); + mov(EmitSize, MaskReg, Mask); + mov(EmitSize, Offset, ZeroReg); // We sadly need to spill a reg for this for the time being // TODO: Remove when scratch registers can be allocated // explicitly. - SpillStaticRegs(false, 1U << Mask.GetCode()); - mov(Mask, SizedZero); + SpillStaticRegs(false, 1U << Mask.Idx()); + mov(EmitSize, Mask, ZeroReg); // Main loop - bind(&NextBit); - rbit(BitReg, MaskReg); - clz(BitReg, BitReg); - sub(SubMaskReg, MaskReg, 1); - ands(MaskReg, SubMaskReg, MaskReg); - lsrv(BitReg, Input, BitReg); - and_(BitReg, BitReg, 1); - lslv(BitReg, BitReg, Offset); - add(Offset, Offset, 1); - orr(Mask, BitReg, Mask); - b(&NextBit, Condition::ne); - mov(Dest, Mask); + Bind(&NextBit); + rbit(EmitSize, BitReg, MaskReg); + clz(EmitSize, BitReg, BitReg); + sub(EmitSize, SubMaskReg, MaskReg, 1); + ands(EmitSize, MaskReg.R(), SubMaskReg.R(), MaskReg.R()); + lsrv(EmitSize, BitReg, Input, BitReg); + and_(EmitSize, BitReg, BitReg, 1); + lslv(EmitSize, BitReg, BitReg, Offset); + add(EmitSize, Offset, Offset, 1); + orr(EmitSize, Mask, BitReg, Mask); + b(ARMEmitter::Condition::CC_NE, &NextBit); + mov(EmitSize, Dest, Mask); // Restore our mask register before leaving // TODO: Also remove along with above TODO. - FillStaticRegs(false, 1U << Mask.GetCode()); + FillStaticRegs(false, 1U << Mask.Idx()); b(&Done); // Early exit - bind(&EarlyExit); - mov(Dest, SizedZero); + Bind(&EarlyExit); + mov(EmitSize, Dest, ZeroReg); // All done with nothing to do. - bind(&Done); + Bind(&Done); } DEF_OP(LDiv) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize >= 4 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Upper = GetReg(Op->Upper.ID()); + const auto Lower = GetReg(Op->Lower.ID()); + const auto Divisor = GetReg(Op->Divisor.ID()); // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64 switch (OpSize) { case 2: { - uxth(TMP1.W(), GetReg(Op->Lower.ID())); - bfi(TMP1.W(), GetReg(Op->Upper.ID()), 16, 16); - sxth(TMP2.W(), GetReg(Op->Divisor.ID())); - sdiv(GetReg(Node), TMP1.W(), TMP2.W()); + uxth(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 16, 16); + sxth(EmitSize, TMP2, Divisor); + sdiv(EmitSize, Dst, TMP1, TMP2); break; } case 4: { - mov(TMP1, GetReg(Op->Lower.ID())); - bfi(TMP1, GetReg(Op->Upper.ID()), 32, 32); - sxtw(TMP2, GetReg(Op->Divisor.ID())); - sdiv(GetReg(Node), TMP1, TMP2); + mov(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 32, 32); + sxtw(TMP2, Divisor); + sdiv(EmitSize, Dst, TMP1, TMP2); break; } case 8: { - auto Upper64Bit = GetReg(Op->Upper.ID()); - auto Lower64Bit = GetReg(Op->Lower.ID()); - auto Divisor = GetReg(Op->Divisor.ID()); - Label Only64Bit{}; - Label LongDIVRet{}; + ARMEmitter::ForwardLabel Only64Bit{}; + ARMEmitter::ForwardLabel LongDIVRet{}; // Check if the upper bits match the top bit of the lower 64-bits // Sign extend the top bit of lower bits - sbfx(TMP1, Lower64Bit, 63, 1); - eor(TMP1, TMP1, Upper64Bit); + sbfx(EmitSize, TMP1, Lower, 63, 1); + eor(EmitSize, TMP1, TMP1, Upper); // If the sign bit matches then the result is zero - cbz(TMP1, &Only64Bit); + cbz(EmitSize, TMP1, &Only64Bit); // Long divide { - mov(x0, Upper64Bit); - mov(x1, Lower64Bit); - mov(x2, Divisor); + mov(EmitSize, ARMEmitter::Reg::r0, Upper); + mov(EmitSize, ARMEmitter::Reg::r1, Lower); + mov(EmitSize, ARMEmitter::Reg::r2, Divisor); + + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LDIVHandler)); + blr(ARMEmitter::Reg::r3); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LDIVHandler))); - blr(x3); // Move result to its destination register - mov(GetReg(Node), x0); + mov(EmitSize, Dst, ARMEmitter::Reg::r0); // Skip 64-bit path b(&LongDIVRet); } - bind(&Only64Bit); + Bind(&Only64Bit); // 64-Bit only { - sdiv(GetReg(Node), Lower64Bit, Divisor); + sdiv(EmitSize, Dst, Lower, Divisor); } - bind(&LongDIVRet); + Bind(&LongDIVRet); break; } default: @@ -691,55 +660,58 @@ DEF_OP(LDiv) { DEF_OP(LUDiv) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize >= 4 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Upper = GetReg(Op->Upper.ID()); + const auto Lower = GetReg(Op->Lower.ID()); + const auto Divisor = GetReg(Op->Divisor.ID()); // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64= switch (OpSize) { case 2: { - uxth(TMP1.W(), GetReg(Op->Lower.ID())); - bfi(TMP1.W(), GetReg(Op->Upper.ID()), 16, 16); - udiv(GetReg(Node), TMP1.W(), GetReg(Op->Divisor.ID())); + uxth(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 16, 16); + udiv(EmitSize, Dst, TMP1, Divisor); break; } case 4: { - mov(TMP1, GetReg(Op->Lower.ID())); - bfi(TMP1, GetReg(Op->Upper.ID()), 32, 32); - udiv(GetReg(Node), TMP1, GetReg(Op->Divisor.ID())); + mov(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 32, 32); + udiv(EmitSize, Dst, TMP1, Divisor); break; } case 8: { - auto Upper64Bit = GetReg(Op->Upper.ID()); - auto Lower64Bit = GetReg(Op->Lower.ID()); - auto Divisor = GetReg(Op->Divisor.ID()); - Label Only64Bit{}; - Label LongDIVRet{}; + ARMEmitter::ForwardLabel Only64Bit{}; + ARMEmitter::ForwardLabel LongDIVRet{}; // Check the upper bits for zero // If the upper bits are zero then we can do a 64-bit divide - cbz(Upper64Bit, &Only64Bit); + cbz(EmitSize, Upper, &Only64Bit); // Long divide { - mov(x0, Upper64Bit); - mov(x1, Lower64Bit); - mov(x2, Divisor); + mov(EmitSize, ARMEmitter::Reg::r0, Upper); + mov(EmitSize, ARMEmitter::Reg::r1, Lower); + mov(EmitSize, ARMEmitter::Reg::r2, Divisor); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LUDIVHandler))); - blr(x3); + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LUDIVHandler)); + blr(ARMEmitter::Reg::r3); // Move result to its destination register - mov(GetReg(Node), x0); + mov(EmitSize, Dst, ARMEmitter::Reg::r0); // Skip 64-bit path b(&LongDIVRet); } - bind(&Only64Bit); + Bind(&Only64Bit); // 64-Bit only { - udiv(GetReg(Node), Lower64Bit, Divisor); + udiv(EmitSize, Dst, Lower, Divisor); } - bind(&LongDIVRet); + Bind(&LongDIVRet); break; } default: @@ -751,69 +723,66 @@ DEF_OP(LUDiv) { DEF_OP(LRem) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize >= 4 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Upper = GetReg(Op->Upper.ID()); + const auto Lower = GetReg(Op->Lower.ID()); + const auto Divisor = GetReg(Op->Divisor.ID()); // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64 switch (OpSize) { case 2: { - auto Divisor = GetReg(Op->Divisor.ID()); - - uxth(TMP1.W(), GetReg(Op->Lower.ID())); - bfi(TMP1.W(), GetReg(Op->Upper.ID()), 16, 16); - sxth(w3, Divisor); - sdiv(TMP2.W(), TMP1.W(), w3); - - msub(GetReg(Node), TMP2.W(), w3, TMP1.W()); + uxth(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 16, 16); + sxth(EmitSize, TMP2, Divisor); + sdiv(EmitSize, TMP3, TMP1, TMP2); + msub(EmitSize, Dst, TMP3, TMP2, TMP1); break; } case 4: { - auto Divisor = GetReg(Op->Divisor.ID()); - - mov(TMP1, GetReg(Op->Lower.ID())); - bfi(TMP1, GetReg(Op->Upper.ID()), 32, 32); - sxtw(x3, Divisor); - sdiv(TMP2, TMP1, x3); - - msub(GetReg(Node), TMP2.W(), w3, TMP1.W()); + mov(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 32, 32); + sxtw(TMP3, Divisor); + sdiv(EmitSize, TMP2, TMP1, TMP3); + msub(EmitSize, Dst, TMP2, TMP3, TMP1); break; } case 8: { - auto Upper64Bit = GetReg(Op->Upper.ID()); - auto Lower64Bit = GetReg(Op->Lower.ID()); - auto Divisor = GetReg(Op->Divisor.ID()); - Label Only64Bit{}; - Label LongDIVRet{}; + ARMEmitter::ForwardLabel Only64Bit{}; + ARMEmitter::ForwardLabel LongDIVRet{}; // Check if the upper bits match the top bit of the lower 64-bits // Sign extend the top bit of lower bits - sbfx(TMP1, Lower64Bit, 63, 1); - eor(TMP1, TMP1, Upper64Bit); + sbfx(EmitSize, TMP1, Lower, 63, 1); + eor(EmitSize, TMP1, TMP1, Upper); // If the sign bit matches then the result is zero - cbz(TMP1, &Only64Bit); + cbz(EmitSize, TMP1, &Only64Bit); // Long divide { - mov(x0, Upper64Bit); - mov(x1, Lower64Bit); - mov(x2, Divisor); + mov(EmitSize, ARMEmitter::Reg::r0, Upper); + mov(EmitSize, ARMEmitter::Reg::r1, Lower); + mov(EmitSize, ARMEmitter::Reg::r2, Divisor); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LREMHandler))); - blr(x3); + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LREMHandler)); + blr(ARMEmitter::Reg::r3); // Move result to its destination register - mov(GetReg(Node), x0); + mov(EmitSize, Dst, ARMEmitter::Reg::r0); // Skip 64-bit path b(&LongDIVRet); } - bind(&Only64Bit); + Bind(&Only64Bit); // 64-Bit only { - sdiv(TMP1, Lower64Bit, Divisor); - msub(GetReg(Node), TMP1, Divisor, Lower64Bit); + sdiv(EmitSize, TMP1, Lower, Divisor); + msub(EmitSize, Dst, TMP1, Divisor, Lower); } - bind(&LongDIVRet); + Bind(&LongDIVRet); break; } default: @@ -825,63 +794,61 @@ DEF_OP(LRem) { DEF_OP(LURem) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize >= 4 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Upper = GetReg(Op->Upper.ID()); + const auto Lower = GetReg(Op->Lower.ID()); + const auto Divisor = GetReg(Op->Divisor.ID()); // Each source is OpSize in size // So you can have up to a 128bit divide from x86-64 switch (OpSize) { case 2: { - auto Divisor = GetReg(Op->Divisor.ID()); - - uxth(TMP1.W(), GetReg(Op->Lower.ID())); - bfi(TMP1.W(), GetReg(Op->Upper.ID()), 16, 16); - udiv(TMP2.W(), TMP1.W(), Divisor); - msub(GetReg(Node), TMP2.W(), Divisor, TMP1.W()); + uxth(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 16, 16); + udiv(EmitSize, TMP2, TMP1, Divisor); + msub(EmitSize, Dst, TMP2, Divisor, TMP1); break; } case 4: { - auto Divisor = GetReg(Op->Divisor.ID()); - - mov(TMP1, GetReg(Op->Lower.ID())); - bfi(TMP1, GetReg(Op->Upper.ID()), 32, 32); - udiv(TMP2, TMP1, Divisor); - - msub(GetReg(Node), TMP2, Divisor, TMP1); + mov(EmitSize, TMP1, Lower); + bfi(EmitSize, TMP1, Upper, 32, 32); + udiv(EmitSize, TMP2, TMP1, Divisor); + msub(EmitSize, Dst, TMP2, Divisor, TMP1); break; } case 8: { - auto Upper64Bit = GetReg(Op->Upper.ID()); - auto Lower64Bit = GetReg(Op->Lower.ID()); - auto Divisor = GetReg(Op->Divisor.ID()); - Label Only64Bit{}; - Label LongDIVRet{}; + ARMEmitter::ForwardLabel Only64Bit{}; + ARMEmitter::ForwardLabel LongDIVRet{}; // Check the upper bits for zero // If the upper bits are zero then we can do a 64-bit divide - cbz(Upper64Bit, &Only64Bit); + cbz(EmitSize, Upper, &Only64Bit); // Long divide { - mov(x0, Upper64Bit); - mov(x1, Lower64Bit); - mov(x2, Divisor); + mov(EmitSize, ARMEmitter::Reg::r0, Upper); + mov(EmitSize, ARMEmitter::Reg::r1, Lower); + mov(EmitSize, ARMEmitter::Reg::r2, Divisor); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LUREMHandler))); - blr(x3); + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LUREMHandler)); + blr(ARMEmitter::Reg::r3); // Move result to its destination register - mov(GetReg(Node), x0); + mov(EmitSize, Dst, ARMEmitter::Reg::r0); // Skip 64-bit path b(&LongDIVRet); } - bind(&Only64Bit); + Bind(&Only64Bit); // 64-Bit only { - udiv(TMP1, Lower64Bit, Divisor); - msub(GetReg(Node), TMP1, Divisor, Lower64Bit); + udiv(EmitSize, TMP1, Lower, Divisor); + msub(EmitSize, Dst, TMP1, Divisor, Lower); } - bind(&LongDIVRet); + Bind(&LongDIVRet); break; } default: @@ -894,145 +861,136 @@ DEF_OP(Not) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 4: - mvn(GetReg(Node), GetReg(Op->Src.ID())); - break; - case 8: - mvn(GetReg(Node), GetReg(Op->Src.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unsupported Not size: {}", OpSize); - } + LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + + mvn(EmitSize, Dst, Src); } DEF_OP(Popcount) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + switch (OpSize) { case 0x1: - fmov(VTMP1.S(), GetReg(Op->Src.ID())); + fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src); // only use lowest byte - cnt(VTMP1.V8B(), VTMP1.V8B()); + cnt(FEXCore::ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D()); break; case 0x2: - fmov(VTMP1.S(), GetReg(Op->Src.ID())); - cnt(VTMP1.V8B(), VTMP1.V8B()); + fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src); + cnt(FEXCore::ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D()); // only count two lowest bytes - addp(VTMP1.V8B(), VTMP1.V8B(), VTMP1.V8B()); + addp(FEXCore::ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D(), VTMP1.D()); break; case 0x4: - fmov(VTMP1.S(), GetReg(Op->Src.ID())); - cnt(VTMP1.V8B(), VTMP1.V8B()); + fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src); + cnt(FEXCore::ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D()); // fmov has zero extended, unused bytes are zero - addv(VTMP1.B(), VTMP1.V8B()); + addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D()); break; case 0x8: - fmov(VTMP1.D(), GetReg(Op->Src.ID())); - cnt(VTMP1.V8B(), VTMP1.V8B()); + fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), Src); + cnt(FEXCore::ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D()); // fmov has zero extended, unused bytes are zero - addv(VTMP1.B(), VTMP1.V8B()); + addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D()); break; default: LOGMAN_MSG_A_FMT("Unsupported Popcount size: {}", OpSize); } - auto Dst = GetReg(Node); - umov(Dst.W(), VTMP1.B(), 0); + umov(Dst, VTMP1, 0); } DEF_OP(FindLSB) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - auto Dst = GetReg(Node); - auto Src = GetReg(Op->Src.ID()); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + if (OpSize != 8) { - ubfx(TMP1, Src, 0, OpSize * 8); - cmp(TMP1, 0); - rbit(TMP1, TMP1); - clz(Dst, TMP1); - csinv(Dst, Dst, xzr, ne); + ubfx(EmitSize, TMP1, Src, 0, OpSize * 8); + cmp(EmitSize, TMP1, 0); + rbit(EmitSize, TMP1, TMP1); } else { - rbit(TMP1, Src); - cmp(Src, 0); - clz(Dst, TMP1); - csinv(Dst, Dst, xzr, ne); + rbit(EmitSize, TMP1, Src); + cmp(EmitSize, Src, 0); } + + clz(EmitSize, Dst, TMP1); + csinv(EmitSize, Dst, Dst, ARMEmitter::Reg::zr, ARMEmitter::Condition::CC_NE); + } DEF_OP(FindMSB) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - auto Dst = GetReg(Node); - switch (OpSize) { - case 2: - movz(TMP1, OpSize * 8 - 1); - lsl(Dst.W(), GetReg(Op->Src.ID()), 16); - orr(Dst.W(), Dst.W(), 0x8000); - clz(Dst.W(), Dst.W()); - sub(Dst, TMP1, Dst); - break; - case 4: - movz(TMP1, OpSize * 8 - 1); - clz(Dst.W(), GetReg(Op->Src.ID())); - sub(Dst, TMP1, Dst); - break; - case 8: - movz(TMP1, OpSize * 8 - 1); - clz(Dst, GetReg(Op->Src.ID())); - sub(Dst, TMP1, Dst); - break; - default: - LOGMAN_MSG_A_FMT("Unknown FindMSB size: {}", OpSize); - break; + LOGMAN_THROW_AA_FMT(OpSize == 2 || OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + + movz(ARMEmitter::Size::i64Bit, TMP1, OpSize * 8 - 1); + + if (OpSize == 2) { + lsl(EmitSize, Dst, Src, 16); + orr(EmitSize, Dst, Dst, 0x8000); + clz(EmitSize, Dst, Dst); } + else { + clz(EmitSize, Dst, Src); + } + + sub(ARMEmitter::Size::i64Bit, Dst, TMP1, Dst); } DEF_OP(FindTrailingZeros) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 2: - rbit(GetReg(Node), GetReg(Op->Src.ID())); - orr(GetReg(Node), GetReg(Node), 0x8000); - clz(GetReg(Node), GetReg(Node)); - break; - case 4: - rbit(GetReg(Node), GetReg(Op->Src.ID())); - clz(GetReg(Node), GetReg(Node)); - break; - case 8: - rbit(GetReg(Node), GetReg(Op->Src.ID())); - clz(GetReg(Node), GetReg(Node)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown FindTrailingZeros size: {}", OpSize); - break; + LOGMAN_THROW_AA_FMT(OpSize == 2 || OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + + rbit(EmitSize, Dst, Src); + + if (OpSize == 2) { + orr(EmitSize, Dst, Dst, 0x8000); } + + clz(EmitSize, Dst, Dst); } DEF_OP(CountLeadingZeroes) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 2: - lsl(GetReg(Node), GetReg(Op->Src.ID()), 16); - orr(GetReg(Node), GetReg(Node), 0x8000); - clz(GetReg(Node), GetReg(Node)); - break; - case 4: - clz(GetReg(Node), GetReg(Op->Src.ID())); - break; - case 8: - clz(GetReg(Node), GetReg(Op->Src.ID())); - break; - default: - LOGMAN_MSG_A_FMT("Unknown CountLeadingZeroes size: {}", OpSize); - break; + LOGMAN_THROW_AA_FMT(OpSize == 2 || OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + + if (OpSize == 2) { + lsl(EmitSize, Dst, Src, 16); + orr(EmitSize, Dst, Dst, 0x8000); + clz(EmitSize, Dst, Dst); + } + else { + clz(EmitSize, Dst, Src); } } @@ -1040,20 +998,15 @@ DEF_OP(Rev) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 2: - rev(GetReg(Node), GetReg(Op->Src.ID())); - lsr(GetReg(Node), GetReg(Node), 16); - break; - case 4: - rev(GetReg(Node), GetReg(Op->Src.ID())); - break; - case 8: - rev(GetReg(Node), GetReg(Op->Src.ID())); - break; - default: - LOGMAN_MSG_A_FMT("Unknown REV size: {}", OpSize); - break; + LOGMAN_THROW_AA_FMT(OpSize == 2 || OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize); + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + + rev(EmitSize, Dst, Src); + if (OpSize == 2) { + lsr(EmitSize, Dst, Dst, 16); } } @@ -1061,24 +1014,20 @@ DEF_OP(Bfi) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; - switch (OpSize) { - case 1: - case 2: - case 4: { - auto Dst = GetReg(Node); - mov(TMP1.W(), GetReg(Op->Dest.ID())); - bfi(TMP1.W(), GetReg(Op->Src.ID()), Op->lsb, Op->Width); - ubfx(Dst, TMP1.W(), 0, OpSize * 8); - break; - } - case 8: - mov(TMP1, GetReg(Op->Dest.ID())); - bfi(TMP1, GetReg(Op->Src.ID()), Op->lsb, Op->Width); - mov(GetReg(Node), TMP1); - break; - default: - LOGMAN_MSG_A_FMT("Unknown BFI size: {}", OpSize); - break; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Dst = GetReg(Node); + const auto SrcDst = GetReg(Op->Dest.ID()); + const auto Src = GetReg(Op->Src.ID()); + + mov(EmitSize, TMP1, SrcDst); + bfi(EmitSize, TMP1, Src, Op->lsb, Op->Width); + + if (OpSize == 8) { + mov(EmitSize, Dst, TMP1.R()); + } + else { + ubfx(EmitSize, Dst, TMP1, 0, OpSize * 8); } } @@ -1087,66 +1036,69 @@ DEF_OP(Bfe) { LOGMAN_THROW_AA_FMT(IROp->Size <= 8, "OpSize is too large for BFE: {}", IROp->Size); LOGMAN_THROW_AA_FMT(Op->Width != 0, "Invalid BFE width of 0"); - auto Dst = GetReg(Node); - ubfx(Dst, GetReg(Op->Src.ID()), Op->lsb, Op->Width); + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); + + ubfx(ARMEmitter::Size::i64Bit, Dst, Src, Op->lsb, Op->Width); } DEF_OP(Sbfe) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; + const auto Dst = GetReg(Node); + const auto Src = GetReg(Op->Src.ID()); - auto Dst = GetReg(Node); - if (OpSize == 8) { - sbfx(Dst, GetReg(Op->Src.ID()), Op->lsb, Op->Width); - } else { - LogMan::Msg::DFmt("Unimplemented Sbfe size"); - } + sbfx(ARMEmitter::Size::i64Bit, Dst, Src, Op->lsb, Op->Width); } -#define GRCMP(Node) (Op->CompareSize == 4 ? GetReg(Node) : GetReg(Node)) - -#define GRFCMP(Node) (Op->CompareSize == 4 ? GetVReg(Node).S() : GetVReg(Node).D()) - -Condition MapSelectCC(IR::CondClassType Cond) { +ARMEmitter::Condition MapSelectCC(IR::CondClassType Cond) { switch (Cond.Val) { - case FEXCore::IR::COND_EQ: return Condition::eq; - case FEXCore::IR::COND_NEQ: return Condition::ne; - case FEXCore::IR::COND_SGE: return Condition::ge; - case FEXCore::IR::COND_SLT: return Condition::lt; - case FEXCore::IR::COND_SGT: return Condition::gt; - case FEXCore::IR::COND_SLE: return Condition::le; - case FEXCore::IR::COND_UGE: return Condition::cs; - case FEXCore::IR::COND_ULT: return Condition::cc; - case FEXCore::IR::COND_UGT: return Condition::hi; - case FEXCore::IR::COND_ULE: return Condition::ls; - case FEXCore::IR::COND_FLU: return Condition::lt; - case FEXCore::IR::COND_FGE: return Condition::ge; - case FEXCore::IR::COND_FLEU:return Condition::le; - case FEXCore::IR::COND_FGT: return Condition::gt; - case FEXCore::IR::COND_FU: return Condition::vs; - case FEXCore::IR::COND_FNU: return Condition::vc; + case FEXCore::IR::COND_EQ: return ARMEmitter::Condition::CC_EQ; + case FEXCore::IR::COND_NEQ: return ARMEmitter::Condition::CC_NE; + case FEXCore::IR::COND_SGE: return ARMEmitter::Condition::CC_GE; + case FEXCore::IR::COND_SLT: return ARMEmitter::Condition::CC_LT; + case FEXCore::IR::COND_SGT: return ARMEmitter::Condition::CC_GT; + case FEXCore::IR::COND_SLE: return ARMEmitter::Condition::CC_LE; + case FEXCore::IR::COND_UGE: return ARMEmitter::Condition::CC_CS; + case FEXCore::IR::COND_ULT: return ARMEmitter::Condition::CC_CC; + case FEXCore::IR::COND_UGT: return ARMEmitter::Condition::CC_HI; + case FEXCore::IR::COND_ULE: return ARMEmitter::Condition::CC_LS; + case FEXCore::IR::COND_FLU: return ARMEmitter::Condition::CC_LT; + case FEXCore::IR::COND_FGE: return ARMEmitter::Condition::CC_GE; + case FEXCore::IR::COND_FLEU:return ARMEmitter::Condition::CC_LE; + case FEXCore::IR::COND_FGT: return ARMEmitter::Condition::CC_GT; + case FEXCore::IR::COND_FU: return ARMEmitter::Condition::CC_VS; + case FEXCore::IR::COND_FNU: return ARMEmitter::Condition::CC_VC; case FEXCore::IR::COND_VS: case FEXCore::IR::COND_VC: case FEXCore::IR::COND_MI: case FEXCore::IR::COND_PL: default: LOGMAN_MSG_A_FMT("Unsupported compare type"); - return Condition::nv; + return ARMEmitter::Condition::CC_NV; } } DEF_OP(Select) { auto Op = IROp->C(); + const uint8_t OpSize = IROp->Size; + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto CompareEmitSize = Op->CompareSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; uint64_t Const; if (IsGPR(Op->Cmp1.ID())) { + const auto Src1 = GetReg(Op->Cmp1.ID()); + if (IsInlineConstant(Op->Cmp2, &Const)) - cmp(GRCMP(Op->Cmp1.ID()), Const); - else - cmp(GRCMP(Op->Cmp1.ID()), GRCMP(Op->Cmp2.ID())); + cmp(CompareEmitSize, Src1, Const); + else { + const auto Src2 = GetReg(Op->Cmp2.ID()); + cmp(CompareEmitSize, Src1, Src2); + } } else if (IsFPR(Op->Cmp1.ID())) { - fcmp(GRFCMP(Op->Cmp1.ID()), GRFCMP(Op->Cmp2.ID())); + const auto Src1 = GetVReg(Op->Cmp1.ID()); + const auto Src2 = GetVReg(Op->Cmp2.ID()); + fcmp(Op->CompareSize == 8 ? ARMEmitter::ScalarRegSize::i64Bit : ARMEmitter::ScalarRegSize::i32Bit, Src1, Src2); } else { LOGMAN_MSG_A_FMT("Select: Expected GPR or FPR"); } @@ -1157,13 +1109,15 @@ DEF_OP(Select) { bool is_const_true = IsInlineConstant(Op->TrueVal, &const_true); bool is_const_false = IsInlineConstant(Op->FalseVal, &const_false); + ARMEmitter::Register Dst = GetReg(Node); + if (is_const_true || is_const_false) { if (is_const_false != true || is_const_true != true || const_true != 1 || const_false != 0) { LOGMAN_MSG_A_FMT("Select: Unsupported compare inline parameters"); } - cset(GRS(Node), cc); + cset(EmitSize, Dst, cc); } else { - csel(GRS(Node), GRS(Op->TrueVal.ID()), GRS(Op->FalseVal.ID()), cc); + csel(EmitSize, Dst, GetReg(Op->TrueVal.ID()), GetReg(Op->FalseVal.ID()), cc); } } @@ -1178,21 +1132,22 @@ DEF_OP(VExtractToGPR) { const auto Offset = ElementSizeBits * Op->Index; const auto Is256Bit = Offset >= SSERegBitSize; + const auto Dst = GetReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); - const auto PerformMove = [&](const aarch64::VRegister& reg, int index) { + const auto PerformMove = [&](const ARMEmitter::VRegister reg, int index) { switch (OpSize) { case 1: - umov(GetReg(Node), reg.V16B(), index); + umov(Dst, Vector, index); break; case 2: - umov(GetReg(Node), reg.V8H(), index); + umov(Dst, Vector, index); break; case 4: - umov(GetReg(Node), reg.V4S(), index); + umov(Dst, Vector, index); break; case 8: - umov(GetReg(Node), reg.V2D(), index); + umov(Dst, Vector, index); break; default: LOGMAN_MSG_A_FMT("Unhandled ExtractElementSize: {}", OpSize); @@ -1217,9 +1172,9 @@ DEF_OP(VExtractToGPR) { // We need to use the upper 128-bit lane, so lets move it down. // Inverting our dedicated predicate for 128-bit operations selects // all of the top lanes. We can then compact those into a temporary. - const auto CompactPred = p0; - not_(CompactPred.VnB(), PRED_TMP_32B.Zeroing(), PRED_TMP_16B.VnB()); - compact(VTMP1.Z().VnD(), CompactPred, Vector.Z().VnD()); + const auto CompactPred = ARMEmitter::PReg::p0; + not_(CompactPred, PRED_TMP_32B, PRED_TMP_16B); + compact(ARMEmitter::SubRegSize::i64Bit, VTMP1, CompactPred, Vector); // Sanitize the zero-based index to work on the now-moved // upper half of the vector. @@ -1246,88 +1201,73 @@ DEF_OP(VExtractToGPR) { DEF_OP(Float_ToGPR_ZS) { auto Op = IROp->C(); - aarch64::Register Dst{}; - aarch64::VRegister Src{}; + + ARMEmitter::Register Dst = GetReg(Node); + ARMEmitter::VRegister Src = GetVReg(Op->Scalar.ID()); + const auto DestSize = IROp->Size == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + if (Op->SrcElementSize == 8) { - Src = GetVReg(Op->Scalar.ID()).D(); + fcvtzs(DestSize, Dst, Src.D()); } else { - Src = GetVReg(Op->Scalar.ID()).S(); + fcvtzs(DestSize, Dst, Src.S()); } - - if (IROp->Size == 8) { - Dst = GetReg(Node); - } - else { - Dst = GetReg(Node); - } - - fcvtzs(Dst, Src); } DEF_OP(Float_ToGPR_S) { auto Op = IROp->C(); - aarch64::Register Dst{}; - aarch64::VRegister Src{}; + ARMEmitter::Register Dst = GetReg(Node); + ARMEmitter::VRegister Src = GetVReg(Op->Scalar.ID()); + const auto DestSize = IROp->Size == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + if (Op->SrcElementSize == 8) { - frinti(VTMP1.D(), GetVReg(Op->Scalar.ID()).D()); - Src = VTMP1.D(); + frinti(VTMP1.D(), Src.D()); + fcvtzs(DestSize, Dst, VTMP1.D()); } else { - frinti(VTMP1.S(), GetVReg(Op->Scalar.ID()).S()); - Src = VTMP1.S(); + frinti(VTMP1.S(), Src.S()); + fcvtzs(DestSize, Dst, VTMP1.S()); } - - if (IROp->Size == 8) { - Dst = GetReg(Node); - } - else { - Dst = GetReg(Node); - } - - fcvtzs(Dst, Src); } DEF_OP(FCmp) { auto Op = IROp->C(); + const auto EmitSubSize = Op->ElementSize == 8 ? ARMEmitter::ScalarRegSize::i64Bit : ARMEmitter::ScalarRegSize::i32Bit; - if (Op->ElementSize == 4) { - fcmp(GetVReg(Op->Scalar1.ID()).S(), GetVReg(Op->Scalar2.ID()).S()); - } - else { - fcmp(GetVReg(Op->Scalar1.ID()).D(), GetVReg(Op->Scalar2.ID()).D()); - } - auto Dst = GetReg(Node); + ARMEmitter::Register Dst = GetReg(Node); + ARMEmitter::VRegister Scalar1 = GetVReg(Op->Scalar1.ID()); + ARMEmitter::VRegister Scalar2 = GetVReg(Op->Scalar2.ID()); + fcmp(EmitSubSize, Scalar1, Scalar2); bool set = false; if (Op->Flags & (1 << IR::FCMP_FLAG_EQ)) { LOGMAN_THROW_AA_FMT(IR::FCMP_FLAG_EQ == 0, "IR::FCMP_FLAG_EQ must equal 0"); // EQ or unordered - cset(Dst, Condition::eq); // Z = 1 - csinc(Dst, Dst, xzr, Condition::vc); // IF !V ? Z : 1 + cset(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Condition::CC_EQ); // Z = 1 + csinc(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::zr, ARMEmitter::Condition::CC_VC); // IF !V ? Z : 1 set = true; } if (Op->Flags & (1 << IR::FCMP_FLAG_LT)) { // LT or unordered - cset(TMP2, Condition::lt); + cset(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Condition::CC_LT); if (!set) { - lsl(Dst, TMP2, IR::FCMP_FLAG_LT); + lsl(ARMEmitter::Size::i64Bit, Dst, TMP2, IR::FCMP_FLAG_LT); set = true; } else { - bfi(Dst, TMP2, IR::FCMP_FLAG_LT, 1); + bfi(ARMEmitter::Size::i64Bit, Dst, TMP2, IR::FCMP_FLAG_LT, 1); } } if (Op->Flags & (1 << IR::FCMP_FLAG_UNORDERED)) { - cset(TMP2, Condition::vs); + cset(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Condition::CC_VS); if (!set) { - lsl(Dst, TMP2, IR::FCMP_FLAG_UNORDERED); + lsl(ARMEmitter::Size::i64Bit, Dst, TMP2, IR::FCMP_FLAG_UNORDERED); set = true; } else { - bfi(Dst, TMP2, IR::FCMP_FLAG_UNORDERED, 1); + bfi(ARMEmitter::Size::i64Bit, Dst, TMP2, IR::FCMP_FLAG_UNORDERED, 1); } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/Arm64Relocations.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/Arm64Relocations.cpp index 98fecf178..f85564db1 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/Arm64Relocations.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/Arm64Relocations.cpp @@ -9,7 +9,7 @@ $end_info$ #include "Interface/HLE/Thunks/Thunks.h" namespace FEXCore::CPU { - + uint64_t Arm64JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) { switch (Op) { case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER: @@ -22,18 +22,18 @@ uint64_t Arm64JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiter return ~0ULL; } -void Arm64JITCore::InsertNamedThunkRelocation(vixl::aarch64::Register Reg, const IR::SHA256Sum &Sum) { +void Arm64JITCore::InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR::SHA256Sum &Sum) { Relocation MoveABI{}; MoveABI.NamedThunkMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE; // Offset is the offset from the entrypoint of the block auto CurrentCursor = GetCursorAddress(); MoveABI.NamedThunkMove.Offset = CurrentCursor - GuestEntry; MoveABI.NamedThunkMove.Symbol = Sum; - MoveABI.NamedThunkMove.RegisterIndex = Reg.GetCode(); + MoveABI.NamedThunkMove.RegisterIndex = Reg.Idx(); uint64_t Pointer = reinterpret_cast(EmitterCTX->ThunkHandler->LookupThunk(Sum)); - LoadConstant(Reg, Pointer, EmitterCTX->Config.CacheObjectCodeCompilation()); + LoadConstant(ARMEmitter::Size::i64Bit, Reg, Pointer, EmitterCTX->Config.CacheObjectCodeCompilation()); Relocations.emplace_back(MoveABI); } @@ -41,7 +41,7 @@ Arm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXC uint64_t Pointer = GetNamedSymbolLiteral(Op); Arm64JITCore::NamedSymbolLiteralPair Lit { - .Lit = Literal(Pointer), + .Lit = Pointer, .MoveABI = { .NamedSymbolLiteral = { .Header = { @@ -60,20 +60,21 @@ void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit) { auto CurrentCursor = GetCursorAddress(); Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - GuestEntry; - place(&Lit.Lit); + Bind(&Lit.Loc); + dc64(Lit.Lit); Relocations.emplace_back(Lit.MoveABI); } -void Arm64JITCore::InsertGuestRIPMove(vixl::aarch64::Register Reg, uint64_t Constant) { +void Arm64JITCore::InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant) { Relocation MoveABI{}; MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE; // Offset is the offset from the entrypoint of the block auto CurrentCursor = GetCursorAddress(); MoveABI.GuestRIPMove.Offset = CurrentCursor - GuestEntry; MoveABI.GuestRIPMove.GuestRIP = Constant; - MoveABI.GuestRIPMove.RegisterIndex = Reg.GetCode(); + MoveABI.GuestRIPMove.RegisterIndex = Reg.Idx(); - LoadConstant(Reg, Constant, EmitterCTX->Config.CacheObjectCodeCompilation()); + LoadConstant(ARMEmitter::Size::i64Bit, Reg, Constant, EmitterCTX->Config.CacheObjectCodeCompilation()); Relocations.emplace_back(MoveABI); } @@ -87,11 +88,10 @@ bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uin case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: { uint64_t Pointer = GetNamedSymbolLiteral(Reloc->NamedSymbolLiteral.Symbol); // Relocation occurs at the cursorEntry + offset relative to that cursor - GetBuffer()->SetCursorOffset(CursorEntry + Reloc->NamedSymbolLiteral.Offset); + SetCursorOffset(CursorEntry + Reloc->NamedSymbolLiteral.Offset); // Generate a literal so we can place it - Literal Lit(Pointer); - place(&Lit); + dc64(Pointer); DataIndex += sizeof(Reloc->NamedSymbolLiteral); break; @@ -103,8 +103,8 @@ bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uin } // Relocation occurs at the cursorEntry + offset relative to that cursor. - GetBuffer()->SetCursorOffset(CursorEntry + Reloc->NamedThunkMove.Offset); - LoadConstant(vixl::aarch64::XRegister(Reloc->NamedThunkMove.RegisterIndex), Pointer, true); + SetCursorOffset(CursorEntry + Reloc->NamedThunkMove.Offset); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc->NamedThunkMove.RegisterIndex), Pointer, true); DataIndex += sizeof(Reloc->NamedThunkMove); break; } @@ -117,8 +117,8 @@ bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uin } // Relocation occurs at the cursorEntry + offset relative to that cursor. - GetBuffer()->SetCursorOffset(CursorEntry + Reloc->GuestRIPMove.Offset); - LoadConstant(vixl::aarch64::XRegister(Reloc->GuestRIPMove.RegisterIndex), Pointer, true); + SetCursorOffset(CursorEntry + Reloc->GuestRIPMove.Offset); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc->GuestRIPMove.RegisterIndex), Pointer, true); DataIndex += sizeof(Reloc->GuestRIPMove); break; } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp index 9b7cb76be..9f8284fd6 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp @@ -5,917 +5,436 @@ $end_info$ */ #include "Interface/Context/Context.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/JIT/Arm64/JITClass.h" namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(CASPair) { auto Op = IROp->C(); + LOGMAN_THROW_AA_FMT(IROp->ElementSize == 4 || IROp->ElementSize == 8, "Wrong element size"); // Size is the size of each pair element - auto Dst = GetRegPair(Node); - auto Expected = GetRegPair(Op->Expected.ID()); - auto Desired = GetRegPair(Op->Desired.ID()); - auto MemSrc = GetReg(Op->Addr.ID()); + auto Dst = GetRegPair(Node); + auto Expected = GetRegPair(Op->Expected.ID()); + auto Desired = GetRegPair(Op->Desired.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + const auto EmitSize = IROp->ElementSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; if (CTX->HostFeatures.SupportsAtomics) { - mov(TMP3, Expected.first); - mov(TMP4, Expected.second); + mov(EmitSize, TMP3, Expected.first); + mov(EmitSize, TMP4, Expected.second); - switch (IROp->ElementSize) { - case 4: - caspal(TMP3.W(), TMP4.W(), Desired.first.W(), Desired.second.W(), MemOperand(MemSrc)); - mov(Dst.first.W(), TMP3.W()); - mov(Dst.second.W(), TMP4.W()); - break; - case 8: - caspal(TMP3.X(), TMP4.X(), Desired.first.X(), Desired.second.X(), MemOperand(MemSrc)); - mov(Dst.first, TMP3); - mov(Dst.second, TMP4); - break; - default: LOGMAN_MSG_A_FMT("Unsupported: {}", IROp->ElementSize); - } + caspal(EmitSize, TMP3, TMP4, Desired.first, Desired.second, MemSrc); + mov(EmitSize, Dst.first, TMP3.R()); + mov(EmitSize, Dst.second, TMP4.R()); } else { - switch (IROp->ElementSize) { - case 4: { - aarch64::Label LoopTop; - aarch64::Label LoopNotExpected; - aarch64::Label LoopExpected; - bind(&LoopTop); + ARMEmitter::BackwardLabel LoopTop; + ARMEmitter::ForwardLabel LoopNotExpected; + ARMEmitter::ForwardLabel LoopExpected; + Bind(&LoopTop); - ldaxp(TMP2.W(), TMP3.W(), MemOperand(MemSrc)); - cmp(TMP2.W(), Expected.first.W()); - ccmp(TMP3.W(), Expected.second.W(), NoFlag, Condition::eq); - b(&LoopNotExpected, Condition::ne); - stlxp(TMP2.W(), Desired.first.W(), Desired.second.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - mov(Dst.first.W(), Expected.first.W()); - mov(Dst.second.W(), Expected.second.W()); + ldaxp(EmitSize, TMP2, TMP3, MemSrc); + cmp(EmitSize, TMP2, Expected.first); + ccmp(EmitSize, TMP3, Expected.second, ARMEmitter::StatusFlags::None, ARMEmitter::Condition::CC_EQ); + b(ARMEmitter::Condition::CC_NE, &LoopNotExpected); + stlxp(EmitSize, TMP2, Desired.first, Desired.second, MemSrc); + cbnz(EmitSize, TMP2, &LoopTop); + mov(EmitSize, Dst.first, Expected.first); + mov(EmitSize, Dst.second, Expected.second); - b(&LoopExpected); + b(&LoopExpected); - bind(&LoopNotExpected); - mov(Dst.first.W(), TMP2.W()); - mov(Dst.second.W(), TMP3.W()); - // exclusive monitor needs to be cleared here - // Might have hit the case where ldaxr was hit but stlxr wasn't - clrex(); - bind(&LoopExpected); - break; - } - case 8: { - aarch64::Label LoopTop; - aarch64::Label LoopNotExpected; - aarch64::Label LoopExpected; - bind(&LoopTop); - - ldaxp(TMP2.X(), TMP3.X(), MemOperand(MemSrc)); - cmp(TMP2.X(), Expected.first.X()); - ccmp(TMP3.X(), Expected.second.X(), NoFlag, Condition::eq); - b(&LoopNotExpected, Condition::ne); - stlxp(TMP2.X(), Desired.first.X(), Desired.second.X(), MemOperand(MemSrc)); - cbnz(TMP2.X(), &LoopTop); - mov(Dst.first.X(), Expected.first.X()); - mov(Dst.second.X(), Expected.second.X()); - - b(&LoopExpected); - - bind(&LoopNotExpected); - mov(Dst.first.X(), TMP2.X()); - mov(Dst.second.X(), TMP3.X()); - // exclusive monitor needs to be cleared here - // Might have hit the case where ldaxr was hit but stlxr wasn't - clrex(); - bind(&LoopExpected); - break; - } - default: LOGMAN_MSG_A_FMT("Unsupported: {}", IROp->ElementSize); - } + Bind(&LoopNotExpected); + mov(EmitSize, Dst.first, TMP2.R()); + mov(EmitSize, Dst.second, TMP3.R()); + // exclusive monitor needs to be cleared here + // Might have hit the case where ldaxr was hit but stlxr wasn't + clrex(); + Bind(&LoopExpected); } } DEF_OP(CAS) { auto Op = IROp->C(); uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); // DataSrc = *Src1 // if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc // This will write to memory! Careful! - auto Expected = GetReg(Op->Expected.ID()); - auto Desired = GetReg(Op->Desired.ID()); - auto MemSrc = GetReg(Op->Addr.ID()); + auto Expected = GetReg(Op->Expected.ID()); + auto Desired = GetReg(Op->Desired.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - mov(TMP2, Expected); - switch (OpSize) { - case 1: casalb(TMP2.W(), Desired.W(), MemOperand(MemSrc)); break; - case 2: casalh(TMP2.W(), Desired.W(), MemOperand(MemSrc)); break; - case 4: casal(TMP2.W(), Desired.W(), MemOperand(MemSrc)); break; - case 8: casal(TMP2.X(), Desired.X(), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unsupported: {}", OpSize); - } - mov(GetReg(Node), TMP2); + mov(EmitSize, TMP2, Expected); + casal(SubEmitSize, TMP2, Desired, MemSrc); + mov(EmitSize, GetReg(Node), TMP2.R()); } else { - switch (OpSize) { - case 1: { - aarch64::Label LoopTop; - aarch64::Label LoopNotExpected; - aarch64::Label LoopExpected; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - cmp(TMP2.W(), Operand(Expected.W(), Extend::UXTB)); - b(&LoopNotExpected, Condition::ne); - stlxrb(TMP3.W(), Desired.W(), MemOperand(MemSrc)); - cbnz(TMP3.W(), &LoopTop); - mov(GetReg(Node), Expected.W()); - b(&LoopExpected); - - bind(&LoopNotExpected); - mov(GetReg(Node), TMP2.W()); - // exclusive monitor needs to be cleared here - // Might have hit the case where ldaxr was hit but stlxr wasn't - clrex(); - bind(&LoopExpected); - break; - } - case 2: { - aarch64::Label LoopTop; - aarch64::Label LoopNotExpected; - aarch64::Label LoopExpected; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - cmp(TMP2.W(), Operand(Expected.W(), Extend::UXTH)); - b(&LoopNotExpected, Condition::ne); - stlxrh(TMP3.W(), Desired.W(), MemOperand(MemSrc)); - cbnz(TMP3.W(), &LoopTop); - mov(GetReg(Node), Expected.W()); - b(&LoopExpected); - - bind(&LoopNotExpected); - mov(GetReg(Node), TMP2.W()); - // exclusive monitor needs to be cleared here - // Might have hit the case where ldaxr was hit but stlxr wasn't - clrex(); - bind(&LoopExpected); - break; - } - case 4: { - aarch64::Label LoopTop; - aarch64::Label LoopNotExpected; - aarch64::Label LoopExpected; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - cmp(TMP2.W(), Expected.W()); - b(&LoopNotExpected, Condition::ne); - stlxr(TMP3.W(), Desired.W(), MemOperand(MemSrc)); - cbnz(TMP3.W(), &LoopTop); - mov(GetReg(Node), Expected.W()); - b(&LoopExpected); - - bind(&LoopNotExpected); - mov(GetReg(Node), TMP2.W()); - // exclusive monitor needs to be cleared here - // Might have hit the case where ldaxr was hit but stlxr wasn't - clrex(); - bind(&LoopExpected); - break; - } - case 8: { - aarch64::Label LoopTop; - aarch64::Label LoopNotExpected; - aarch64::Label LoopExpected; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - cmp(TMP2, Expected); - b(&LoopNotExpected, Condition::ne); - stlxr(TMP2, Desired, MemOperand(MemSrc)); - cbnz(TMP2, &LoopTop); - mov(GetReg(Node), Expected); - b(&LoopExpected); - - bind(&LoopNotExpected); - mov(GetReg(Node), TMP2); - // exclusive monitor needs to be cleared here - // Might have hit the case where ldaxr was hit but stlxr wasn't - clrex(); - bind(&LoopExpected); - - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", OpSize); + ARMEmitter::BackwardLabel LoopTop; + ARMEmitter::ForwardLabel LoopNotExpected; + ARMEmitter::ForwardLabel LoopExpected; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + if (OpSize == 1) { + cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTB, 0); } + else if (OpSize == 2) { + cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTH, 0); + } + else { + cmp(EmitSize, TMP2, Expected); + } + b(ARMEmitter::Condition::CC_NE, &LoopNotExpected); + stlxr(SubEmitSize, TMP3, Desired, MemSrc); + cbnz(EmitSize, TMP3, &LoopTop); + mov(EmitSize, GetReg(Node), Expected); + b(&LoopExpected); + + Bind(&LoopNotExpected); + mov(EmitSize, GetReg(Node), TMP2.R()); + // exclusive monitor needs to be cleared here + // Might have hit the case where ldaxr was hit but stlxr wasn't + clrex(); + Bind(&LoopExpected); } } DEF_OP(AtomicAdd) { auto Op = IROp->C(); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); - auto MemSrc = GetReg(Op->Addr.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - switch (IROp->Size) { - case 1: staddlb(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 2: staddlh(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 4: staddl(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 8: staddl(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + staddl(SubEmitSize, Src, MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - add(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - add(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - add(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - add(TMP2, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP2, TMP2, MemOperand(MemSrc)); - cbnz(TMP2, &LoopTop); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + add(EmitSize, TMP2, TMP2, Src); + stlxr(SubEmitSize, TMP2, TMP2, MemSrc); + cbnz(EmitSize, TMP2, &LoopTop); } } DEF_OP(AtomicSub) { auto Op = IROp->C(); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); - auto MemSrc = GetReg(Op->Addr.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - neg(TMP2, GetReg(Op->Value.ID())); - switch (IROp->Size) { - case 1: staddlb(TMP2.W(), MemOperand(MemSrc)); break; - case 2: staddlh(TMP2.W(), MemOperand(MemSrc)); break; - case 4: staddl(TMP2.W(), MemOperand(MemSrc)); break; - case 8: staddl(TMP2.X(), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + neg(EmitSize, TMP2, Src); + staddl(SubEmitSize, TMP2, MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - sub(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - sub(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - sub(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - sub(TMP2, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP2, TMP2, MemOperand(MemSrc)); - cbnz(TMP2, &LoopTop); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + sub(EmitSize, TMP2, TMP2, Src); + stlxr(SubEmitSize, TMP2, TMP2, MemSrc); + cbnz(EmitSize, TMP2, &LoopTop); } } DEF_OP(AtomicAnd) { auto Op = IROp->C(); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); - auto MemSrc = GetReg(Op->Addr.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - mvn(TMP2, GetReg(Op->Value.ID())); - switch (IROp->Size) { - case 1: stclrlb(TMP2.W(), MemOperand(MemSrc)); break; - case 2: stclrlh(TMP2.W(), MemOperand(MemSrc)); break; - case 4: stclrl(TMP2.W(), MemOperand(MemSrc)); break; - case 8: stclrl(TMP2.X(), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + mvn(EmitSize, TMP2, Src); + stclrl(SubEmitSize, TMP2, MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - and_(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - and_(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - and_(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - and_(TMP2, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP2, TMP2, MemOperand(MemSrc)); - cbnz(TMP2, &LoopTop); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + and_(EmitSize, TMP2, TMP2, Src); + stlxr(SubEmitSize, TMP2, TMP2, MemSrc); + cbnz(EmitSize, TMP2, &LoopTop); } } DEF_OP(AtomicOr) { auto Op = IROp->C(); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); - auto MemSrc = GetReg(Op->Addr.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - switch (IROp->Size) { - case 1: stsetlb(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 2: stsetlh(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 4: stsetl(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 8: stsetl(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + stsetl(SubEmitSize, Src, MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - orr(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - orr(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - orr(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - orr(TMP2, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP2, TMP2, MemOperand(MemSrc)); - cbnz(TMP2, &LoopTop); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + orr(EmitSize, TMP2, TMP2, Src); + stlxr(SubEmitSize, TMP2, TMP2, MemSrc); + cbnz(EmitSize, TMP2, &LoopTop); } } DEF_OP(AtomicXor) { auto Op = IROp->C(); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); - auto MemSrc = GetReg(Op->Addr.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - switch (IROp->Size) { - case 1: steorlb(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 2: steorlh(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 4: steorl(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - case 8: steorl(GetReg(Op->Value.ID()), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + steorl(SubEmitSize, Src, MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - eor(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - eor(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - eor(TMP2.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc)); - cbnz(TMP2.W(), &LoopTop); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - eor(TMP2, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP2, TMP2, MemOperand(MemSrc)); - cbnz(TMP2, &LoopTop); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + eor(EmitSize, TMP2, TMP2, Src); + stlxr(SubEmitSize, TMP2, TMP2, MemSrc); + cbnz(EmitSize, TMP2, &LoopTop); } } DEF_OP(AtomicSwap) { auto Op = IROp->C(); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); - auto MemSrc = GetReg(Op->Addr.ID()); + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - mov(TMP2, GetReg(Op->Value.ID())); - switch (IROp->Size) { - case 1: swpalb(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 2: swpalh(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 4: swpal(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 8: swpal(TMP2.X(), GetReg(Node), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + mov(EmitSize, TMP2, Src); + ldswpal(SubEmitSize, TMP2, GetReg(Node), MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - stlxrb(TMP4.W(), GetReg(Op->Value.ID()), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - uxtb(GetReg(Node), TMP2.W()); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - stlxrh(TMP4.W(), GetReg(Op->Value.ID()), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - uxtw(GetReg(Node), TMP2.W()); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - stlxr(TMP4.W(), GetReg(Op->Value.ID()), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - stlxr(TMP4, GetReg(Op->Value.ID()), MemOperand(MemSrc)); - cbnz(TMP4, &LoopTop); - mov(GetReg(Node), TMP2.X()); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + stlxr(SubEmitSize, TMP4, Src, MemSrc); + cbnz(EmitSize, TMP4, &LoopTop); + ubfm(EmitSize, GetReg(Node), TMP2, 0, OpSize * 8 - 1); } } DEF_OP(AtomicFetchAdd) { auto Op = IROp->C(); - auto MemSrc = GetReg(Op->Addr.ID()); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - switch (IROp->Size) { - case 1: ldaddalb(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 2: ldaddalh(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 4: ldaddal(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 8: ldaddal(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - add(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - add(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - add(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - add(TMP3, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP4, TMP3, MemOperand(MemSrc)); - cbnz(TMP4, &LoopTop); - mov(GetReg(Node), TMP2); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + add(EmitSize, TMP3, TMP2, Src); + stlxr(SubEmitSize, TMP4, TMP3, MemSrc); + cbnz(EmitSize, TMP4, &LoopTop); + mov(EmitSize, GetReg(Node), TMP2.R()); } } DEF_OP(AtomicFetchSub) { auto Op = IROp->C(); - auto MemSrc = GetReg(Op->Addr.ID()); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - neg(TMP2, GetReg(Op->Value.ID())); - switch (IROp->Size) { - case 1: ldaddalb(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 2: ldaddalh(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 4: ldaddal(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 8: ldaddal(TMP2.X(), GetReg(Node), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + neg(EmitSize, TMP2, Src); + ldaddal(SubEmitSize, TMP2, GetReg(Node), MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - sub(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - sub(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - sub(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - sub(TMP3, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP4, TMP3, MemOperand(MemSrc)); - cbnz(TMP4, &LoopTop); - mov(GetReg(Node), TMP2); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + sub(EmitSize, TMP3, TMP2, Src); + stlxr(SubEmitSize, TMP4, TMP3, MemSrc); + cbnz(EmitSize, TMP4, &LoopTop); + mov(EmitSize, GetReg(Node), TMP2.R()); } } DEF_OP(AtomicFetchAnd) { auto Op = IROp->C(); - auto MemSrc = GetReg(Op->Addr.ID()); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - mvn(TMP2, GetReg(Op->Value.ID())); - switch (IROp->Size) { - case 1: ldclralb(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 2: ldclralh(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 4: ldclral(TMP2.W(), GetReg(Node), MemOperand(MemSrc)); break; - case 8: ldclral(TMP2.X(), GetReg(Node), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + mvn(EmitSize, TMP2, Src); + ldclral(SubEmitSize, TMP2, GetReg(Node), MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - and_(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - and_(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - and_(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - and_(TMP3, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP4, TMP3, MemOperand(MemSrc)); - cbnz(TMP4, &LoopTop); - mov(GetReg(Node), TMP2); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + and_(EmitSize, TMP3, TMP2, Src); + stlxr(SubEmitSize, TMP4, TMP3, MemSrc); + cbnz(EmitSize, TMP4, &LoopTop); + mov(EmitSize, GetReg(Node), TMP2.R()); } } DEF_OP(AtomicFetchOr) { auto Op = IROp->C(); - auto MemSrc = GetReg(Op->Addr.ID()); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - switch (IROp->Size) { - case 1: ldsetalb(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 2: ldsetalh(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 4: ldsetal(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 8: ldsetal(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - orr(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - orr(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - orr(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - orr(TMP3, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP4, TMP3, MemOperand(MemSrc)); - cbnz(TMP4, &LoopTop); - mov(GetReg(Node), TMP2); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + orr(EmitSize, TMP3, TMP2, Src); + stlxr(SubEmitSize, TMP4, TMP3, MemSrc); + cbnz(EmitSize, TMP4, &LoopTop); + mov(EmitSize, GetReg(Node), TMP2.R()); } } DEF_OP(AtomicFetchXor) { auto Op = IROp->C(); - auto MemSrc = GetReg(Op->Addr.ID()); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + + auto MemSrc = GetReg(Op->Addr.ID()); + auto Src = GetReg(Op->Value.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; if (CTX->HostFeatures.SupportsAtomics) { - switch (IROp->Size) { - case 1: ldeoralb(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 2: ldeoralh(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 4: ldeoral(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - case 8: ldeoral(GetReg(Op->Value.ID()), GetReg(Node), MemOperand(MemSrc)); break; - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc); } else { - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - eor(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - eor(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - eor(TMP3.W(), TMP2.W(), GetReg(Op->Value.ID())); - stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - eor(TMP3, TMP2, GetReg(Op->Value.ID())); - stlxr(TMP4, TMP3, MemOperand(MemSrc)); - cbnz(TMP4, &LoopTop); - mov(GetReg(Node), TMP2); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + eor(EmitSize, TMP3, TMP2, Src); + stlxr(SubEmitSize, TMP4, TMP3, MemSrc); + cbnz(EmitSize, TMP4, &LoopTop); + mov(EmitSize, GetReg(Node), TMP2.R()); } } DEF_OP(AtomicFetchNeg) { auto Op = IROp->C(); - auto MemSrc = GetReg(Op->Addr.ID()); + uint8_t OpSize = IROp->Size; + LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); - // TMP2-TMP3 - switch (IROp->Size) { - case 1: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrb(TMP2.W(), MemOperand(MemSrc)); - neg(TMP3.W(), TMP2.W()); - stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 2: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxrh(TMP2.W(), MemOperand(MemSrc)); - neg(TMP3.W(), TMP2.W()); - stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 4: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2.W(), MemOperand(MemSrc)); - neg(TMP3.W(), TMP2.W()); - stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc)); - cbnz(TMP4.W(), &LoopTop); - mov(GetReg(Node), TMP2.W()); - break; - } - case 8: { - aarch64::Label LoopTop; - bind(&LoopTop); - ldaxr(TMP2, MemOperand(MemSrc)); - neg(TMP3, TMP2); - stlxr(TMP4, TMP3, MemOperand(MemSrc)); - cbnz(TMP4, &LoopTop); - mov(GetReg(Node), TMP2); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size); - } + auto MemSrc = GetReg(Op->Addr.ID()); + + const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; + + ARMEmitter::BackwardLabel LoopTop; + Bind(&LoopTop); + ldaxr(SubEmitSize, TMP2, MemSrc); + neg(EmitSize, TMP3, TMP2); + stlxr(SubEmitSize, TMP4, TMP3, MemSrc); + cbnz(EmitSize, TMP4, &LoopTop); + mov(EmitSize, GetReg(Node), TMP2.R()); } #undef DEF_OP diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp index f95322e8f..d0ecddad4 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp @@ -6,6 +6,7 @@ $end_info$ #include "Interface/Context/Context.h" #include "FEXCore/IR/IR.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/LookupCache.h" #include "Interface/Core/JIT/Arm64/JITClass.h" @@ -17,8 +18,6 @@ $end_info$ #include namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(SignalReturn) { @@ -27,12 +26,11 @@ DEF_OP(SignalReturn) { // Now branch to our signal return helper // This can't be a direct branch since the code needs to live at a constant location - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler))); - br(x0); + ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler)); + br(ARMEmitter::Reg::r0); } DEF_OP(CallbackReturn) { - // spill back to CTX SpillStaticRegs(); @@ -41,14 +39,14 @@ DEF_OP(CallbackReturn) { // We can now lower the ref counter again - ldr(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter))); - sub(w2, w2, 1); - str(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter))); + ldr(ARMEmitter::WReg::w2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)); + sub(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 1); + str(ARMEmitter::WReg::w2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)); // We need to adjust an additional 8 bytes to get back to the original "misaligned" RSP state - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP]))); - add(x2, x2, 8); - str(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP]))); + ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, 8); + str(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])); PopCalleeSavedRegisters(); @@ -59,39 +57,41 @@ DEF_OP(CallbackReturn) { DEF_OP(ExitFunction) { auto Op = IROp->C(); - Label FullLookup; - ResetStack(); - aarch64::Register RipReg; uint64_t NewRIP; if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) { - Literal l_BranchHost{ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker}; - Literal l_BranchGuest{NewRIP}; + ARMEmitter::ForwardLabel l_BranchHost; + ARMEmitter::ForwardLabel l_BranchGuest; - ldr(x0, &l_BranchHost); - blr(x0); + ldr(ARMEmitter::XReg::x0, &l_BranchHost); + blr(ARMEmitter::Reg::r0); + + Bind(&l_BranchHost); + dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker); + Bind(&l_BranchGuest); + dc64(NewRIP); - place(&l_BranchHost); - place(&l_BranchGuest); } else { - RipReg = GetReg(Op->NewRIP.ID()); + + ARMEmitter::ForwardLabel FullLookup; + auto RipReg = GetReg(Op->NewRIP.ID()); // L1 Cache - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer))); + ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer)); - and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK); - add(x0, x0, Operand(x3, Shift::LSL, 4)); + and_(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, RipReg, LookupCache::L1_ENTRIES_MASK); + add(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::XReg::x3, ARMEmitter::ShiftType::LSL, 4); - ldp(x1, x0, MemOperand(x0)); - cmp(x0, RipReg); - b(&FullLookup, Condition::ne); - br(x1); + ldp(ARMEmitter::XReg::x1, ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, 0); + cmp(ARMEmitter::XReg::x0, RipReg.X()); + b(ARMEmitter::Condition::CC_NE, &FullLookup); + br(ARMEmitter::Reg::r1); - bind(&FullLookup); - ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop))); - str(RipReg, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip))); + Bind(&FullLookup); + ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop)); + str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)); br(TMP1); } } @@ -103,66 +103,65 @@ DEF_OP(Jump) { PendingTargetLabel = &JumpTargets.try_emplace(Target).first->second; } -#define GRCMP(Node) (Op->CompareSize == 4 ? GetReg(Node) : GetReg(Node)) -#define GRFCMP(Node) (Op->CompareSize == 4 ? GetVReg(Node).S() : GetVReg(Node).D()) - -static Condition MapBranchCC(IR::CondClassType Cond) { +static ARMEmitter::Condition MapBranchCC(IR::CondClassType Cond) { switch (Cond.Val) { - case FEXCore::IR::COND_EQ: return Condition::eq; - case FEXCore::IR::COND_NEQ: return Condition::ne; - case FEXCore::IR::COND_SGE: return Condition::ge; - case FEXCore::IR::COND_SLT: return Condition::lt; - case FEXCore::IR::COND_SGT: return Condition::gt; - case FEXCore::IR::COND_SLE: return Condition::le; - case FEXCore::IR::COND_UGE: return Condition::cs; - case FEXCore::IR::COND_ULT: return Condition::cc; - case FEXCore::IR::COND_UGT: return Condition::hi; - case FEXCore::IR::COND_ULE: return Condition::ls; - case FEXCore::IR::COND_FLU: return Condition::lt; - case FEXCore::IR::COND_FGE: return Condition::ge; - case FEXCore::IR::COND_FLEU:return Condition::le; - case FEXCore::IR::COND_FGT: return Condition::gt; - case FEXCore::IR::COND_FU: return Condition::vs; - case FEXCore::IR::COND_FNU: return Condition::vc; + case FEXCore::IR::COND_EQ: return ARMEmitter::Condition::CC_EQ; + case FEXCore::IR::COND_NEQ: return ARMEmitter::Condition::CC_NE; + case FEXCore::IR::COND_SGE: return ARMEmitter::Condition::CC_GE; + case FEXCore::IR::COND_SLT: return ARMEmitter::Condition::CC_LT; + case FEXCore::IR::COND_SGT: return ARMEmitter::Condition::CC_GT; + case FEXCore::IR::COND_SLE: return ARMEmitter::Condition::CC_LE; + case FEXCore::IR::COND_UGE: return ARMEmitter::Condition::CC_CS; + case FEXCore::IR::COND_ULT: return ARMEmitter::Condition::CC_CC; + case FEXCore::IR::COND_UGT: return ARMEmitter::Condition::CC_HI; + case FEXCore::IR::COND_ULE: return ARMEmitter::Condition::CC_LS; + case FEXCore::IR::COND_FLU: return ARMEmitter::Condition::CC_LT; + case FEXCore::IR::COND_FGE: return ARMEmitter::Condition::CC_GE; + case FEXCore::IR::COND_FLEU:return ARMEmitter::Condition::CC_LE; + case FEXCore::IR::COND_FGT: return ARMEmitter::Condition::CC_GT; + case FEXCore::IR::COND_FU: return ARMEmitter::Condition::CC_VS; + case FEXCore::IR::COND_FNU: return ARMEmitter::Condition::CC_VC; case FEXCore::IR::COND_VS: case FEXCore::IR::COND_VC: case FEXCore::IR::COND_MI: case FEXCore::IR::COND_PL: default: LOGMAN_MSG_A_FMT("Unsupported compare type"); - return Condition::nv; + return ARMEmitter::Condition::CC_NV; } } - DEF_OP(CondJump) { auto Op = IROp->C(); - Label *TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second; + auto TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second; uint64_t Const; const bool isConst = IsInlineConstant(Op->Cmp2, &Const); + const auto Size = Op->CompareSize == 4 ? ARMEmitter::Size::i32Bit : ARMEmitter::Size::i64Bit; + const auto SubSize = ARMEmitter::ToVectorSizePair(Op->CompareSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i64Bit); + if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_EQ) { LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR"); - cbz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel); + cbz(Size, GetReg(Op->Cmp1.ID()), TrueTargetLabel); } else if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_NEQ) { LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR"); - cbnz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel); + cbnz(Size, GetReg(Op->Cmp1.ID()), TrueTargetLabel); } else { if (IsGPR(Op->Cmp1.ID())) { if (isConst) { - cmp(GRCMP(Op->Cmp1.ID()), Const); + cmp(Size, GetReg(Op->Cmp1.ID()), Const); } else { - cmp(GRCMP(Op->Cmp1.ID()), GRCMP(Op->Cmp2.ID())); + cmp(Size, GetReg(Op->Cmp1.ID()), GetReg(Op->Cmp2.ID())); } } else if (IsFPR(Op->Cmp1.ID())) { - fcmp(GRFCMP(Op->Cmp1.ID()), GRFCMP(Op->Cmp2.ID())); + fcmp(SubSize.Scalar, GetVReg(Op->Cmp1.ID()), GetVReg(Op->Cmp2.ID())); } else { LOGMAN_MSG_A_FMT("CondJump: Expected GPR or FPR"); } - b(TrueTargetLabel, MapBranchCC(Op->Cond)); + b(MapBranchCC(Op->Cond), TrueTargetLabel); } PendingTargetLabel = &JumpTargets.try_emplace(Op->FalseBlock.ID()).first->second; @@ -187,23 +186,25 @@ DEF_OP(Syscall) { } uint64_t SPOffset = AlignUp(FEXCore::HLE::SyscallArguments::MAX_ARGS * 8, 16); - sub(sp, sp, SPOffset); + sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset); for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS; ++i) { if (Op->Header.Args[i].IsInvalid()) continue; - str(GetReg(Op->Header.Args[i].ID()), MemOperand(sp, i * 8)); + str(GetReg(Op->Header.Args[i].ID()).X(), ARMEmitter::Reg::rsp, i * 8); } - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj))); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc))); - mov(x1, STATE); - mov(x2, sp); + ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj)); + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc)); + mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R()); + + // SP supporting move + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::rsp, 0); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x3); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); #else - blr(x3); + blr(ARMEmitter::Reg::r3); #endif - add(sp, sp, SPOffset); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset); if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) != FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY && (Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) { @@ -219,7 +220,7 @@ DEF_OP(Syscall) { if ((Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) { // Move result to its destination register - mov(GetReg(Node), x0); + mov(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0); } } @@ -236,8 +237,8 @@ DEF_OP(InlineSyscall) { // X6: Arg6 - Doesn't exist in x86-64 land. RA INTERSECT // One argument is removed from the SyscallArguments::MAX_ARGS since the first argument was syscall number - const static std::array RegArgs = {{ - x0, x1, x2, x3, x4, x5 + const static std::array RegArgs = {{ + ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::XReg::x2, ARMEmitter::XReg::x3, ARMEmitter::XReg::x4, ARMEmitter::XReg::x5 }}; bool Intersects{}; @@ -246,12 +247,12 @@ DEF_OP(InlineSyscall) { for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) { if (Op->Header.Args[i].IsInvalid()) break; - auto Reg = GetReg(Op->Header.Args[i].ID()); - if (Reg.GetCode() == x8.GetCode() || - Reg.GetCode() == x4.GetCode() || - Reg.GetCode() == x5.GetCode()) { + auto Reg = GetReg(Op->Header.Args[i].ID()); + if (Reg.Idx() == ARMEmitter::Reg::r8.Idx() || + Reg.Idx() == ARMEmitter::Reg::r4.Idx() || + Reg.Idx() == ARMEmitter::Reg::r5.Idx()) { - SpillMask |= (1U << Reg.GetCode()); + SpillMask |= (1U << Reg.Idx()); Intersects = true; } } @@ -266,66 +267,31 @@ DEF_OP(InlineSyscall) { // 16bit LoadConstant to be a single instruction // We must always spill at least one register (x8) so this value always has a bit set // This gives the signal handler a value to check to see if we are in a syscall at all - LoadConstant(x0, SpillMask & 0xFFFF); - str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo))); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SpillMask & 0xFFFF); + str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)); // Now that we have claimed to be a syscall we can set up the arguments + const auto EmitSize = CTX->Config.Is64BitMode() ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + const auto EmitSubSize = CTX->Config.Is64BitMode() ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i32Bit; if (Intersects) { for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) { if (Op->Header.Args[i].IsInvalid()) break; - if (CTX->Config.Is64BitMode()) { - auto Reg = GetReg(Op->Header.Args[i].ID()); - // In the case of intersection with x4, x5, or x8 then these are currently SRA - // for registers RAX, RBX, and RSI. Which have just been spilled - // Just load back from the context. Could be slightly smarter but this is fairly uncommon - if (Reg.GetCode() == x8.GetCode()) { - ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI]))); - } - else if (Reg.GetCode() == x4.GetCode()) { - ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX]))); - } - else if (Reg.GetCode() == x5.GetCode()) { - ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX]))); - } + auto Reg = GetReg(Op->Header.Args[i].ID()); + // In the case of intersection with x4, x5, or x8 then these are currently SRA + // for registers RAX, RBX, and RSI. Which have just been spilled + // Just load back from the context. Could be slightly smarter but this is fairly uncommon + if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r8.Idx()) { + ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI])); } - } - - for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) { - if (Op->Header.Args[i].IsInvalid()) break; - - if (CTX->Config.Is64BitMode()) { - auto Reg = GetReg(Op->Header.Args[i].ID()); - // In the case of intersection with x4, x5, or x8 then these are currently SRA - // for registers RAX, RBX, and RSI. Which have just been spilled - // Just load back from the context. Could be slightly smarter but this is fairly uncommon - if (Reg.GetCode() == x8.GetCode()) { - ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI]))); - } - else if (Reg.GetCode() == x4.GetCode()) { - ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX]))); - } - else if (Reg.GetCode() == x5.GetCode()) { - ldr(RegArgs[i], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX]))); - } - else { - mov(RegArgs[i], Reg); - } + else if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r4.Idx()) { + ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX])); + } + else if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r5.Idx()) { + ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX])); } else { - auto Reg = GetReg(Op->Header.Args[i].ID()); - if (Reg.GetCode() == w8.GetCode()) { - ldr(RegArgs[i].W(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI]))); - } - else if (Reg.GetCode() == w4.GetCode()) { - ldr(RegArgs[i].W(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX]))); - } - else if (Reg.GetCode() == w5.GetCode()) { - ldr(RegArgs[i].W(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX]))); - } - else { - uxtw(RegArgs[i].W(), Reg); - } + mov(EmitSize, RegArgs[i].R(), Reg); } } } @@ -333,16 +299,11 @@ DEF_OP(InlineSyscall) { for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS-1; ++i) { if (Op->Header.Args[i].IsInvalid()) break; - if (CTX->Config.Is64BitMode()) { - mov(RegArgs[i], GetReg(Op->Header.Args[i].ID())); - } - else { - uxtw(RegArgs[i], GetReg(Op->Header.Args[i].ID())); - } + mov(EmitSize, RegArgs[i].R(), GetReg(Op->Header.Args[i].ID())); } } - LoadConstant(x8, Op->HostSyscallNumber); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, Op->HostSyscallNumber); svc(0); // On updated signal mask we can receive a signal RIGHT HERE @@ -353,16 +314,11 @@ DEF_OP(InlineSyscall) { // Now the registers we've spilled are back in their original host registers // We can safely claim we are no longer in a syscall - str(xzr, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo))); + str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)); // Result is now in x0 // Move result to its destination register - if (CTX->Config.Is64BitMode()) { - mov(GetReg(Node), x0); - } - else { - uxtw(GetReg(Node), x0); - } + mov(EmitSize, GetReg(Node), ARMEmitter::Reg::r0); } } @@ -376,14 +332,14 @@ DEF_OP(Thunk) { PushDynamicRegsAndLR(TMP1); - mov(x0, GetReg(Op->ArgPtr.ID())); + mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->ArgPtr.ID())); auto thunkFn = ThreadState->CTX->ThunkHandler->LookupThunk(Op->ThunkNameHash); - LoadConstant(x2, (uintptr_t)thunkFn); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, (uintptr_t)thunkFn); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PopDynamicRegsAndLR(); @@ -397,43 +353,45 @@ DEF_OP(ValidateCode) { int len = Op->CodeLength; int idx = 0; - LoadConstant(GetReg(Node), 0); - LoadConstant(x0, Entry + Op->Offset); - LoadConstant(x1, 1); + LoadConstant(ARMEmitter::Size::i64Bit, GetReg(Node), 0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, Entry + Op->Offset); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, 1); + + const auto Dst = GetReg(Node); while (len >= 8) { - ldr(x2, MemOperand(x0, idx)); - LoadConstant(x3, *(const uint32_t *)(OldCode + idx)); - cmp(x2, x3); - csel(GetReg(Node), GetReg(Node), x1, Condition::eq); + ldr(ARMEmitter::XReg::x2, ARMEmitter::Reg::r0, idx); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint32_t *)(OldCode + idx)); + cmp(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3); + csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ); len -= 8; idx += 8; } while (len >= 4) { - ldr(w2, MemOperand(x0, idx)); - LoadConstant(w3, *(const uint32_t *)(OldCode + idx)); - cmp(w2, w3); - csel(GetReg(Node), GetReg(Node), x1, Condition::eq); + ldr(ARMEmitter::WReg::w2, ARMEmitter::Reg::r0, idx); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint32_t *)(OldCode + idx)); + cmp(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3); + csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ); len -= 4; idx += 4; } while (len >= 2) { - ldrh(w2, MemOperand(x0, idx)); - LoadConstant(w3, *(const uint16_t *)(OldCode + idx)); - cmp(w2, w3); - csel(GetReg(Node), GetReg(Node), x1, Condition::eq); + ldrh(ARMEmitter::Reg::r2, ARMEmitter::Reg::r0, idx); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint16_t *)(OldCode + idx)); + cmp(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3); + csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ); len -= 2; idx += 2; } while (len >= 1) { - ldrb(w2, MemOperand(x0, idx)); - LoadConstant(w3, *(const uint8_t *)(OldCode + idx)); - cmp(w2, w3); - csel(GetReg(Node), GetReg(Node), x1, Condition::eq); + ldrb(ARMEmitter::Reg::r2, ARMEmitter::Reg::r0, idx); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, *(const uint8_t *)(OldCode + idx)); + cmp(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r3); + csel(ARMEmitter::Size::i64Bit, Dst, Dst, ARMEmitter::Reg::r1, ARMEmitter::Condition::CC_EQ); len -= 1; idx += 1; } @@ -446,15 +404,15 @@ DEF_OP(ThreadRemoveCodeEntry) { PushDynamicRegsAndLR(TMP1); - mov(x0, STATE); - LoadConstant(x1, Entry); + mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R()); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Entry); - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT))); + ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT)); SpillStaticRegs(); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif FillStaticRegs(); @@ -471,24 +429,25 @@ DEF_OP(CPUID) { // x0 = CPUID Handler // x1 = CPUID Function // x2 = CPUID Leaf - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj))); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction))); - mov(x1, GetReg(Op->Function.ID())); - mov(x2, GetReg(Op->Leaf.ID())); + ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)); + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction)); + mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetReg(Op->Function.ID())); + mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, GetReg(Op->Leaf.ID())); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(x3); + GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3); #else - blr(x3); + blr(ARMEmitter::Reg::r3); #endif FillStaticRegs(); + PopDynamicRegsAndLR(); // Results are in x0, x1 // Results want to be in a i64v2 vector - auto Dst = GetRegPair(Node); - mov(Dst.first, x0); - mov(Dst.second, x1); + auto Dst = GetRegPair(Node); + mov(ARMEmitter::Size::i64Bit, Dst.first, ARMEmitter::Reg::r0); + mov(ARMEmitter::Size::i64Bit, Dst.second, ARMEmitter::Reg::r1); } #undef DEF_OP diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp index 553ddd15c..26c8f2361 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp @@ -4,12 +4,10 @@ tags: backend|arm64 $end_info$ */ +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/JIT/Arm64/JITClass.h" namespace FEXCore::CPU { - -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(VInsGPR) { const auto Op = IROp->C(); @@ -19,8 +17,16 @@ DEF_OP(VInsGPR) { const auto ElementSize = Op->Header.ElementSize; const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2 || ElementSize == 1, "Unexpected {} size", __func__); + const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit; + const auto ElementsPer128Bit = 16 / ElementSize; + const auto Dst = GetVReg(Node); const auto DestVector = GetVReg(Op->DestVector.ID()); + const auto Src = GetReg(Op->Src.ID()); if (HostSupportsSVE && Is256Bit) { const auto ElementSizeBits = ElementSize * 8; @@ -47,99 +53,56 @@ DEF_OP(VInsGPR) { if (InUpperLane) { // Move the upper lane down for the insertion. - const auto CompactPred = p0; - not_(CompactPred.VnB(), PRED_TMP_32B.Zeroing(), PRED_TMP_16B.VnB()); - compact(VTMP1.Z().VnD(), CompactPred, DestVector.Z().VnD()); + const auto CompactPred = ARMEmitter::PReg::p0; + not_(CompactPred, PRED_TMP_32B.Zeroing(), PRED_TMP_16B); + compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, DestVector); } // Put data in place for destructive SPLICE below. - mov(Dst.Z().VnD(), DestVector.Z().VnD()); + mov(Dst.Z(), DestVector.Z()); // Inserts the GPR value into the given V register. // Also automatically adjusts the index in the case of using the // moved upper lane. - const auto Insert = [&](const aarch64::VRegister& reg, int index) { - switch (ElementSize) { - case 1: - if (InUpperLane) { - index -= 16; - } - ins(reg.V16B(), index, GetReg(Op->Src.ID())); - break; - case 2: - if (InUpperLane) { - index -= 8; - } - ins(reg.V8H(), index, GetReg(Op->Src.ID())); - break; - case 4: - if (InUpperLane) { - index -= 4; - } - ins(reg.V4S(), index, GetReg(Op->Src.ID())); - break; - case 8: - if (InUpperLane) { - index -= 2; - } - ins(reg.V2D(), index, GetReg(Op->Src.ID())); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + const auto Insert = [&](const FEXCore::ARMEmitter::VRegister& reg, int index) { + if (InUpperLane) { + index -= ElementsPer128Bit; } + ins(SubEmitSize, reg, index, Src); }; if (InUpperLane) { Insert(VTMP1, DestIdx); - splice(Dst.Z().VnD(), PRED_TMP_16B, Dst.Z().VnD(), VTMP1.Z().VnD()); + splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP1.Z()); } else { Insert(Dst, DestIdx); - splice(Dst.Z().VnD(), PRED_TMP_16B, Dst.Z().VnD(), DestVector.Z().VnD()); + splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), DestVector.Z()); } } else { - mov(Dst, DestVector); - - switch (ElementSize) { - case 1: { - ins(Dst.V16B(), DestIdx, GetReg(Op->Src.ID())); - break; - } - case 2: { - ins(Dst.V8H(), DestIdx, GetReg(Op->Src.ID())); - break; - } - case 4: { - ins(Dst.V4S(), DestIdx, GetReg(Op->Src.ID())); - break; - } - case 8: { - ins(Dst.V2D(), DestIdx, GetReg(Op->Src.ID())); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + mov(Dst.Q(), DestVector.Q()); + ins(SubEmitSize, Dst, DestIdx, Src); } } DEF_OP(VCastFromGPR) { auto Op = IROp->C(); + auto Dst = GetVReg(Node); + auto Src = GetReg(Op->Src.ID()); + switch (Op->Header.ElementSize) { case 1: - uxtb(TMP1.W(), GetReg(Op->Src.ID())); - fmov(GetVReg(Node).S(), TMP1.W()); + uxtb(ARMEmitter::Size::i32Bit, TMP1, Src); + fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1); break; case 2: - uxth(TMP1.W(), GetReg(Op->Src.ID())); - fmov(GetVReg(Node).S(), TMP1.W()); + uxth(ARMEmitter::Size::i32Bit, TMP1, Src); + fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1); break; case 4: - fmov(GetVReg(Node).S(), GetReg(Op->Src.ID()).W()); + fmov(ARMEmitter::Size::i32Bit, Dst.S(), Src); break; case 8: - fmov(GetVReg(Node).D(), GetReg(Op->Src.ID()).X()); + fmov(ARMEmitter::Size::i64Bit, Dst.D(), Src); break; default: LOGMAN_MSG_A_FMT("Unknown castGPR element size: {}", Op->Header.ElementSize); } @@ -151,21 +114,24 @@ DEF_OP(Float_FromGPR_S) { const uint16_t ElementSize = Op->Header.ElementSize; const uint16_t Conv = (ElementSize << 8) | Op->SrcElementSize; + auto Dst = GetVReg(Node); + auto Src = GetReg(Op->Src.ID()); + switch (Conv) { case 0x0404: { // Float <- int32_t - scvtf(GetVReg(Node).S(), GetReg(Op->Src.ID())); + scvtf(ARMEmitter::Size::i32Bit, Dst.S(), Src); break; } case 0x0408: { // Float <- int64_t - scvtf(GetVReg(Node).S(), GetReg(Op->Src.ID())); + scvtf(ARMEmitter::Size::i64Bit, Dst.S(), Src); break; } case 0x0804: { // Double <- int32_t - scvtf(GetVReg(Node).D(), GetReg(Op->Src.ID())); + scvtf(ARMEmitter::Size::i32Bit, Dst.D(), Src); break; } case 0x0808: { // Double <- int64_t - scvtf(GetVReg(Node).D(), GetReg(Op->Src.ID())); + scvtf(ARMEmitter::Size::i64Bit, Dst.D(), Src); break; } default: @@ -178,13 +144,17 @@ DEF_OP(Float_FromGPR_S) { DEF_OP(Float_FToF) { auto Op = IROp->C(); const uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize; + + auto Dst = GetVReg(Node); + auto Src = GetVReg(Op->Scalar.ID()); + switch (Conv) { case 0x0804: { // Double <- Float - fcvt(GetVReg(Node).D(), GetVReg(Op->Scalar.ID()).S()); + fcvt(Dst.D(), Src.S()); break; } case 0x0408: { // Float <- Double - fcvt(GetVReg(Node).S(), GetVReg(Op->Scalar.ID()).D()); + fcvt(Dst.S(), Src.D()); break; } default: LOGMAN_MSG_A_FMT("Unknown FCVT sizes: 0x{:x}", Conv); @@ -198,41 +168,18 @@ DEF_OP(Vector_SToF) { const auto ElementSize = Op->Header.ElementSize; const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__); + const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit; + const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); - if (HostSupportsSVE && Is256Bit) { - const auto Mask = PRED_TMP_32B.Merging(); - - switch (ElementSize) { - case 2: - scvtf(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - break; - case 4: - scvtf(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - break; - case 8: - scvtf(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize); - break; - } + const auto Mask = PRED_TMP_32B; + scvtf(Dst.Z(), SubEmitSize, Mask.Merging(), Vector.Z(), SubEmitSize); } else { - switch (ElementSize) { - case 2: - scvtf(Dst.V8H(), Vector.V8H()); - break; - case 4: - scvtf(Dst.V4S(), Vector.V4S()); - break; - case 8: - scvtf(Dst.V2D(), Vector.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize); - break; - } + scvtf(SubEmitSize, Dst.Q(), Vector.Q()); } } @@ -243,41 +190,18 @@ DEF_OP(Vector_FToZS) { const auto ElementSize = Op->Header.ElementSize; const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__); + const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit; + const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); - if (HostSupportsSVE && Is256Bit) { - const auto Mask = PRED_TMP_32B.Merging(); - - switch (ElementSize) { - case 2: - fcvtzs(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - break; - case 4: - fcvtzs(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - break; - case 8: - fcvtzs(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize); - break; - } + const auto Mask = PRED_TMP_32B; + fcvtzs(Dst, SubEmitSize, Mask.Merging(), Vector, SubEmitSize); } else { - switch (ElementSize) { - case 2: - fcvtzs(Dst.V8H(), Vector.V8H()); - break; - case 4: - fcvtzs(Dst.V4S(), Vector.V4S()); - break; - case 8: - fcvtzs(Dst.V2D(), Vector.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize); - break; - } + fcvtzs(SubEmitSize, Dst.Q(), Vector.Q()); } } @@ -288,47 +212,23 @@ DEF_OP(Vector_FToS) { const auto ElementSize = Op->Header.ElementSize; const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__); + const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit; + const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); if (HostSupportsSVE && Is256Bit) { - const auto Mask = PRED_TMP_32B.Merging(); - - switch (ElementSize) { - case 2: - frinti(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - fcvtzs(Dst.Z().VnH(), Mask, Dst.Z().VnH()); - break; - case 4: - frinti(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - fcvtzs(Dst.Z().VnS(), Mask, Dst.Z().VnS()); - break; - case 8: - frinti(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - fcvtzs(Dst.Z().VnD(), Mask, Dst.Z().VnD()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize); - break; - } + const auto Mask = PRED_TMP_32B; + frinti(SubEmitSize, Dst, Mask.Merging(), Vector); + fcvtzs(Dst, SubEmitSize, Mask.Merging(), Dst, SubEmitSize); } else { - switch (ElementSize) { - case 2: - frinti(Dst.V8H(), Vector.V8H()); - fcvtzs(Dst.V8H(), Dst.V8H()); - break; - case 4: - frinti(Dst.V4S(), Vector.V4S()); - fcvtzs(Dst.V4S(), Dst.V4S()); - break; - case 8: - frinti(Dst.V2D(), Vector.V2D()); - fcvtzs(Dst.V2D(), Dst.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize); - break; - } + const auto Dst = GetVReg(Node); + const auto Vector = GetVReg(Op->Vector.ID()); + frinti(SubEmitSize, Dst.Q(), Vector.Q()); + fcvtzs(SubEmitSize, Dst.Q(), Dst.Q()); } } @@ -340,6 +240,11 @@ DEF_OP(Vector_FToF) { const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; const auto Conv = (ElementSize << 8) | Op->SrcElementSize; + LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__); + const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit; + const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); @@ -361,23 +266,23 @@ DEF_OP(Vector_FToF) { switch (Conv) { case 0x0402: { // Float <- Half - zip1(Dst.Z().VnH(), Vector.Z().VnH(), Vector.Z().VnH()); - fcvtlt(Dst.Z().VnS(), Mask, Dst.Z().VnH()); + zip1(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Vector.Z(), Vector.Z()); + fcvtlt(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Mask, Dst.Z()); break; } case 0x0804: { // Double <- Float - zip1(Dst.Z().VnS(), Vector.Z().VnS(), Vector.Z().VnS()); - fcvtlt(Dst.Z().VnD(), Mask, Dst.Z().VnS()); + zip1(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Vector.Z(), Vector.Z()); + fcvtlt(FEXCore::ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Dst.Z()); break; } case 0x0204: { // Half <- Float - fcvtnt(Dst.Z().VnH(), Mask, Vector.Z().VnS()); - uzp2(Dst.Z().VnH(), Dst.Z().VnH(), Dst.Z().VnH()); + fcvtnt(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst, Mask, Vector); + uzp2(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Dst.Z(), Dst.Z()); break; } case 0x0408: { // Float <- Double - fcvtnt(Dst.Z().VnS(), Mask, Vector.Z().VnD()); - uzp2(Dst.Z().VnS(), Dst.Z().VnS(), Dst.Z().VnS()); + fcvtnt(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst, Mask, Vector); + uzp2(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Dst.Z(), Dst.Z()); break; } default: @@ -386,20 +291,14 @@ DEF_OP(Vector_FToF) { } } else { switch (Conv) { - case 0x0402: { // Float <- Half - fcvtl(Dst.V4S(), Vector.V4H()); - break; - } + case 0x0402: // Float <- Half case 0x0804: { // Double <- Float - fcvtl(Dst.V2D(), Vector.V2S()); - break; - } - case 0x0204: { // Half <- Float - fcvtn(Dst.V4H(), Vector.V4S()); + fcvtl(SubEmitSize, Dst.D(), Vector.D()); break; } + case 0x0204: // Half <- Float case 0x0408: { // Float <- Double - fcvtn(Dst.V2S(), Vector.V2D()); + fcvtn(SubEmitSize, Dst.D(), Vector.D()); break; } default: @@ -415,154 +314,51 @@ DEF_OP(Vector_FToI) { const auto ElementSize = Op->Header.ElementSize; const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2, "Unexpected {} size", __func__); + + const auto SubEmitSize = ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : ARMEmitter::SubRegSize::i16Bit; const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); - + switch (Op->Round) { case FEXCore::IR::Round_Nearest.Val: - switch (ElementSize) { - case 2: - frintn(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - break; - case 4: - frintn(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - break; - case 8: - frintn(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - break; - } + frintn(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break; - case FEXCore::IR::Round_Negative_Infinity.Val: - switch (ElementSize) { - case 2: - frintm(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - break; - case 4: - frintm(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - break; - case 8: - frintm(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - break; - } + frintm(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break; - case FEXCore::IR::Round_Positive_Infinity.Val: - switch (ElementSize) { - case 2: - frintp(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - break; - case 4: - frintp(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - break; - case 8: - frintp(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - break; - } + frintp(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break; - case FEXCore::IR::Round_Towards_Zero.Val: - switch (ElementSize) { - case 2: - frintz(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - break; - case 4: - frintz(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - break; - case 8: - frintz(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - break; - } + frintz(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break; - case FEXCore::IR::Round_Host.Val: - switch (ElementSize) { - case 2: - frinti(Dst.Z().VnH(), Mask, Vector.Z().VnH()); - break; - case 4: - frinti(Dst.Z().VnS(), Mask, Vector.Z().VnS()); - break; - case 8: - frinti(Dst.Z().VnD(), Mask, Vector.Z().VnD()); - break; - } + frinti(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break; } } else { switch (Op->Round) { case FEXCore::IR::Round_Nearest.Val: - switch (ElementSize) { - case 2: - frintn(Dst.V8H(), Vector.V8H()); - break; - case 4: - frintn(Dst.V4S(), Vector.V4S()); - break; - case 8: - frintn(Dst.V2D(), Vector.V2D()); - break; - } + frinti(SubEmitSize, Dst.Q(), Vector.Q()); break; - case FEXCore::IR::Round_Negative_Infinity.Val: - switch (ElementSize) { - case 2: - frintm(Dst.V8H(), Vector.V8H()); - break; - case 4: - frintm(Dst.V4S(), Vector.V4S()); - break; - case 8: - frintm(Dst.V2D(), Vector.V2D()); - break; - } + frintm(SubEmitSize, Dst.Q(), Vector.Q()); break; - case FEXCore::IR::Round_Positive_Infinity.Val: - switch (ElementSize) { - case 2: - frintp(Dst.V8H(), Vector.V8H()); - break; - case 4: - frintp(Dst.V4S(), Vector.V4S()); - break; - case 8: - frintp(Dst.V2D(), Vector.V2D()); - break; - } + frintp(SubEmitSize, Dst.Q(), Vector.Q()); break; - case FEXCore::IR::Round_Towards_Zero.Val: - switch (ElementSize) { - case 2: - frintz(Dst.V8H(), Vector.V8H()); - break; - case 4: - frintz(Dst.V4S(), Vector.V4S()); - break; - case 8: - frintz(Dst.V2D(), Vector.V2D()); - break; - } + frintz(SubEmitSize, Dst.Q(), Vector.Q()); break; - case FEXCore::IR::Round_Host.Val: - switch (ElementSize) { - case 2: - frinti(Dst.V8H(), Vector.V8H()); - break; - case 4: - frinti(Dst.V4S(), Vector.V4S()); - break; - case 8: - frinti(Dst.V2D(), Vector.V2D()); - break; - } + frinti(SubEmitSize, Dst.Q(), Vector.Q()); break; } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/EncryptionOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/EncryptionOps.cpp index 995877292..72c587603 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/EncryptionOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/EncryptionOps.cpp @@ -4,98 +4,104 @@ tags: backend|arm64 $end_info$ */ +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/JIT/Arm64/JITClass.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(AESImc) { auto Op = IROp->C(); - aesimc(GetVReg(Node).V16B(), GetVReg(Op->Vector.ID()).V16B()); + aesimc(GetVReg(Node), GetVReg(Op->Vector.ID())); } DEF_OP(AESEnc) { auto Op = IROp->C(); - eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B()); - mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B()); - aese(VTMP1.V16B(), VTMP2.V16B()); - aesmc(VTMP1.V16B(), VTMP1.V16B()); - eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B()); + eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q()); + mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q()); + aese(VTMP1, VTMP2); + aesmc(VTMP1, VTMP1); + eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q()); } DEF_OP(AESEncLast) { auto Op = IROp->C(); - eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B()); - mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B()); - aese(VTMP1.V16B(), VTMP2.V16B()); - eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B()); + eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q()); + mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q()); + aese(VTMP1, VTMP2); + eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q()); } DEF_OP(AESDec) { auto Op = IROp->C(); - eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B()); - mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B()); - aesd(VTMP1.V16B(), VTMP2.V16B()); - aesimc(VTMP1.V16B(), VTMP1.V16B()); - eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B()); + eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q()); + mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q()); + aesd(VTMP1, VTMP2); + aesimc(VTMP1, VTMP1); + eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q()); } DEF_OP(AESDecLast) { auto Op = IROp->C(); - eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B()); - mov(VTMP1.V16B(), GetVReg(Op->State.ID()).V16B()); - aesd(VTMP1.V16B(), VTMP2.V16B()); - eor(GetVReg(Node).V16B(), VTMP1.V16B(), GetVReg(Op->Key.ID()).V16B()); + eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q()); + mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q()); + aesd(VTMP1, VTMP2); + eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q()); } DEF_OP(AESKeyGenAssist) { auto Op = IROp->C(); - aarch64::Literal ConstantLiteral (0x0C030609'0306090CULL, 0x040B0E01'0B0E0104ULL); - aarch64::Label PastConstant; + ARMEmitter::ForwardLabel Constant; + ARMEmitter::ForwardLabel PastConstant; // Do a "regular" AESE step - eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B()); - mov(VTMP1.V16B(), GetVReg(Op->Src.ID()).V16B()); - aese(VTMP1.V16B(), VTMP2.V16B()); + eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q()); + mov(VTMP1.Q(), GetVReg(Op->Src.ID()).Q()); + aese(VTMP1, VTMP2); // Do a table shuffle to undo ShiftRows - ldr(VTMP3, &ConstantLiteral); + ldr(VTMP3.Q(), &Constant); // Now EOR in the RCON if (Op->RCON) { - tbl(VTMP1.V16B(), VTMP1.V16B(), VTMP3.V16B()); + tbl(VTMP1.Q(), VTMP1.Q(), VTMP3.Q()); - LoadConstant(TMP1, static_cast(Op->RCON) << 32); - dup(VTMP2.V2D(), TMP1); - eor(GetVReg(Node).V16B(), VTMP1.V16B(), VTMP2.V16B()); + LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast(Op->RCON) << 32); + dup(ARMEmitter::SubRegSize::i64Bit, VTMP2.Q(), TMP1); + eor(GetVReg(Node).Q(), VTMP1.Q(), VTMP2.Q()); } else { - tbl(GetVReg(Node).V16B(), VTMP1.V16B(), VTMP3.V16B()); + tbl(GetVReg(Node).Q(), VTMP1.Q(), VTMP3.Q()); } b(&PastConstant); - place(&ConstantLiteral); - bind(&PastConstant); + Bind(&Constant); + dc64(0x040B0E01'0B0E0104ULL); + dc64(0x0C030609'0306090CULL); + Bind(&PastConstant); } DEF_OP(CRC32) { auto Op = IROp->C(); + + const auto Dst = GetReg(Node); + const auto Src1 = GetReg(Op->Src1.ID()); + const auto Src2 = GetReg(Op->Src2.ID()); + switch (Op->SrcSize) { case 1: - crc32cb(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); + crc32cb(Dst.W(), Src1.W(), Src2.W()); break; case 2: - crc32ch(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); + crc32ch(Dst.W(), Src1.W(), Src2.W()); break; case 4: - crc32cw(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); + crc32cw(Dst.W(), Src1.W(), Src2.W()); break; case 8: - crc32cx(GetReg(Node), GetReg(Op->Src1.ID()), GetReg(Op->Src2.ID())); + crc32cx(Dst, Src1, Src2); break; default: LOGMAN_MSG_A_FMT("Unknown CRC32 size: {}", Op->SrcSize); } @@ -104,24 +110,24 @@ DEF_OP(CRC32) { DEF_OP(PCLMUL) { auto Op = IROp->C(); - auto Dst = GetVReg(Node).Q(); - auto Src1 = GetVReg(Op->Src1.ID()).V2D(); - auto Src2 = GetVReg(Op->Src2.ID()).V2D(); + auto Dst = GetVReg(Node); + auto Src1 = GetVReg(Op->Src1.ID()); + auto Src2 = GetVReg(Op->Src2.ID()); switch (Op->Selector) { case 0b00000000: - pmull(Dst, Src1, Src2); + pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), Src1.D(), Src2.D()); break; case 0b00000001: - mov(VTMP1.V1D(), Src1, 1); - pmull(Dst, VTMP1.V2D(), Src2); + dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), Src1.Q(), 1); + pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), VTMP1.D(), Src2.D()); break; case 0b00010000: - mov(VTMP1.V1D(), Src2, 1); - pmull(Dst, VTMP1.V2D(), Src1); + dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Q(), Src2.Q(), 1); + pmull(ARMEmitter::SubRegSize::i128Bit, Dst.D(), VTMP1.D(), Src1.D()); break; case 0b00010001: - pmull2(Dst, Src1, Src2); + pmull2(ARMEmitter::SubRegSize::i128Bit, Dst.Q(), Src1.Q(), Src2.Q()); break; default: LOGMAN_MSG_A_FMT("Unknown PCLMUL selector: {}", Op->Selector); diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/FlagOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/FlagOps.cpp index b9c296ca9..6ced81860 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/FlagOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/FlagOps.cpp @@ -7,13 +7,10 @@ $end_info$ #include "Interface/Core/JIT/Arm64/JITClass.h" namespace FEXCore::CPU { - -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(GetHostFlag) { auto Op = IROp->C(); - ubfx(GetReg(Node), GetReg(Op->Value.ID()), Op->Flag, 1); + ubfx(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(Op->Value.ID()), Op->Flag, 1); } #undef DEF_OP diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index f75784fd7..291b5d7f6 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -11,6 +11,7 @@ $end_info$ */ #include "Interface/Context/Context.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/LookupCache.h" #include "Interface/Core/ArchHelpers/Arm64.h" @@ -77,9 +78,6 @@ static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) { namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; - void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { FallbackInfo Info; if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) { @@ -93,13 +91,13 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - uxth(w0, GetReg(IROp->Args[0].ID())); - ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); - + const auto Src1 = GetReg(IROp->Args[0].ID()); + uxth(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1); + ldr(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x1); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r1); #else - blr(x1); + blr(ARMEmitter::Reg::r1); #endif PopDynamicRegsAndLR(); @@ -112,22 +110,23 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { SpillStaticRegs(); PushDynamicRegsAndLR(TMP1); - - fmov(v0.S(), GetVReg(IROp->Args[0].ID()).S()) ; - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + const auto Src1 = GetVReg(IROp->Args[0].ID()); + fmov(ARMEmitter::SReg::s0, Src1.S()); + ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall<__uint128_t, float>(x0); + GenerateIndirectRuntimeCall<__uint128_t, float>(ARMEmitter::Reg::r0); #else - blr(x0); + blr(ARMEmitter::Reg::r0); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B()); - ins(GetVReg(Node).V2D(), 0, x0); - ins(GetVReg(Node).V8H(), 4, w1); + const auto Dst = GetVReg(Node); + eor(Dst.Q(), Dst.Q(), Dst.Q()); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0); + ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1); } break; @@ -136,21 +135,23 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - mov(v0.D(), GetVReg(IROp->Args[0].ID()).D()); - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(ARMEmitter::DReg::d0, Src1.D()); + ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall<__uint128_t, double>(x0); + GenerateIndirectRuntimeCall<__uint128_t, double>(ARMEmitter::Reg::r0); #else - blr(x0); + blr(ARMEmitter::Reg::r0); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B()); - ins(GetVReg(Node).V2D(), 0, x0); - ins(GetVReg(Node).V8H(), 4, w1); + const auto Dst = GetVReg(Node); + eor(Dst.Q(), Dst.Q(), Dst.Q()); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0); + ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1); } break; @@ -160,26 +161,28 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); + const auto Src1 = GetReg(IROp->Args[0].ID()); if (Info.ABI == FABI_F80_I16) { - uxth(w0, GetReg(IROp->Args[0].ID())); + uxth(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1); } else { - mov(w0, GetReg(IROp->Args[0].ID())); + mov(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1); } - ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + ldr(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall<__uint128_t, uint32_t>(x1); + GenerateIndirectRuntimeCall<__uint128_t, uint32_t>(ARMEmitter::Reg::r1); #else - blr(x1); + blr(ARMEmitter::Reg::r1); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B()); - ins(GetVReg(Node).V2D(), 0, x0); - ins(GetVReg(Node).V8H(), 4, w1); + const auto Dst = GetVReg(Node); + eor(Dst.Q(), Dst.Q(), Dst.Q()); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0); + ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1); } break; @@ -188,21 +191,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); + + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - fmov(GetVReg(Node).S(), v0.S()); + const auto Dst = GetVReg(Node); + fmov(Dst.S(), ARMEmitter::SReg::s0); } break; @@ -211,21 +217,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); + + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - mov(GetVReg(Node).D(), v0.D()); + const auto Dst = GetVReg(Node); + mov(Dst.D(), ARMEmitter::DReg::d0); } break; @@ -234,20 +243,22 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - mov(v0.D(), GetVReg(IROp->Args[0].ID()).D()); - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + const auto Src1 = GetVReg(IROp->Args[0].ID()); + + mov(ARMEmitter::DReg::d0, Src1.D()); + ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x0); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r0); #else - blr(x0); + blr(ARMEmitter::Reg::r0); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - mov(GetVReg(Node).D(), v0.D()); - + const auto Dst = GetVReg(Node); + mov(Dst.D(), ARMEmitter::DReg::d0); } break; @@ -256,21 +267,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - mov(v0.D(), GetVReg(IROp->Args[0].ID()).D()); - mov(v1.D(), GetVReg(IROp->Args[1].ID()).D()); - ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + const auto Src1 = GetVReg(IROp->Args[0].ID()); + const auto Src2 = GetVReg(IROp->Args[1].ID()); + + mov(ARMEmitter::DReg::d0, Src1.D()); + mov(ARMEmitter::DReg::d1, Src2.D()); + ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x0); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r0); #else - blr(x0); + blr(ARMEmitter::Reg::r0); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - mov(GetVReg(Node).D(), v0.D()); - + const auto Dst = GetVReg(Node); + mov(Dst.D(), ARMEmitter::DReg::d0); } break; @@ -279,21 +293,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); + + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - uxth(GetReg(Node), x0); + const auto Dst = GetReg(Node); + uxth(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0); } break; case FABI_I32_F80:{ @@ -301,21 +318,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); + + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - mov(GetReg(Node), w0); + const auto Dst = GetReg(Node); + mov(ARMEmitter::Size::i32Bit, Dst, ARMEmitter::Reg::r0); } break; case FABI_I64_F80:{ @@ -323,21 +343,24 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); + + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x2); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - mov(GetReg(Node), x0); + const auto Dst = GetReg(Node); + mov(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0); } break; case FABI_I64_F80_F80:{ @@ -345,23 +368,27 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); + const auto Src2 = GetVReg(IROp->Args[1].ID()); - umov(x2, GetVReg(IROp->Args[1].ID()).V2D(), 0); - umov(w3, GetVReg(IROp->Args[1].ID()).V8H(), 4); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); - ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r2, Src2, 0); + umov(ARMEmitter::Reg::r3, Src2, 4); + + ldr(ARMEmitter::XReg::x4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall(x4); + GenerateIndirectRuntimeCall(ARMEmitter::Reg::r4); #else - blr(x4); + blr(ARMEmitter::Reg::r4); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - mov(GetReg(Node), x0); + const auto Dst = GetReg(Node); + mov(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0); } break; case FABI_F80_F80:{ @@ -369,23 +396,26 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); - ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); + + ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t>(x2); + GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t>(ARMEmitter::Reg::r2); #else - blr(x2); + blr(ARMEmitter::Reg::r2); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B()); - ins(GetVReg(Node).V2D(), 0, x0); - ins(GetVReg(Node).V8H(), 4, w1); + const auto Dst = GetVReg(Node); + eor(Dst.Q(), Dst.Q(), Dst.Q()); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0); + ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1); } break; case FABI_F80_F80_F80:{ @@ -393,29 +423,32 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) { PushDynamicRegsAndLR(TMP1); - umov(x0, GetVReg(IROp->Args[0].ID()).V2D(), 0); - umov(w1, GetVReg(IROp->Args[0].ID()).V8H(), 4); + const auto Src1 = GetVReg(IROp->Args[0].ID()); + const auto Src2 = GetVReg(IROp->Args[1].ID()); - umov(x2, GetVReg(IROp->Args[1].ID()).V2D(), 0); - umov(w3, GetVReg(IROp->Args[1].ID()).V8H(), 4); + umov(ARMEmitter::Reg::r0, Src1, 0); + umov(ARMEmitter::Reg::r1, Src1, 4); - ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex]))); + umov(ARMEmitter::Reg::r2, Src2, 0); + umov(ARMEmitter::Reg::r3, Src2, 4); + + ldr(ARMEmitter::XReg::x4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); #ifdef VIXL_SIMULATOR - GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t, uint64_t>(x4); + GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r4); #else - blr(x4); + blr(ARMEmitter::Reg::r4); #endif PopDynamicRegsAndLR(); FillStaticRegs(); - eor(GetVReg(Node).V16B(), GetVReg(Node).V16B(), GetVReg(Node).V16B()); - ins(GetVReg(Node).V2D(), 0, x0); - ins(GetVReg(Node).V8H(), 4, w1); + const auto Dst = GetVReg(Node); + eor(Dst.Q(), Dst.Q(), Dst.Q()); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, ARMEmitter::Reg::r0); + ins(ARMEmitter::SubRegSize::i16Bit, Dst, 4, ARMEmitter::Reg::r1); } break; - case FABI_UNKNOWN: default: #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED @@ -435,7 +468,6 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram auto HostCode = Thread->LookupCache->FindBlock(GuestRip); if (!HostCode) { - //fmt::print("ExitFunctionLink: Aborting, {:X} not in cache\n", GuestRip); Frame->State.rip = GuestRip; return Frame->Pointers.Common.DispatcherLoopTop; } @@ -444,25 +476,22 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker; auto offset = HostCode/4 - branch/4; - if (IsInt26(offset)) { + if (vixl::IsInt26(offset)) { // optimal case - can branch directly // patch the code - vixl::aarch64::Assembler emit((uint8_t*)(branch), 24); - vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert); + FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 24); emit.b(offset); - emit.FinalizeCode(); - vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24); + FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 24); // Add de-linking handler Context::Context::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [branch, LinkerAddress]{ - vixl::aarch64::Assembler emit((uint8_t*)(branch), 24); - vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert); - Literal l_BranchHost{LinkerAddress}; - emit.ldr(x0, &l_BranchHost); - emit.blr(x0); - emit.place(&l_BranchHost); - emit.FinalizeCode(); - vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24); + FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 24); + FEXCore::ARMEmitter::ForwardLabel l_BranchHost; + emit.ldr(FEXCore::ARMEmitter::XReg::x0, &l_BranchHost); + emit.blr(FEXCore::ARMEmitter::Reg::r0); + emit.Bind(&l_BranchHost); + emit.dc64(LinkerAddress); + FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 24); }); } else { // fallback case - do a soft-er link by patching the pointer @@ -488,10 +517,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern RAPass = Thread->PassManager->GetPass("RA"); -#if DEBUG - Decoder.AppendVisitor(&Disasm) -#endif - uint32_t NumUsedGPRs = NumGPRs; uint32_t NumUsedGPRPairs = NumGPRPairs; uint32_t UsedRegisterCount = RegisterCount; @@ -559,7 +584,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern } // Must be done after Dispatcher init - SetAllowAssembler(true); ClearCache(); } @@ -582,16 +606,15 @@ void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) { void Arm64JITCore::EmitDetectionString() { const char JITString[] = "FEXJIT::Arm64JITCore::"; - auto Buffer = GetBuffer(); - Buffer->EmitString(JITString); - Buffer->Align(); + EmitString(JITString); + Align(); } void Arm64JITCore::ClearCache() { // Get the backing code buffer auto CodeBuffer = GetEmptyCodeBuffer(); - *GetBuffer() = vixl::CodeBuffer(CodeBuffer->Ptr, CodeBuffer->Size); + SetBuffer(CodeBuffer->Ptr, CodeBuffer->Size); EmitDetectionString(); } @@ -636,7 +659,6 @@ FEXCore::IR::RegisterClassType Arm64JITCore::GetRegClass(IR::NodeID Node) const return FEXCore::IR::RegisterClassType {GetPhys(Node).Class}; } - bool Arm64JITCore::IsFPR(IR::NodeID Node) const { auto Class = GetRegClass(Node); @@ -656,7 +678,6 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, bool GDBEnabled) { FEXCORE_PROFILE_SCOPED("Arm64::CompileCode"); - using namespace aarch64; JumpTargets.clear(); uint32_t SSACount = IR->GetSSACount(); @@ -665,12 +686,12 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, this->DebugData = DebugData; #ifdef VIXL_DISASSEMBLER - const auto DisasmBegin = GetCursorAddress(); + const auto DisasmBegin = GetCursorAddress(); #endif - #ifndef NDEBUG - LoadConstant(x0, Entry); - #endif +#ifndef NDEBUG + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, Entry); +#endif this->IR = IR; @@ -704,7 +725,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, if (GDBEnabled) { auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(GuestEntry, Entry); - GetBuffer()->CursorForward(GDBSize); + CursorIncrement(GDBSize); } //LOGMAN_THROW_A_FMT(RAData->HasFullRA(), "Arm64 JIT only works with RA"); @@ -714,11 +735,11 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, if (SpillSlots) { const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize; - if (IsImmAddSub(TotalSpillSlotsSize)) { - sub(sp, sp, TotalSpillSlotsSize); + if (vixl::aarch64::Assembler::IsImmAddSub(TotalSpillSlotsSize)) { + sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize); } else { - LoadConstant(x0, TotalSpillSlotsSize); - sub(sp, sp, x0); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, TotalSpillSlotsSize); + sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, ARMEmitter::XReg::x0, ARMEmitter::ExtendedType::LSL_64, 0); } } @@ -743,7 +764,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, } PendingTargetLabel = nullptr; - bind(&IsTarget->second); + Bind(&IsTarget->second); } for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) { @@ -769,13 +790,11 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, } PendingTargetLabel = nullptr; - FinalizeCode(); - auto CodeEnd = GetCursorAddress(); - CPU.EnsureIAndDCacheCoherency(GuestEntry, CodeEnd - GuestEntry); + ClearICache(GuestEntry, CodeEnd - GuestEntry); #ifdef VIXL_DISASSEMBLER - const auto DisasmEnd = GetCursorAddress(); + const auto DisasmEnd = GetCursorAddress(); Disasm.DisassembleBuffer(DisasmBegin, DisasmEnd); #endif @@ -796,12 +815,12 @@ void Arm64JITCore::ResetStack() { const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize; - if (IsImmAddSub(TotalSpillSlotsSize)) { - add(sp, sp, TotalSpillSlotsSize); + if (vixl::aarch64::Assembler::IsImmAddSub(TotalSpillSlotsSize)) { + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize); } else { - // Too big to fit in a 12bit immediate - LoadConstant(x0, TotalSpillSlotsSize); - add(sp, sp, x0); + // Too big to fit in a 12bit immediate + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, TotalSpillSlotsSize); + add(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, ARMEmitter::XReg::x0, ARMEmitter::ExtendedType::LSL_64, 0); } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index f6c6d227f..57d1002af 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -8,6 +8,7 @@ $end_info$ #include #include "Interface/Core/ArchHelpers/Arm64Emitter.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/Dispatcher/Dispatcher.h" #include @@ -28,9 +29,6 @@ namespace FEXCore::Core { } namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; - class Arm64JITCore final : public CPUBackend, public Arm64Emitter { public: explicit Arm64JITCore(FEXCore::Context::Context *ctx, @@ -58,12 +56,12 @@ private: FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO); const bool HostSupportsSVE{}; - Label *PendingTargetLabel; + ARMEmitter::BiDirectionalLabel *PendingTargetLabel; FEXCore::Context::Context *CTX; FEXCore::IR::IRListView const *IR; uint64_t Entry; - std::map JumpTargets; + std::map JumpTargets; /** * @name Register Allocation @@ -86,22 +84,8 @@ private: constexpr static uint8_t RA_64 = 1; constexpr static uint8_t RA_FPR = 2; - template - [[nodiscard]] aarch64::Register GetReg(IR::NodeID Node) const; - - template - [[nodiscard]] std::pair GetRegPair(IR::NodeID Node) const; - - [[nodiscard]] FEXCore::IR::RegisterClassType GetRegClass(IR::NodeID Node) const; - - template<> - [[nodiscard]] aarch64::Register GetReg(IR::NodeID Node) const { - return GetReg(Node).W(); - } - - template<> - [[nodiscard]] aarch64::Register GetReg(IR::NodeID Node) const { - auto Reg = GetPhys(Node); + [[nodiscard]] FEXCore::ARMEmitter::Register GetReg(IR::NodeID Node) const { + const auto Reg = GetPhys(Node); LOGMAN_THROW_AA_FMT(Reg.Class == IR::GPRFixedClass.Val || Reg.Class == IR::GPRClass.Val, "Unexpected Class: {}", Reg.Class); @@ -114,22 +98,8 @@ private: FEX_UNREACHABLE; } - template<> - [[nodiscard]] std::pair GetRegPair(IR::NodeID Node) const { - uint32_t Reg = GetPhys(Node).Reg; - auto Pair = RA64Pair[Reg]; - return std::make_pair(Pair.first.W(), Pair.second.W()); - } - - template<> - [[nodiscard]] std::pair GetRegPair(IR::NodeID Node) const { - uint32_t Reg = GetPhys(Node).Reg; - return RA64Pair[Reg]; - } - - - [[nodiscard]] aarch64::VRegister GetVReg(IR::NodeID Node) const { - auto Reg = GetPhys(Node); + [[nodiscard]] FEXCore::ARMEmitter::VRegister GetVReg(IR::NodeID Node) const { + const auto Reg = GetPhys(Node); LOGMAN_THROW_AA_FMT(Reg.Class == IR::FPRFixedClass.Val || Reg.Class == IR::FPRClass.Val, "Unexpected Class: {}", Reg.Class); @@ -142,6 +112,16 @@ private: FEX_UNREACHABLE; } + [[nodiscard]] std::pair GetRegPair(IR::NodeID Node) const { + const auto Reg = GetPhys(Node); + + LOGMAN_THROW_AA_FMT(Reg.Class == IR::GPRPairClass.Val, "Unexpected Class: {}", Reg.Class); + + return RA64Pair[Reg.Reg]; + } + + [[nodiscard]] FEXCore::IR::RegisterClassType GetRegClass(IR::NodeID Node) const; + [[nodiscard]] IR::PhysicalRegister GetPhys(IR::NodeID Node) const { auto PhyReg = RAData->GetNodeRegister(Node); @@ -153,8 +133,8 @@ private: [[nodiscard]] bool IsFPR(IR::NodeID Node) const; [[nodiscard]] bool IsGPR(IR::NodeID Node) const; - [[nodiscard]] MemOperand GenerateMemOperand(uint8_t AccessSize, - aarch64::Register Base, + [[nodiscard]] FEXCore::ARMEmitter::ExtendedMemOperand GenerateMemOperand(uint8_t AccessSize, + FEXCore::ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale); @@ -164,8 +144,8 @@ private: // // TMP1 is safe to use again once this memory operand is used with its // equivalent loads or stores that this was called for. - [[nodiscard]] SVEMemOperand GenerateSVEMemOperand(uint8_t AccessSize, - aarch64::Register Base, + [[nodiscard]] FEXCore::ARMEmitter::SVEMemOperand GenerateSVEMemOperand(uint8_t AccessSize, + FEXCore::ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale); @@ -203,7 +183,8 @@ private: * @brief A literal pair relocation object for named symbol literals */ struct NamedSymbolLiteralPair { - Literal Lit; + ARMEmitter::ForwardLabel Loc; + uint64_t Lit; Relocation MoveABI{}; }; @@ -213,7 +194,7 @@ private: * @param Reg - The GPR to move the thunk handler in to * @param Sum - The hash of the thunk */ - void InsertNamedThunkRelocation(vixl::aarch64::Register Reg, const IR::SHA256Sum &Sum); + void InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR::SHA256Sum &Sum); /** * @brief Inserts a guest GPR move relocation @@ -221,7 +202,7 @@ private: * @param Reg - The GPR to move the guest RIP in to * @param Constant - The guest RIP that will be relocated */ - void InsertGuestRIPMove(vixl::aarch64::Register Reg, uint64_t Constant); + void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant); /** * @brief Inserts a named symbol as a literal in memory diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp index edf3d80ad..f625ae407 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp @@ -5,14 +5,13 @@ $end_info$ */ #include "Interface/Context/Context.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Registers.h" #include "Interface/Core/CPUID.h" #include "Interface/Core/JIT/Arm64/JITClass.h" #include namespace FEXCore::CPU { - -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(LoadContext) { @@ -20,20 +19,20 @@ DEF_OP(LoadContext) { const auto OpSize = IROp->Size; if (Op->Class == FEXCore::IR::GPRClass) { - const auto Operand = MemOperand(STATE, Op->Offset); + auto Dst = GetReg(Node); switch (OpSize) { case 1: - ldrb(GetReg(Node), Operand); + ldrb(Dst, STATE, Op->Offset); break; case 2: - ldrh(GetReg(Node), Operand); + ldrh(Dst, STATE, Op->Offset); break; case 4: - ldr(GetReg(Node), Operand); + ldr(Dst.W(), STATE, Op->Offset); break; case 8: - ldr(GetReg(Node), Operand); + ldr(Dst.X(), STATE, Op->Offset); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadContext size: {}", OpSize); @@ -41,27 +40,27 @@ DEF_OP(LoadContext) { } } else { - const auto Dst = GetVReg(Node); + auto Dst = GetVReg(Node); switch (OpSize) { case 1: - ldr(Dst.B(), MemOperand(STATE, Op->Offset)); + ldrb(Dst, STATE, Op->Offset); break; case 2: - ldr(Dst.H(), MemOperand(STATE, Op->Offset)); + ldrh(Dst, STATE, Op->Offset); break; case 4: - ldr(Dst.S(), MemOperand(STATE, Op->Offset)); + ldr(Dst.S(), STATE, Op->Offset); break; case 8: - ldr(Dst.D(), MemOperand(STATE, Op->Offset)); + ldr(Dst.D(), STATE, Op->Offset); break; case 16: - ldr(Dst, MemOperand(STATE, Op->Offset)); + ldr(Dst.Q(), STATE, Op->Offset); break; case 32: mov(TMP1, Op->Offset); - ld1b(Dst.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(STATE, TMP1)); + ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), STATE, TMP1); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadContext size: {}", OpSize); @@ -75,20 +74,20 @@ DEF_OP(StoreContext) { const auto OpSize = IROp->Size; if (Op->Class == FEXCore::IR::GPRClass) { - const auto Operand = MemOperand(STATE, Op->Offset); + auto Src = GetReg(Op->Value.ID()); switch (OpSize) { case 1: - strb(GetReg(Op->Value.ID()), Operand); + strb(Src, STATE, Op->Offset); break; case 2: - strh(GetReg(Op->Value.ID()), Operand); + strh(Src, STATE, Op->Offset); break; case 4: - str(GetReg(Op->Value.ID()), Operand); + str(Src.W(), STATE, Op->Offset); break; case 8: - str(GetReg(Op->Value.ID()), Operand); + str(Src.X(), STATE, Op->Offset); break; default: LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize); @@ -100,23 +99,23 @@ DEF_OP(StoreContext) { switch (OpSize) { case 1: - str(Src.B(), MemOperand(STATE, Op->Offset)); + strb(Src, STATE, Op->Offset); break; case 2: - str(Src.H(), MemOperand(STATE, Op->Offset)); + strh(Src, STATE, Op->Offset); break; case 4: - str(Src.S(), MemOperand(STATE, Op->Offset)); + str(Src.S(), STATE, Op->Offset); break; case 8: - str(Src.D(), MemOperand(STATE, Op->Offset)); + str(Src.D(), STATE, Op->Offset); break; case 16: - str(Src, MemOperand(STATE, Op->Offset)); + str(Src.Q(), STATE, Op->Offset); break; case 32: mov(TMP1, Op->Offset); - st1b(Src.Z().VnB(), PRED_TMP_32B, SVEMemOperand(STATE, TMP1)); + st1b(Src.Z(), PRED_TMP_32B, STATE, TMP1); break; default: LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize); @@ -125,7 +124,6 @@ DEF_OP(StoreContext) { } } - DEF_OP(LoadRegister) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; @@ -141,24 +139,24 @@ DEF_OP(LoadRegister) { switch (OpSize) { case 1: LOGMAN_THROW_AA_FMT(regOffs == 0 || regOffs == 1, "unexpected regOffs"); - ubfx(GetReg(Node), reg, regOffs * 8, 8); + ubfx(ARMEmitter::Size::i64Bit, GetReg(Node), reg, regOffs * 8, 8); break; case 2: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - ubfx(GetReg(Node), reg, 0, 16); + ubfx(ARMEmitter::Size::i64Bit, GetReg(Node), reg, 0, 16); break; case 4: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - if (GetReg(Node).GetCode() != reg.GetCode()) - mov(GetReg(Node), reg.W()); + if (GetReg(Node).Idx() != reg.Idx()) + mov(GetReg(Node).W(), reg.W()); break; case 8: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - if (GetReg(Node).GetCode() != reg.GetCode()) { - mov(GetReg(Node), reg); + if (GetReg(Node).Idx() != reg.Idx()) { + mov(GetReg(Node).X(), reg.X()); } break; @@ -166,7 +164,8 @@ DEF_OP(LoadRegister) { LOGMAN_MSG_A_FMT("Unhandled LoadRegister GPR size: {}", OpSize); break; } - } else if (Op->Class == IR::FPRClass) { + } + else if (Op->Class == IR::FPRClass) { const auto regSize = HostSupportsSVE ? Core::CPUState::XMM_AVX_REG_SIZE : Core::CPUState::XMM_SSE_REG_SIZE; const auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.xmm.avx.data[0][0])) / regSize; @@ -179,50 +178,46 @@ DEF_OP(LoadRegister) { if (HostSupportsSVE) { const auto regOffs = Op->Offset & 31; - aarch64::Label DataLocation; + ARMEmitter::ForwardLabel DataLocation; const auto LoadPredicate = [this, &DataLocation] { - const auto Predicate = p0; + const auto Predicate = ARMEmitter::PReg::p0; adr(TMP1, &DataLocation); - ldr(Predicate, SVEMemOperand(TMP1)); + ldr(Predicate, TMP1); return Predicate.Merging(); }; - using DataLiteral = aarch64::Literal; - const auto EmitData = [this, &DataLocation](DataLiteral& Data) { - aarch64::Label PastConstant; + const auto EmitData = [this, &DataLocation](uint32_t Value) { + ARMEmitter::ForwardLabel PastConstant; b(&PastConstant); - bind(&DataLocation); - place(&Data); - bind(&PastConstant); + Bind(&DataLocation); + dc32(Value); + Bind(&PastConstant); }; switch (OpSize) { case 1: { LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - mov(host.B(), guest.B()); + dup(ARMEmitter::ScalarRegSize::i8Bit, host, guest, 0); break; } - case 2: { LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); fmov(host.H(), guest.H()); break; } - case 4: { LOGMAN_THROW_AA_FMT((regOffs & 3) == 0, "unexpected regOffs: {}", regOffs); if (regOffs == 0) { - if (host.GetCode() != guest.GetCode()) { + if (host.Idx() != guest.Idx()) { fmov(host.S(), guest.S()); } } else { const auto Predicate = LoadPredicate(); - DataLiteral Data{1U << regOffs}; - dup(VTMP1.Z().VnS(), host.Z().VnS(), 0); - mov(guest.Z().VnS(), Predicate, VTMP1.Z().VnS()); + dup(FEXCore::ARMEmitter::SubRegSize::i32Bit, VTMP1.Z(), host.Z(), 0); + mov(FEXCore::ARMEmitter::SubRegSize::i32Bit, guest.Z(), Predicate, VTMP1.Z()); - EmitData(Data); + EmitData(1U << regOffs); } break; } @@ -230,24 +225,23 @@ DEF_OP(LoadRegister) { case 8: { LOGMAN_THROW_AA_FMT((regOffs & 7) == 0, "unexpected regOffs: {}", regOffs); if (regOffs == 0) { - if (host.GetCode() != guest.GetCode()) { - mov(host.D(), guest.D()); + if (host.Idx() != guest.Idx()) { + dup(ARMEmitter::ScalarRegSize::i64Bit, host, guest, 0); } } else { const auto Predicate = LoadPredicate(); - DataLiteral Data{1U << regOffs}; - dup(VTMP1.Z().VnD(), host.Z().VnD(), 0); - mov(guest.Z().VnD(), Predicate, VTMP1.Z().VnD()); + dup(FEXCore::ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), host.Z(), 0); + mov(FEXCore::ARMEmitter::SubRegSize::i64Bit, guest.Z(), Predicate, VTMP1.Z()); - EmitData(Data); + EmitData(1U << regOffs); } break; } case 16: { LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - if (host.GetCode() != guest.GetCode()) { + if (host.Idx() != guest.Idx()) { mov(host.Q(), guest.Q()); } break; @@ -255,8 +249,8 @@ DEF_OP(LoadRegister) { case 32: { LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - if (host.GetCode() != guest.GetCode()) { - mov(host.Z().VnD(), PRED_TMP_32B.Merging(), guest.Z().VnD()); + if (host.Idx() != guest.Idx()) { + mov(ARMEmitter::SubRegSize::i64Bit, host.Z(), PRED_TMP_32B.Merging(), guest.Z()); } break; } @@ -271,7 +265,7 @@ DEF_OP(LoadRegister) { switch (OpSize) { case 1: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - mov(host.B(), guest.B()); + dup(ARMEmitter::ScalarRegSize::i8Bit, host, guest, 0); break; case 2: @@ -282,28 +276,28 @@ DEF_OP(LoadRegister) { case 4: LOGMAN_THROW_AA_FMT((regOffs & 3) == 0, "unexpected regOffs: {}", regOffs); if (regOffs == 0) { - if (host.GetCode() != guest.GetCode()) { + if (host.Idx() != guest.Idx()) { fmov(host.S(), guest.S()); } } else { - ins(host.V4S(), 0, guest.V4S(), regOffs/4); + ins(ARMEmitter::SubRegSize::i32Bit, host, 0, guest, regOffs/4); } break; case 8: LOGMAN_THROW_AA_FMT((regOffs & 7) == 0, "unexpected regOffs: {}", regOffs); if (regOffs == 0) { - if (host.GetCode() != guest.GetCode()) { - mov(host.D(), guest.D()); + if (host.Idx() != guest.Idx()) { + dup(ARMEmitter::ScalarRegSize::i64Bit, host, guest, 0); } } else { - ins(host.V2D(), 0, guest.V2D(), regOffs/8); + ins(ARMEmitter::SubRegSize::i64Bit, host, 0, guest, regOffs/8); } break; case 16: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - if (host.GetCode() != guest.GetCode()) { + if (host.Idx() != guest.Idx()) { mov(host.Q(), guest.Q()); } break; @@ -329,27 +323,27 @@ DEF_OP(StoreRegister) { LOGMAN_THROW_A_FMT(regId < SRA64.size(), "out of range regId"); const auto reg = SRA64[regId]; + const auto Src = GetReg(Op->Value.ID()); switch (OpSize) { case 1: LOGMAN_THROW_AA_FMT(regOffs == 0 || regOffs == 1, "unexpected regOffs"); - bfi(reg, GetReg(Op->Value.ID()), regOffs * 8, 8); + bfi(ARMEmitter::Size::i64Bit, reg, Src, regOffs * 8, 8); break; case 2: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - bfi(reg, GetReg(Op->Value.ID()), 0, 16); + bfi(ARMEmitter::Size::i64Bit, reg, Src, 0, 16); break; case 4: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - bfi(reg, GetReg(Op->Value.ID()), 0, 32); + bfi(ARMEmitter::Size::i64Bit, reg, Src, 0, 32); break; - case 8: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - if (GetReg(Op->Value.ID()).GetCode() != reg.GetCode()) { - mov(reg, GetReg(Op->Value.ID())); + if (Src.Idx() != reg.Idx()) { + mov(ARMEmitter::Size::i64Bit, reg, Src); } break; @@ -377,11 +371,11 @@ DEF_OP(StoreRegister) { const auto regOffs = Op->Offset & 31; // Compartmentalized setting up of the predicate for the cases that need it. - aarch64::Label DataLocation; + ARMEmitter::ForwardLabel DataLocation; const auto LoadPredicate = [this, &DataLocation] { - const auto Predicate = p0; + const auto Predicate = ARMEmitter::PReg::p0; adr(TMP1, &DataLocation); - ldr(Predicate, SVEMemOperand(TMP1)); + ldr(Predicate, TMP1); return Predicate.Merging(); }; @@ -389,13 +383,12 @@ DEF_OP(StoreRegister) { // emitted data instead of trying to execute it. Place at end of necessary code. // It's helpful to treat LoadPredicate and EmitData as a prologue and epilogue // respectfully. - using DataLiteral = aarch64::Literal; - const auto EmitData = [this, &DataLocation](DataLiteral& Data) { - aarch64::Label PastConstant; + const auto EmitData = [this, &DataLocation](uint32_t Data) { + ARMEmitter::ForwardLabel PastConstant; b(&PastConstant); - bind(&DataLocation); - place(&Data); - bind(&PastConstant); + Bind(&DataLocation); + dc32(Data); + Bind(&PastConstant); }; switch (OpSize) { @@ -403,25 +396,21 @@ DEF_OP(StoreRegister) { LOGMAN_THROW_AA_FMT(regOffs <= 31, "unexpected reg index: {}", regOffs); const auto Predicate = LoadPredicate(); - DataLiteral Data{1U << regOffs}; + dup(ARMEmitter::SubRegSize::i8Bit, VTMP1.Z(), host.Z(), 0); + mov(ARMEmitter::SubRegSize::i8Bit, guest.Z(), Predicate, VTMP1.Z()); - dup(VTMP1.Z().VnB(), host.Z().VnB(), 0); - mov(guest.Z().VnB(), Predicate, VTMP1.Z().VnB()); - - EmitData(Data); + EmitData(1U << regOffs); break; } - + case 2: { LOGMAN_THROW_AA_FMT((regOffs / 2) <= 15, "unexpected reg index: {}", regOffs / 2); const auto Predicate = LoadPredicate(); - DataLiteral Data{1U << regOffs}; + dup(ARMEmitter::SubRegSize::i16Bit, VTMP1.Z(), host.Z(), 0); + mov(ARMEmitter::SubRegSize::i16Bit, guest.Z(), Predicate, VTMP1.Z()); - dup(VTMP1.Z().VnH(), host.Z().VnH(), 0); - mov(guest.Z().VnH(), Predicate, VTMP1.Z().VnH()); - - EmitData(Data); + EmitData(1U << regOffs); break; } @@ -429,12 +418,11 @@ DEF_OP(StoreRegister) { LOGMAN_THROW_AA_FMT((regOffs / 4) <= 7, "unexpected reg index: {}", regOffs / 4); const auto Predicate = LoadPredicate(); - DataLiteral Data{1U << regOffs}; - dup(VTMP1.Z().VnS(), host.Z().VnS(), 0); - mov(guest.Z().VnS(), Predicate, VTMP1.Z().VnS()); + dup(ARMEmitter::SubRegSize::i32Bit, VTMP1.Z(), host.Z(), 0); + mov(ARMEmitter::SubRegSize::i32Bit, guest.Z(), Predicate, VTMP1.Z()); - EmitData(Data); + EmitData(1U << regOffs); break; } @@ -442,18 +430,17 @@ DEF_OP(StoreRegister) { LOGMAN_THROW_AA_FMT((regOffs / 8) <= 3, "unexpected reg index: {}", regOffs / 8); const auto Predicate = LoadPredicate(); - DataLiteral Data{1U << regOffs}; - dup(VTMP1.Z().VnD(), host.Z().VnD(), 0); - mov(guest.Z().VnD(), Predicate, VTMP1.Z().VnD()); + dup(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), host.Z(), 0); + mov(ARMEmitter::SubRegSize::i64Bit, guest.Z(), Predicate, VTMP1.Z()); - EmitData(Data); + EmitData(1U << regOffs); break; } case 16: { LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - if (guest.GetCode() != host.GetCode()) { + if (guest.Idx() != host.Idx()) { mov(guest.Q(), host.Q()); } break; @@ -461,8 +448,8 @@ DEF_OP(StoreRegister) { case 32: { LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - if (guest.GetCode() != host.GetCode()) { - mov(guest.Z().VnD(), PRED_TMP_32B.Merging(), host.Z().VnD()); + if (guest.Idx() != host.Idx()) { + mov(ARMEmitter::SubRegSize::i64Bit, guest.Z(), PRED_TMP_32B.Merging(), host.Z()); } break; } @@ -476,27 +463,29 @@ DEF_OP(StoreRegister) { switch (OpSize) { case 1: - ins(guest.V16B(), regOffs, host.V16B(), 0); + ins(ARMEmitter::SubRegSize::i8Bit, guest, regOffs, host, 0); break; case 2: LOGMAN_THROW_AA_FMT((regOffs & 1) == 0, "unexpected regOffs: {}", regOffs); - ins(guest.V8H(), regOffs/2, host.V8H(), 0); + ins(ARMEmitter::SubRegSize::i16Bit, guest, regOffs / 2, host, 0); break; case 4: LOGMAN_THROW_AA_FMT((regOffs & 3) == 0, "unexpected regOffs: {}", regOffs); - ins(guest.V4S(), regOffs/4, host.V4S(), 0); + // XXX: This had a bug with insert of size 16bit + ins(ARMEmitter::SubRegSize::i32Bit, guest, regOffs / 4, host, 0); break; case 8: LOGMAN_THROW_AA_FMT((regOffs & 7) == 0, "unexpected regOffs: {}", regOffs); - ins(guest.V2D(), regOffs / 8, host.V2D(), 0); + // XXX: This had a bug with insert of size 16bit + ins(ARMEmitter::SubRegSize::i64Bit, guest, regOffs / 8, host, 0); break; case 16: LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs: {}", regOffs); - if (guest.GetCode() != host.GetCode()) { + if (guest.Idx() != host.Idx()) { mov(guest.Q(), host.Q()); } break; @@ -516,7 +505,7 @@ DEF_OP(LoadContextIndexed) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto Index = GetReg(Op->Index.ID()); + const auto Index = GetReg(Op->Index.ID()); if (Op->Class == FEXCore::IR::GPRClass) { switch (Op->Stride) { @@ -524,24 +513,23 @@ DEF_OP(LoadContextIndexed) { case 2: case 4: case 8: { - LoadConstant(TMP1, Op->Stride); - mul(TMP1, Index, TMP1); - add(TMP1, STATE, TMP1); - - const auto Operand = MemOperand(TMP1, Op->BaseOffset); + LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Stride); + mul(ARMEmitter::Size::i64Bit, TMP1, Index, TMP1); + add(ARMEmitter::Size::i64Bit, TMP1, STATE, TMP1.R()); + const auto Dst = GetReg(Node); switch (OpSize) { case 1: - ldrb(GetReg(Node), Operand); + ldrb(Dst, TMP1, Op->BaseOffset); break; case 2: - ldrh(GetReg(Node), Operand); + ldrh(Dst, TMP1, Op->BaseOffset); break; case 4: - ldr(GetReg(Node), Operand); + ldr(Dst.W(), TMP1, Op->BaseOffset); break; case 8: - ldr(GetReg(Node), Operand); + ldr(Dst.X(), TMP1, Op->BaseOffset); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadContextIndexed size: {}", OpSize); @@ -565,32 +553,36 @@ DEF_OP(LoadContextIndexed) { case 8: case 16: case 32: { - LoadConstant(TMP1, Op->Stride); - mul(TMP1, Index, TMP1); - add(TMP1, STATE, TMP1); + LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Stride); + mul(ARMEmitter::Size::i64Bit, TMP1, Index, TMP1); + add(ARMEmitter::Size::i64Bit, TMP1, STATE, TMP1.R()); const auto Dst = GetVReg(Node); switch (OpSize) { case 1: + ldrb(Dst, TMP1, Op->BaseOffset); + break; case 2: + ldrh(Dst, TMP1, Op->BaseOffset); + break; case 4: - case 8: { - const auto NewDst = VRegister(Dst.GetCode(), OpSize * 8); - ldr(NewDst, MemOperand(TMP1, Op->BaseOffset)); + ldr(Dst.S(), TMP1, Op->BaseOffset); + break; + case 8: + ldr(Dst.D(), TMP1, Op->BaseOffset); break; - } case 16: if (Op->BaseOffset % 16 == 0) { - ldr(Dst, MemOperand(TMP1, Op->BaseOffset)); + ldr(Dst.Q(), TMP1, Op->BaseOffset); } else { - add(TMP1, TMP1, Op->BaseOffset); - ldur(Dst, MemOperand(TMP1, Op->BaseOffset)); + add(ARMEmitter::Size::i64Bit, TMP1, TMP1, Op->BaseOffset); + ldur(Dst.Q(), TMP1, Op->BaseOffset); } break; case 32: mov(TMP2, Op->BaseOffset); - ld1b(Dst.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(TMP1, TMP2)); + ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), TMP1, TMP2); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadContextIndexed size: {}", OpSize); @@ -609,34 +601,32 @@ DEF_OP(StoreContextIndexed) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto Index = GetReg(Op->Index.ID()); + const auto Index = GetReg(Op->Index.ID()); if (Op->Class == FEXCore::IR::GPRClass) { - const auto Value = GetReg(Op->Value.ID()); + const auto Value = GetReg(Op->Value.ID()); switch (Op->Stride) { case 1: case 2: case 4: case 8: { - LoadConstant(TMP1, Op->Stride); - mul(TMP1, Index, TMP1); - add(TMP1, STATE, TMP1); - - const auto Operand = MemOperand(TMP1, Op->BaseOffset); + LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Stride); + mul(ARMEmitter::Size::i64Bit, TMP1, Index, TMP1); + add(ARMEmitter::Size::i64Bit, TMP1, STATE, TMP1.R()); switch (OpSize) { case 1: - strb(Value, Operand); + strb(Value, TMP1, Op->BaseOffset); break; case 2: - strh(Value, Operand); + strh(Value, TMP1, Op->BaseOffset); break; case 4: - str(Value.W(), Operand); + str(Value.W(), TMP1, Op->BaseOffset); break; case 8: - str(Value, Operand); + str(Value.X(), TMP1, Op->BaseOffset); break; default: LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed size: {}", OpSize); @@ -662,30 +652,34 @@ DEF_OP(StoreContextIndexed) { case 8: case 16: case 32: { - LoadConstant(TMP1, Op->Stride); - mul(TMP1, Index, TMP1); - add(TMP1, STATE, TMP1); + LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Stride); + mul(ARMEmitter::Size::i64Bit, TMP1, Index, TMP1); + add(ARMEmitter::Size::i64Bit, TMP1, STATE, TMP1.R()); switch (OpSize) { case 1: + strb(Value, TMP1, Op->BaseOffset); + break; case 2: + strh(Value, TMP1, Op->BaseOffset); + break; case 4: - case 8: { - const auto NewValue = VRegister(Value.GetCode(), OpSize * 8); - str(NewValue, MemOperand(TMP1, Op->BaseOffset)); + str(Value.S(), TMP1, Op->BaseOffset); + break; + case 8: + str(Value.D(), TMP1, Op->BaseOffset); break; - } case 16: if (Op->BaseOffset % 16 == 0) { - str(Value, MemOperand(TMP1, Op->BaseOffset)); + str(Value.Q(), TMP1, Op->BaseOffset); } else { - add(TMP1, TMP1, Op->BaseOffset); - stur(Value, MemOperand(TMP1, Op->BaseOffset)); + add(ARMEmitter::Size::i64Bit, TMP1, TMP1, Op->BaseOffset); + stur(Value.Q(), TMP1, Op->BaseOffset); } break; case 32: mov(TMP2, Op->BaseOffset); - st1b(Value.Z().VnB(), PRED_TMP_32B, SVEMemOperand(TMP1, TMP2)); + st1b(Value.Z(), PRED_TMP_32B, TMP1, TMP2); break; default: LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed size: {}", OpSize); @@ -706,21 +700,22 @@ DEF_OP(SpillRegister) { const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize; if (Op->Class == FEXCore::IR::GPRClass) { + const auto Src = GetReg(Op->Value.ID()); switch (OpSize) { case 1: { - strb(GetReg(Op->Value.ID()), MemOperand(sp, SlotOffset)); + strb(Src, ARMEmitter::Reg::rsp, SlotOffset); break; } case 2: { - strh(GetReg(Op->Value.ID()), MemOperand(sp, SlotOffset)); + strh(Src, ARMEmitter::Reg::rsp, SlotOffset); break; } case 4: { - str(GetReg(Op->Value.ID()), MemOperand(sp, SlotOffset)); + str(Src.W(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 8: { - str(GetReg(Op->Value.ID()), MemOperand(sp, SlotOffset)); + str(Src.X(), ARMEmitter::Reg::rsp, SlotOffset); break; } default: @@ -732,20 +727,20 @@ DEF_OP(SpillRegister) { switch (OpSize) { case 4: { - str(Src.S(), MemOperand(sp, SlotOffset)); + str(Src.S(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 8: { - str(Src.D(), MemOperand(sp, SlotOffset)); + str(Src.D(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 16: { - str(Src, MemOperand(sp, SlotOffset)); + str(Src.Q(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 32: { mov(TMP3, SlotOffset); - st1b(Src.Z().VnB(), PRED_TMP_32B, SVEMemOperand(sp, TMP3)); + st1b(Src.Z(), PRED_TMP_32B, ARMEmitter::Reg::rsp, TMP3); break; } default: @@ -763,21 +758,22 @@ DEF_OP(FillRegister) { const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize; if (Op->Class == FEXCore::IR::GPRClass) { + const auto Dst = GetReg(Node); switch (OpSize) { case 1: { - ldrb(GetReg(Node), MemOperand(sp, SlotOffset)); + ldrb(Dst, ARMEmitter::Reg::rsp, SlotOffset); break; } case 2: { - ldrh(GetReg(Node), MemOperand(sp, SlotOffset)); + ldrh(Dst, ARMEmitter::Reg::rsp, SlotOffset); break; } case 4: { - ldr(GetReg(Node), MemOperand(sp, SlotOffset)); + ldr(Dst.W(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 8: { - ldr(GetReg(Node), MemOperand(sp, SlotOffset)); + ldr(Dst.X(), ARMEmitter::Reg::rsp, SlotOffset); break; } default: @@ -789,20 +785,20 @@ DEF_OP(FillRegister) { switch (OpSize) { case 4: { - ldr(Dst.S(), MemOperand(sp, SlotOffset)); + ldr(Dst.S(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 8: { - ldr(Dst.D(), MemOperand(sp, SlotOffset)); + ldr(Dst.D(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 16: { - ldr(Dst, MemOperand(sp, SlotOffset)); + ldr(Dst.Q(), ARMEmitter::Reg::rsp, SlotOffset); break; } case 32: { mov(TMP3, SlotOffset); - ld1b(Dst.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(sp, TMP3)); + ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), ARMEmitter::Reg::rsp, TMP3); break; } default: @@ -816,32 +812,35 @@ DEF_OP(FillRegister) { DEF_OP(LoadFlag) { auto Op = IROp->C(); - auto Dst = GetReg(Node); - ldrb(Dst, MemOperand(STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)); + auto Dst = GetReg(Node); + ldrb(Dst, STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag); } DEF_OP(StoreFlag) { auto Op = IROp->C(); - strb(GetReg(Op->Value.ID()), MemOperand(STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)); + strb(GetReg(Op->Value.ID()), STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag); } -MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) { +FEXCore::ARMEmitter::ExtendedMemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, + FEXCore::ARMEmitter::Register Base, + IR::OrderedNodeWrapper Offset, + IR::MemOffsetType OffsetType, + uint8_t OffsetScale) { if (Offset.IsInvalid()) { - return MemOperand(Base); + return FEXCore::ARMEmitter::ExtendedMemOperand(Base, ARMEmitter::IndexType::OFFSET, 0); } else { if (OffsetScale != 1 && OffsetScale != AccessSize) { - LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetScale: {}", OffsetScale); + LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetScale: {}", OffsetScale); } uint64_t Const; if (IsInlineConstant(Offset, &Const)) { - return MemOperand(Base, Const); + return FEXCore::ARMEmitter::ExtendedMemOperand(Base, ARMEmitter::IndexType::OFFSET, Const); } else { - auto RegOffset = GetReg(Offset.ID()); + auto RegOffset = GetReg(Offset.ID()); switch(OffsetType.Val) { - case IR::MEM_OFFSET_SXTX.Val: return MemOperand(Base, RegOffset, Extend::SXTX, (int)std::log2(OffsetScale) ); - case IR::MEM_OFFSET_UXTW.Val: return MemOperand(Base, RegOffset.W(), Extend::UXTW, (int)std::log2(OffsetScale) ); - case IR::MEM_OFFSET_SXTW.Val: return MemOperand(Base, RegOffset.W(), Extend::SXTW, (int)std::log2(OffsetScale) ); - + case IR::MEM_OFFSET_SXTX.Val: return FEXCore::ARMEmitter::ExtendedMemOperand(Base, RegOffset, FEXCore::ARMEmitter::ExtendedType::SXTX, (int)std::log2(OffsetScale) ); + case IR::MEM_OFFSET_UXTW.Val: return FEXCore::ARMEmitter::ExtendedMemOperand(Base, RegOffset, FEXCore::ARMEmitter::ExtendedType::UXTW, (int)std::log2(OffsetScale) ); + case IR::MEM_OFFSET_SXTW.Val: return FEXCore::ARMEmitter::ExtendedMemOperand(Base, RegOffset, FEXCore::ARMEmitter::ExtendedType::SXTW, (int)std::log2(OffsetScale) ); default: LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetType: {}", OffsetType.Val); break; } } @@ -850,19 +849,30 @@ MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Registe FEX_UNREACHABLE; } -SVEMemOperand Arm64JITCore::GenerateSVEMemOperand(uint8_t AccessSize, - aarch64::Register Base, +static auto ConvertExtendedType(IR::MemOffsetType OffsetType) -> ARMEmitter::ExtendedType { + switch (OffsetType.Val) { + case IR::MEM_OFFSET_SXTX.Val: return ARMEmitter::ExtendedType::SXTX; + case IR::MEM_OFFSET_UXTW.Val: return ARMEmitter::ExtendedType::UXTW; + case IR::MEM_OFFSET_SXTW.Val: return ARMEmitter::ExtendedType::SXTW; + + default: LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetType: {}", OffsetType.Val); break; + } + FEX_UNREACHABLE; +} + +FEXCore::ARMEmitter::SVEMemOperand Arm64JITCore::GenerateSVEMemOperand(uint8_t AccessSize, + FEXCore::ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, [[maybe_unused]] uint8_t OffsetScale) { if (Offset.IsInvalid()) { - return SVEMemOperand(Base); + return FEXCore::ARMEmitter::SVEMemOperand(Base.X(), 0); } uint64_t Const{}; if (IsInlineConstant(Offset, &Const)) { if (Const == 0) { - return SVEMemOperand(Base); + return FEXCore::ARMEmitter::SVEMemOperand(Base.X(), 0); } const auto SignedConst = static_cast(Const); @@ -885,13 +895,13 @@ SVEMemOperand Arm64JITCore::GenerateSVEMemOperand(uint8_t AccessSize, // then we can encode it as an immediate offset. // if (IsCleanlyDivisible && Index >= -8 && Index <= 7) { - return SVEMemOperand(Base, static_cast(Index), SVE_MUL_VL); + return FEXCore::ARMEmitter::SVEMemOperand(Base.X(), static_cast(Index)); } // If we can't do that for whatever reason, then unfortunately, we need // to move it over to a temporary to use as an offset. mov(TMP1, Const); - return SVEMemOperand(Base, TMP1); + return FEXCore::ARMEmitter::SVEMemOperand(Base.X(), TMP1); } // Otherwise handle it like normal. @@ -901,19 +911,19 @@ SVEMemOperand Arm64JITCore::GenerateSVEMemOperand(uint8_t AccessSize, LOGMAN_THROW_A_FMT(OffsetType.Val == IR::MEM_OFFSET_SXTX.Val, "Currently only the default offset type (SXTX) is supported."); - const auto RegOffset = GetReg(Offset.ID()); - return SVEMemOperand(Base, RegOffset); + const auto RegOffset = GetReg(Offset.ID()); + return FEXCore::ARMEmitter::SVEMemOperand(Base.X(), RegOffset.X()); } DEF_OP(LoadMem) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto MemReg = GetReg(Op->Addr.ID()); + const auto MemReg = GetReg(Op->Addr.ID()); + const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); if (Op->Class == FEXCore::IR::GPRClass) { - const auto Dst = GetReg(Node); - const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); + const auto Dst = GetReg(Node); switch (OpSize) { case 1: @@ -926,7 +936,7 @@ DEF_OP(LoadMem) { ldr(Dst.W(), MemSrc); break; case 8: - ldr(Dst, MemSrc); + ldr(Dst.X(), MemSrc); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadMem size: {}", OpSize); @@ -938,18 +948,23 @@ DEF_OP(LoadMem) { switch (OpSize) { case 1: + ldrb(Dst, MemSrc); + break; case 2: + ldrh(Dst, MemSrc); + break; case 4: + ldr(Dst.S(), MemSrc); + break; case 8: - case 16: { - const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - const auto NewDst = VRegister(Dst.GetCode(), OpSize * 8); - ldr(NewDst, MemSrc); + ldr(Dst.D(), MemSrc); + break; + case 16: + ldr(Dst.Q(), MemSrc); break; - } case 32: { const auto Operand = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - ld1b(Dst.Z().VnB(), PRED_TMP_32B.Zeroing(), Operand); + ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), Operand); break; } default: @@ -963,43 +978,32 @@ DEF_OP(LoadMemTSO) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto MemReg = GetReg(Op->Addr.ID()); - - const auto GetMemSrc = [&] { - const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - - if (CTX->HostFeatures.SupportsTSOImm9) { - // RCPC2 means that the offset must be an inline constant - LOGMAN_THROW_A_FMT(MemSrc.IsRegisterOffset() == false, - "RCPC2 doesn't support register offset. Only Immediate offset"); - } else { - LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid(), "LoadMemTSO: No offset allowed"); - } - - return MemSrc; - }; + const auto MemReg = GetReg(Op->Addr.ID()); if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) { - const auto MemSrc = GetMemSrc(); + const auto Dst = GetReg(Node); + uint64_t Offset = 0; + if (!Op->Offset.IsInvalid()) { + (void)IsInlineConstant(Op->Offset, &Offset); + } if (OpSize == 1) { // 8bit load is always aligned to natural alignment - const auto Dst = GetReg(Node); - ldapurb(Dst, MemSrc); + const auto Dst = GetReg(Node); + ldapurb(Dst, MemReg, Offset); } else { // Aligned nop(); - const auto Dst = GetReg(Node); switch (OpSize) { case 2: - ldapurh(Dst, MemSrc); + ldapurh(Dst, MemReg, Offset); break; case 4: - ldapur(Dst.W(), MemSrc); + ldapur(Dst.W(), MemReg, Offset); break; case 8: - ldapur(Dst, MemSrc); + ldapur(Dst.X(), MemReg, Offset); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", OpSize); @@ -1009,26 +1013,24 @@ DEF_OP(LoadMemTSO) { } } else if (CTX->HostFeatures.SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) { - const auto MemSrc = GetMemSrc(); + const auto Dst = GetReg(Node); if (OpSize == 1) { // 8bit load is always aligned to natural alignment - const auto Dst = GetReg(Node); - ldaprb(Dst, MemSrc); + ldaprb(Dst, MemReg); } else { // Aligned - const auto Dst = GetReg(Node); nop(); switch (OpSize) { case 2: - ldaprh(Dst, MemSrc); + ldaprh(Dst, MemReg); break; case 4: - ldapr(Dst.W(), MemSrc); + ldapr(Dst.W(), MemReg); break; case 8: - ldapr(Dst, MemSrc); + ldapr(Dst.X(), MemReg); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", OpSize); @@ -1038,26 +1040,23 @@ DEF_OP(LoadMemTSO) { } } else if (Op->Class == FEXCore::IR::GPRClass) { - const auto MemSrc = GetMemSrc(); - + const auto Dst = GetReg(Node); if (OpSize == 1) { // 8bit load is always aligned to natural alignment - const auto Dst = GetReg(Node); - ldarb(Dst, MemSrc); + ldarb(Dst, MemReg); } else { // Aligned - const auto Dst = GetReg(Node); nop(); switch (OpSize) { case 2: - ldarh(Dst, MemSrc); + ldarh(Dst, MemReg); break; case 4: - ldar(Dst.W(), MemSrc); + ldar(Dst.W(), MemReg); break; case 8: - ldar(Dst, MemSrc); + ldar(Dst.X(), MemReg); break; default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", OpSize); @@ -1067,29 +1066,35 @@ DEF_OP(LoadMemTSO) { } } else { - dmb(InnerShareable, BarrierAll); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); const auto Dst = GetVReg(Node); + const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); switch (OpSize) { case 1: + ldrb(Dst, MemSrc); + break; case 2: + ldrh(Dst, MemSrc); + break; case 4: + ldr(Dst.S(), MemSrc); + break; case 8: - case 16: { - const auto MemSrc = GetMemSrc(); - const auto NewDst = VRegister(Dst.GetCode(), OpSize * 8); - ldr(NewDst, MemSrc); + ldr(Dst.D(), MemSrc); + break; + case 16: + ldr(Dst.Q(), MemSrc); break; - } case 32: { const auto MemSrc = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - ld1b(Dst.Z().VnB(), PRED_TMP_32B.Zeroing(), MemSrc); + ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), MemSrc); break; } default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", OpSize); break; } - dmb(InnerShareable, BarrierAll); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); } } @@ -1097,23 +1102,23 @@ DEF_OP(StoreMem) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto MemReg = GetReg(Op->Addr.ID()); + const auto MemReg = GetReg(Op->Addr.ID()); + const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); if (Op->Class == FEXCore::IR::GPRClass) { - const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - + const auto Src = GetReg(Op->Value.ID()); switch (OpSize) { case 1: - strb(GetReg(Op->Value.ID()), MemSrc); + strb(Src, MemSrc); break; case 2: - strh(GetReg(Op->Value.ID()), MemSrc); + strh(Src, MemSrc); break; case 4: - str(GetReg(Op->Value.ID()), MemSrc); + str(Src.W(), MemSrc); break; case 8: - str(GetReg(Op->Value.ID()), MemSrc); + str(Src.X(), MemSrc); break; default: LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", OpSize); @@ -1124,19 +1129,29 @@ DEF_OP(StoreMem) { const auto Src = GetVReg(Op->Value.ID()); switch (OpSize) { - case 1: - case 2: - case 4: - case 8: + case 1: { + strb(Src, MemSrc); + break; + } + case 2: { + strh(Src, MemSrc); + break; + } + case 4: { + str(Src.S(), MemSrc); + break; + } + case 8: { + str(Src.D(), MemSrc); + break; + } case 16: { - const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - const auto NewSrc = VRegister(Src.GetCode(), OpSize * 8); - str(NewSrc, MemSrc); + str(Src.Q(), MemSrc); break; } case 32: { const auto MemSrc = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - st1b(Src.Z().VnB(), PRED_TMP_32B, MemSrc); + st1b(Src.Z(), PRED_TMP_32B, MemSrc); break; } default: @@ -1150,40 +1165,30 @@ DEF_OP(StoreMemTSO) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto MemReg = GetReg(Op->Addr.ID()); - - const auto GetMemSrc = [&] { - const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - - if (CTX->HostFeatures.SupportsTSOImm9) { - // RCPC2 means that the offset must be an inline constant - LOGMAN_THROW_A_FMT(MemSrc.IsRegisterOffset() == false, "RCPC2 doesn't support register offset. Only Immediate offset"); - } - else { - LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid(), "StoreMemTSO: No offset allowed"); - } - - return MemSrc; - }; + const auto MemReg = GetReg(Op->Addr.ID()); if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) { - const auto MemSrc = GetMemSrc(); + const auto Src = GetReg(Op->Value.ID()); + uint64_t Offset = 0; + if (!Op->Offset.IsInvalid()) { + (void)IsInlineConstant(Op->Offset, &Offset); + } if (OpSize == 1) { // 8bit load is always aligned to natural alignment - stlurb(GetReg(Op->Value.ID()), MemSrc); + stlurb(Src, MemReg, Offset); } else { nop(); switch (OpSize) { case 2: - stlurh(GetReg(Op->Value.ID()), MemSrc); + stlurh(Src, MemReg, Offset); break; case 4: - stlur(GetReg(Op->Value.ID()), MemSrc); + stlur(Src.W(), MemReg, Offset); break; case 8: - stlur(GetReg(Op->Value.ID()), MemSrc); + stlur(Src.X(), MemReg, Offset); break; default: LOGMAN_MSG_A_FMT("Unhandled StoreMemTSO size: {}", OpSize); @@ -1193,23 +1198,23 @@ DEF_OP(StoreMemTSO) { } } else if (Op->Class == FEXCore::IR::GPRClass) { - const auto MemSrc = GetMemSrc(); + const auto Src = GetReg(Op->Value.ID()); if (OpSize == 1) { // 8bit load is always aligned to natural alignment - stlrb(GetReg(Op->Value.ID()), MemSrc); + stlrb(Src, MemReg); } else { nop(); switch (OpSize) { case 2: - stlrh(GetReg(Op->Value.ID()), MemSrc); + stlrh(Src, MemReg); break; case 4: - stlr(GetReg(Op->Value.ID()), MemSrc); + stlr(Src.W(), MemReg); break; case 8: - stlr(GetReg(Op->Value.ID()), MemSrc); + stlr(Src.X(), MemReg); break; default: LOGMAN_MSG_A_FMT("Unhandled StoreMemTSO size: {}", OpSize); @@ -1219,29 +1224,35 @@ DEF_OP(StoreMemTSO) { } } else { - dmb(InnerShareable, BarrierAll); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); const auto Src = GetVReg(Op->Value.ID()); + const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); switch (OpSize) { case 1: + strb(Src, MemSrc); + break; case 2: + strh(Src, MemSrc); + break; case 4: + str(Src.S(), MemSrc); + break; case 8: - case 16: { - const auto MemSrc = GetMemSrc(); - const auto NewSrc = VRegister(Src.GetCode(), OpSize * 8); - str(NewSrc, MemSrc); + str(Src.D(), MemSrc); + break; + case 16: + str(Src.Q(), MemSrc); break; - } case 32: { const auto Operand = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - st1b(Src.Z().VnB(), PRED_TMP_32B, Operand); + st1b(Src.Z(), PRED_TMP_32B, Operand); break; } default: LOGMAN_MSG_A_FMT("Unhandled StoreMemTSO size: {}", OpSize); break; } - dmb(InnerShareable, BarrierAll); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); } } @@ -1249,27 +1260,26 @@ DEF_OP(ParanoidLoadMemTSO) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto Addr = GetReg(Op->Addr.ID()); - const auto MemSrc = MemOperand(Addr); + const auto Addr = GetReg(Op->Addr.ID()); if (!Op->Offset.IsInvalid()) { LOGMAN_MSG_A_FMT("ParanoidLoadMemTSO: No offset allowed"); } if (Op->Class == FEXCore::IR::GPRClass) { - const auto Dst = GetReg(Node); + const auto Dst = GetReg(Node); switch (OpSize) { case 1: - ldarb(Dst, MemSrc); + ldarb(Dst, Addr); break; case 2: - ldarh(Dst, MemSrc); + ldarh(Dst, Addr); break; case 4: - ldar(Dst.W(), MemSrc); + ldar(Dst.W(), Addr); break; case 8: - ldar(Dst, MemSrc); + ldar(Dst.X(), Addr); break; default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); @@ -1280,32 +1290,32 @@ DEF_OP(ParanoidLoadMemTSO) { const auto Dst = GetVReg(Node); switch (OpSize) { case 1: - ldarb(TMP1.W(), MemSrc); - fmov(Dst.B(), TMP1.B()); + ldarb(TMP1, Addr); + ins(ARMEmitter::SubRegSize::i8Bit, Dst, 0, TMP1); break; case 2: - ldarh(TMP1.W(), MemSrc); - fmov(Dst.H(), TMP1.W()); + ldarh(TMP1, Addr); + ins(ARMEmitter::SubRegSize::i16Bit, Dst, 0, TMP1); break; case 4: - ldar(TMP1.W(), MemSrc); - fmov(Dst.S(), TMP1.W()); + ldar(TMP1.W(), Addr); + ins(ARMEmitter::SubRegSize::i32Bit, Dst, 0, TMP1); break; case 8: - ldar(TMP1, MemSrc); - fmov(Dst.D(), TMP1); + ldar(TMP1, Addr); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, TMP1); break; case 16: nop(); - ldaxp(TMP1, TMP2, MemSrc); + ldaxp(ARMEmitter::Size::i64Bit, TMP1, TMP2, Addr); clrex(); - mov(Dst.V2D(), 0, TMP1); - mov(Dst.V2D(), 1, TMP2); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, TMP1); + ins(ARMEmitter::SubRegSize::i64Bit, Dst, 1, TMP2); break; case 32: - dmb(InnerShareable, BarrierAll); - ld1b(Dst.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(Addr)); - dmb(InnerShareable, BarrierAll); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); + ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), Addr); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); break; default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); @@ -1318,26 +1328,26 @@ DEF_OP(ParanoidStoreMemTSO) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; - const auto Addr = GetReg(Op->Addr.ID()); - const auto MemSrc = MemOperand(Addr); + const auto Addr = GetReg(Op->Addr.ID()); if (!Op->Offset.IsInvalid()) { LOGMAN_MSG_A_FMT("ParanoidStoreMemTSO: No offset allowed"); } if (Op->Class == FEXCore::IR::GPRClass) { + const auto Src = GetReg(Op->Value.ID()); switch (OpSize) { case 1: - stlrb(GetReg(Op->Value.ID()), MemSrc); + stlrb(Src, Addr); break; case 2: - stlrh(GetReg(Op->Value.ID()), MemSrc); + stlrh(Src, Addr); break; case 4: - stlr(GetReg(Op->Value.ID()), MemSrc); + stlr(Src.W(), Addr); break; case 8: - stlr(GetReg(Op->Value.ID()), MemSrc); + stlr(Src.X(), Addr); break; default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); @@ -1349,38 +1359,38 @@ DEF_OP(ParanoidStoreMemTSO) { switch (OpSize) { case 1: - mov(TMP1.W(), Src.V16B(), 0); - stlrb(TMP1, MemSrc); + umov(TMP1, Src, 0); + stlrb(TMP1, Addr); break; case 2: - mov(TMP1.W(), Src.V8H(), 0); - stlrh(TMP1, MemSrc); + umov(TMP1, Src, 0); + stlrh(TMP1, Addr); break; case 4: - mov(TMP1.W(), Src.V4S(), 0); - stlr(TMP1.W(), MemSrc); + umov(TMP1, Src, 0); + stlr(TMP1.W(), Addr); break; case 8: - mov(TMP1, Src.V2D(), 0); - stlr(TMP1, MemSrc); + umov(TMP1, Src, 0); + stlr(TMP1, Addr); break; case 16: { // Move vector to GPRs - mov(TMP1, Src.V2D(), 0); - mov(TMP2, Src.V2D(), 1); - Label B; - bind(&B); + umov(TMP1, Src, 0); + umov(TMP2, Src, 1); + ARMEmitter::BackwardLabel B; + Bind(&B); // ldaxp must not have both the destination registers be the same - ldaxp(xzr, TMP3, MemSrc); // <- Can hit SIGBUS. Overwritten with DMB - stlxp(TMP3, TMP1, TMP2, MemSrc); // <- Can also hit SIGBUS - cbnz(TMP3, &B); // < Overwritten with DMB + ldaxp(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::zr, TMP3, Addr); // <- Can hit SIGBUS. Overwritten with DMB + stlxp(ARMEmitter::Size::i64Bit, TMP3, TMP1, TMP2, Addr); // <- Can also hit SIGBUS + cbnz(ARMEmitter::Size::i64Bit, TMP3, &B); // < Overwritten with DMB break; } case 32: { - dmb(InnerShareable, BarrierAll); - st1b(Src.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(Addr)); - dmb(InnerShareable, BarrierAll); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); + st1b(Src, PRED_TMP_32B, Addr, 0); + dmb(FEXCore::ARMEmitter::BarrierScope::ISH); break; } default: @@ -1393,36 +1403,36 @@ DEF_OP(ParanoidStoreMemTSO) { DEF_OP(CacheLineClear) { auto Op = IROp->C(); - auto MemReg = GetReg(Op->Addr.ID()); + auto MemReg = GetReg(Op->Addr.ID()); // Clear dcache only // icache doesn't matter here since the guest application shouldn't be calling clflush on JIT code. - mov(TMP1, MemReg); + mov(TMP1, MemReg.X()); for (size_t i = 0; i < std::max(1U, CTX->HostFeatures.DCacheLineSize / 64U); ++i) { - dc(DataCacheOp::CVAU, TMP1); - add(TMP1, TMP1, CTX->HostFeatures.DCacheLineSize); + dc(ARMEmitter::DataCacheOperation::CVAU, TMP1); + add(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->HostFeatures.DCacheLineSize); } - dsb(InnerShareable, BarrierAll); + dsb(FEXCore::ARMEmitter::BarrierScope::ISH); } DEF_OP(CacheLineZero) { auto Op = IROp->C(); - auto MemReg = GetReg(Op->Addr.ID()); + auto MemReg = GetReg(Op->Addr.ID()); if (CTX->HostFeatures.SupportsCLZERO) { // We can use this instruction directly - dc(DataCacheOp::ZVA, MemReg); + dc(ARMEmitter::DataCacheOperation::ZVA, MemReg); } else { // We must walk the cacheline ourselves // Force cacheline alignment - and_(TMP1, MemReg, ~(CPUIDEmu::CACHELINE_SIZE - 1)); + and_(ARMEmitter::Size::i64Bit, TMP1, MemReg, ~(CPUIDEmu::CACHELINE_SIZE - 1)); // This will end up being four STPs // Depending on uarch it could be slightly more efficient in instructions emitted // and uops to use vector pair STP, but we want the non-temporal bit specifically here for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += 16) { - stnp(xzr, xzr, MemOperand(TMP1, i, Offset)); + stnp(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, TMP1, i); } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp index 8b5b04c59..453ac9b06 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp @@ -5,12 +5,11 @@ $end_info$ */ #include +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" #include "Interface/Core/JIT/Arm64/JITClass.h" #include "FEXCore/Debug/InternalThreadState.h" namespace FEXCore::CPU { -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(GuestOpcode) { @@ -23,13 +22,13 @@ DEF_OP(Fence) { auto Op = IROp->C(); switch (Op->Fence) { case IR::Fence_Load.Val: - dmb(FullSystem, BarrierReads); + dmb(FEXCore::ARMEmitter::BarrierScope::LD); break; case IR::Fence_LoadStore.Val: - dmb(FullSystem, BarrierAll); + dmb(FEXCore::ARMEmitter::BarrierScope::SY); break; case IR::Fence_Store.Val: - dmb(FullSystem, BarrierWrites); + dmb(FEXCore::ARMEmitter::BarrierScope::ST); break; default: LOGMAN_MSG_A_FMT("Unknown Fence: {}", Op->Fence); break; } @@ -52,88 +51,88 @@ DEF_OP(Break) { uint64_t Constant{}; memcpy(&Constant, &State, sizeof(State)); - LoadConstant(x1, Constant); - str(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData))); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Constant); + str(ARMEmitter::XReg::x1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData)); switch (Op->Reason.Signal) { case SIGILL: - ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGILL))); + ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGILL)); br(TMP1); break; case SIGTRAP: - ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP))); + ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)); br(TMP1); break; case SIGSEGV: - ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGSEGV))); + ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGSEGV)); br(TMP1); break; default: - ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP))); + ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)); br(TMP1); break; } } DEF_OP(GetRoundingMode) { - auto Dst = GetReg(Node); - mrs(Dst, FPCR); - lsr(Dst, Dst, 22); + auto Dst = GetReg(Node); + mrs(Dst, ARMEmitter::SystemRegister::FPCR); + lsr(ARMEmitter::Size::i64Bit, Dst, Dst, 22); // FTZ is already in the correct location // Rounding mode is different - and_(TMP1, Dst, 0b11); + and_(ARMEmitter::Size::i64Bit, TMP1, Dst, 0b11); - cmp(TMP1, 1); - LoadConstant(TMP3, IR::ROUND_MODE_POSITIVE_INFINITY); - csel(TMP2, TMP3, xzr, vixl::aarch64::Condition::eq); + cmp(ARMEmitter::Size::i64Bit, TMP1, 1); + LoadConstant(ARMEmitter::Size::i64Bit, TMP3, IR::ROUND_MODE_POSITIVE_INFINITY); + csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, ARMEmitter::Reg::zr, ARMEmitter::Condition::CC_EQ); - cmp(TMP1, 2); - LoadConstant(TMP3, IR::ROUND_MODE_NEGATIVE_INFINITY); - csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + cmp(ARMEmitter::Size::i64Bit, TMP1, 2); + LoadConstant(ARMEmitter::Size::i64Bit, TMP3, IR::ROUND_MODE_NEGATIVE_INFINITY); + csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ); - cmp(TMP1, 3); - LoadConstant(TMP3, IR::ROUND_MODE_TOWARDS_ZERO); - csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + cmp(ARMEmitter::Size::i64Bit, TMP1, 3); + LoadConstant(ARMEmitter::Size::i64Bit, TMP3, IR::ROUND_MODE_TOWARDS_ZERO); + csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ); - orr(Dst, Dst, TMP2); + orr(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2.R()); - bfi(Dst, TMP2, 0, 2); + bfi(ARMEmitter::Size::i64Bit, Dst, TMP2, 0, 2); } DEF_OP(SetRoundingMode) { auto Op = IROp->C(); - auto Src = GetReg(Op->RoundMode.ID()); + auto Src = GetReg(Op->RoundMode.ID()); // Setup the rounding flags correctly - and_(TMP1, Src, 0b11); + and_(ARMEmitter::Size::i64Bit, TMP1, Src, 0b11); - cmp(TMP1, IR::ROUND_MODE_POSITIVE_INFINITY); - LoadConstant(TMP3, 1); - csel(TMP2, TMP3, xzr, vixl::aarch64::Condition::eq); + cmp(ARMEmitter::Size::i64Bit, TMP1, IR::ROUND_MODE_POSITIVE_INFINITY); + LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 1); + csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, ARMEmitter::Reg::zr, ARMEmitter::Condition::CC_EQ); - cmp(TMP1, IR::ROUND_MODE_NEGATIVE_INFINITY); - LoadConstant(TMP3, 2); - csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + cmp(ARMEmitter::Size::i64Bit, TMP1, IR::ROUND_MODE_NEGATIVE_INFINITY); + LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 2); + csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ); - cmp(TMP1, IR::ROUND_MODE_TOWARDS_ZERO); - LoadConstant(TMP3, 3); - csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + cmp(ARMEmitter::Size::i64Bit, TMP1, IR::ROUND_MODE_TOWARDS_ZERO); + LoadConstant(ARMEmitter::Size::i64Bit, TMP3, 3); + csel(ARMEmitter::Size::i64Bit, TMP2, TMP3, TMP2, ARMEmitter::Condition::CC_EQ); - mrs(TMP1, FPCR); + mrs(TMP1, ARMEmitter::SystemRegister::FPCR); // vixl simulator doesn't support anything beyond ties-to-even rounding #ifndef VIXL_SIMULATOR // Insert the rounding flags - bfi(TMP1, TMP2, 22, 2); + bfi(ARMEmitter::Size::i64Bit, TMP1, TMP2, 22, 2); #endif // Insert the FTZ flag - lsr(TMP2, Src, 2); - bfi(TMP1, TMP2, 24, 1); + lsr(ARMEmitter::Size::i64Bit, TMP2, Src, 2); + bfi(ARMEmitter::Size::i64Bit, TMP1, TMP2, 24, 1); // Now save the new FPCR - msr(FPCR, TMP1); + msr(ARMEmitter::SystemRegister::FPCR, TMP1); } DEF_OP(Print) { @@ -143,17 +142,16 @@ DEF_OP(Print) { SpillStaticRegs(); if (IsGPR(Op->Value.ID())) { - mov(x0, GetReg(Op->Value.ID())); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue))); + mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->Value.ID())); + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue)); } else { - fmov(x0, GetVReg(Op->Value.ID()).V1D()); - // Bug in vixl that source vector needs to b V1D rather than V2D? - fmov(x1, GetVReg(Op->Value.ID()).V1D(), 1); - ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue))); + fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetVReg(Op->Value.ID()), false); + fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetVReg(Op->Value.ID()), true); + ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue)); } - blr(x3); + blr(ARMEmitter::Reg::r3); FillStaticRegs(); PopDynamicRegsAndLR(); @@ -173,27 +171,27 @@ DEF_OP(ProcessorID) { // 16bit LoadConstant to be a single instruction // We must always spill at least one register (x8) so this value always has a bit set // This gives the signal handler a value to check to see if we are in a syscall at all - LoadConstant(x0, SpillMask & 0xFFFF); - str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo))); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SpillMask & 0xFFFF); + str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)); // Allocate some temporary space for storing the uint32_t CPU and Node IDs - sub(sp, sp, 16); + sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16); // Load the getcpu syscall number - LoadConstant(x8, SYS_getcpu); + LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, SYS_getcpu); // CPU pointer in x0 - add(x0, sp, 0); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0); // Node in x1 - add(x1, sp, 4); + add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, ARMEmitter::Reg::rsp, 4); svc(0); // On updated signal mask we can receive a signal RIGHT HERE // Load the values returned by the kernel - ldp(w0, w1, MemOperand(sp)); + ldp(ARMEmitter::WReg::w0, ARMEmitter::WReg::w1, ARMEmitter::Reg::rsp); // Deallocate stack space - sub(sp, sp, 16); + sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16); // Now that we are done in the syscall we need to carefully peel back the state // First unspill the registers from before @@ -201,14 +199,13 @@ DEF_OP(ProcessorID) { // Now the registers we've spilled are back in their original host registers // We can safely claim we are no longer in a syscall - str(xzr, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo))); - + str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)); // Now store the result in the destination in the expected format // uint32_t Res = (node << 12) | cpu; // CPU is in w0 // Node is in w1 - orr(GetReg(Node), x0, Operand(x1, LSL, 12)); + orr(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0, ARMEmitter::Reg::r1, ARMEmitter::ShiftType::LSL, 12); } DEF_OP(RDRAND) { @@ -216,21 +213,21 @@ DEF_OP(RDRAND) { // Results are in x0, x1 // Results want to be in a i64v2 vector - auto Dst = GetRegPair(Node); + auto Dst = GetRegPair(Node); if (Op->GetReseeded) { - mrs(Dst.first, RNDRRS); + mrs(Dst.first, ARMEmitter::SystemRegister::RNDRRS); } else { - mrs(Dst.first, RNDR); + mrs(Dst.first, ARMEmitter::SystemRegister::RNDR); } // If the rng number is valid then NZCV is 0b0000, otherwise NZCV is 0b0100 - cset(Dst.second, Condition::ne); + cset(ARMEmitter::Size::i64Bit, Dst.second, ARMEmitter::Condition::CC_NE); } DEF_OP(Yield) { - hint(SystemHint::YIELD); + yield(); } #undef DEF_OP diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MoveOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MoveOps.cpp index 090eb9443..81db3603b 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MoveOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MoveOps.cpp @@ -7,64 +7,37 @@ $end_info$ #include "Interface/Core/JIT/Arm64/JITClass.h" namespace FEXCore::CPU { - -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(ExtractElementPair) { auto Op = IROp->C(); - switch (Op->Header.Size) { - case 4: { - auto Src = GetRegPair(Op->Pair.ID()); - std::array Regs = {Src.first, Src.second}; - mov (GetReg(Node), Regs[Op->Element]); - break; - } - case 8: { - auto Src = GetRegPair(Op->Pair.ID()); - std::array Regs = {Src.first, Src.second}; - mov (GetReg(Node), Regs[Op->Element]); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Size"); break; - } + LOGMAN_THROW_AA_FMT(Op->Header.Size == 4 || Op->Header.Size == 8, "Invalid size"); + const auto EmitSize = Op->Header.Size == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; + + const auto Src = GetRegPair(Op->Pair.ID()); + const std::array Regs = {Src.first, Src.second}; + mov(EmitSize, GetReg(Node), Regs[Op->Element]); } DEF_OP(CreateElementPair) { auto Op = IROp->C(); - std::pair Dst; - aarch64::Register RegFirst; - aarch64::Register RegSecond; - aarch64::Register RegTmp; + LOGMAN_THROW_AA_FMT(IROp->ElementSize == 4 || IROp->ElementSize == 8, "Invalid size"); + std::pair Dst = GetRegPair(Node); + ARMEmitter::Register RegFirst = GetReg(Op->Lower.ID()); + ARMEmitter::Register RegSecond = GetReg(Op->Upper.ID()); + ARMEmitter::Register RegTmp = TMP1.R(); - switch (IROp->ElementSize) { - case 4: { - Dst = GetRegPair(Node); - RegFirst = GetReg(Op->Lower.ID()); - RegSecond = GetReg(Op->Upper.ID()); - RegTmp = w0; - break; - } - case 8: { - Dst = GetRegPair(Node); - RegFirst = GetReg(Op->Lower.ID()); - RegSecond = GetReg(Op->Upper.ID()); - RegTmp = x0; - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Size"); break; - } + const auto EmitSize = IROp->ElementSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; - if (Dst.first.GetCode() != RegSecond.GetCode()) { - mov(Dst.first, RegFirst); - mov(Dst.second, RegSecond); - } else if (Dst.second.GetCode() != RegFirst.GetCode()) { - mov(Dst.second, RegSecond); - mov(Dst.first, RegFirst); + if (Dst.first.Idx() != RegSecond.Idx()) { + mov(EmitSize, Dst.first, RegFirst); + mov(EmitSize, Dst.second, RegSecond); + } else if (Dst.second.Idx() != RegFirst.Idx()) { + mov(EmitSize, Dst.second, RegSecond); + mov(EmitSize, Dst.first, RegFirst); } else { - mov(RegTmp, RegFirst); - mov(Dst.second, RegSecond); - mov(Dst.first, RegTmp); + mov(EmitSize, RegTmp, RegFirst); + mov(EmitSize, Dst.second, RegSecond); + mov(EmitSize, Dst.first, RegTmp); } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 1e758346d..7d15eb220 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -4,29 +4,26 @@ tags: backend|arm64 $end_info$ */ +#include "Interface/Core/ArchHelpers/CodeEmitter/Emitter.h" +#include "Interface/Core/ArchHelpers/CodeEmitter/Registers.h" #include "Interface/Core/JIT/Arm64/JITClass.h" namespace FEXCore::CPU { - -using namespace vixl; -using namespace vixl::aarch64; #define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node) DEF_OP(VectorZero) { + const auto Dst = GetVReg(Node); if (HostSupportsSVE) { - const auto Dst = GetVReg(Node).Z().VnD(); - eor(Dst, Dst, Dst); + eor(Dst.Z(), Dst.Z(), Dst.Z()); } else { const uint8_t OpSize = IROp->Size; switch (OpSize) { case 8: { - const auto Dst = GetVReg(Node).V8B(); - eor(Dst, Dst, Dst); + eor(Dst.D(), Dst.D(), Dst.D()); break; } case 16: { - const auto Dst = GetVReg(Node).V16B(); - eor(Dst, Dst, Dst); + eor(Dst.Q(), Dst.Q(), Dst.Q()); break; } default: @@ -38,45 +35,34 @@ DEF_OP(VectorZero) { DEF_OP(VectorImm) { auto Op = IROp->C(); - - const uint8_t OpSize = IROp->Size; const uint8_t ElementSize = Op->Header.ElementSize; - const uint8_t Elements = OpSize / ElementSize; + + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + + const auto Dst = GetVReg(Node); if (HostSupportsSVE) { - const auto Dst = [&] { - const auto Tmp = GetVReg(Node).Z(); - switch (ElementSize) { - case 1: - return Tmp.VnB(); - case 2: - return Tmp.VnH(); - case 4: - return Tmp.VnS(); - case 8: - return Tmp.VnD(); - default: - LOGMAN_MSG_A_FMT("Unhandled element size: {}", ElementSize); - return Tmp; - } - }(); - if (ElementSize > 1 && (Op->Immediate & 0x80)) { // SVE dup uses sign extension where VectorImm wants zext - LoadConstant(TMP1.X(), Op->Immediate); - dup(Dst, TMP1.X()); + LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Immediate); + dup(SubRegSize, Dst.Z(), TMP1); } else { - dup(Dst, static_cast(Op->Immediate)); + dup_imm(SubRegSize, Dst.Z(), static_cast(Op->Immediate)); } } else { if (ElementSize == 8) { // movi with 64bit element size doesn't do what we want here - LoadConstant(TMP1.X(), Op->Immediate); - dup(GetVReg(Node).V2D(), TMP1.X()); + LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Immediate); + dup(SubRegSize, Dst.Q(), TMP1.R()); } else { - movi(GetVReg(Node).VCast(OpSize * 8, Elements), Op->Immediate); + movi(SubRegSize, Dst.Q(), Op->Immediate); } } } @@ -90,34 +76,30 @@ DEF_OP(VMov) { switch (OpSize) { case 1: { - eor(VTMP1.V16B(), VTMP1.V16B(), VTMP1.V16B()); - mov(VTMP1.V16B(), 0, Source.V16B(), 0); - mov(Dst, VTMP1); + eor(VTMP1.Q(), VTMP1.Q(), VTMP1.Q()); + ins(ARMEmitter::SubRegSize::i8Bit, VTMP1, 0, Source, 0); + mov(Dst.Q(), VTMP1.Q()); break; } case 2: { - eor(VTMP1.V16B(), VTMP1.V16B(), VTMP1.V16B()); - mov(VTMP1.V8H(), 0, Source.V8H(), 0); - mov(Dst, VTMP1); + eor(VTMP1.Q(), VTMP1.Q(), VTMP1.Q()); + ins(ARMEmitter::SubRegSize::i16Bit, VTMP1, 0, Source, 0); + mov(Dst.Q(), VTMP1.Q()); break; } case 4: { - eor(VTMP1.V16B(), VTMP1.V16B(), VTMP1.V16B()); - mov(VTMP1.V4S(), 0, Source.V4S(), 0); - mov(Dst, VTMP1); + eor(VTMP1.Q(), VTMP1.Q(), VTMP1.Q()); + ins(ARMEmitter::SubRegSize::i32Bit, VTMP1, 0, Source, 0); + mov(Dst.Q(), VTMP1.Q()); break; } case 8: { - mov(Dst.V8B(), Source.V8B()); + mov(Dst.D(), Source.D()); break; } case 16: { - if (HostSupportsSVE) { - mov(Dst.V16B(), Source.V16B()); - } else { - if (Dst.GetCode() != Source.GetCode()) { - mov(Dst.V16B(), Source.V16B()); - } + if (HostSupportsSVE || Dst.Idx() != Source.Idx()) { + mov(Dst.Q(), Source.Q()); } break; } @@ -126,8 +108,8 @@ DEF_OP(VMov) { // (*cough* AVX-512 *cough*) make sure to change this to treat // 256-bit moves with zero extending behavior instead of doing only // a regular SVE move into a 512-bit register. - if (Dst.GetCode() != Source.GetCode()) { - mov(Dst.Z().VnD(), Source.Z().VnD()); + if (Dst.Idx() != Source.Idx()) { + mov(Dst.Z(), Source.Z()); } break; } @@ -145,9 +127,9 @@ DEF_OP(VAnd) { const auto Vector2 = GetVReg(Op->Vector2.ID()); if (HostSupportsSVE) { - and_(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); + and_(Dst.Z(), Vector1.Z(), Vector2.Z()); } else { - and_(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); + and_(Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -159,9 +141,9 @@ DEF_OP(VBic) { const auto Vector2 = GetVReg(Op->Vector2.ID()); if (HostSupportsSVE) { - bic(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); + bic(Dst.Z(), Vector1.Z(), Vector2.Z()); } else { - bic(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); + bic(Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -173,9 +155,9 @@ DEF_OP(VOr) { const auto Vector2 = GetVReg(Op->Vector2.ID()); if (HostSupportsSVE) { - orr(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); + orr(Dst.Z(), Vector1.Z(), Vector2.Z()); } else { - orr(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); + orr(Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -187,9 +169,9 @@ DEF_OP(VXor) { const auto Vector2 = GetVReg(Op->Vector2.ID()); if (HostSupportsSVE) { - eor(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); + eor(Dst.Z(), Vector1.Z(), Vector2.Z()); } else { - eor(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); + eor(Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -202,42 +184,17 @@ DEF_OP(VAdd) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); - switch (ElementSize) { - case 1: { - if (HostSupportsSVE) { - add(Dst.Z().VnB(), Vector1.Z().VnB(), Vector2.Z().VnB()); - } else { - add(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - } - break; - } - case 2: { - if (HostSupportsSVE) { - add(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - } else { - add(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - } - break; - } - case 4: { - if (HostSupportsSVE) { - add(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - } else { - add(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - } - break; - } - case 8: { - if (HostSupportsSVE) { - add(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - } else { - add(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { + add(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); + } + else { + add(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -250,42 +207,17 @@ DEF_OP(VSub) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); - switch (ElementSize) { - case 1: { - if (HostSupportsSVE) { - sub(Dst.Z().VnB(), Vector1.Z().VnB(), Vector2.Z().VnB()); - } else { - sub(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - } - break; - } - case 2: { - if (HostSupportsSVE) { - sub(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - } else { - sub(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - } - break; - } - case 4: { - if (HostSupportsSVE) { - sub(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - } else { - sub(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - } - break; - } - case 8: { - if (HostSupportsSVE) { - sub(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - } else { - sub(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { + sub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); + } + else { + sub(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -298,42 +230,17 @@ DEF_OP(VUQAdd) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); - switch (ElementSize) { - case 1: { - if (HostSupportsSVE) { - uqadd(Dst.Z().VnB(), Vector1.Z().VnB(), Vector2.Z().VnB()); - } else { - uqadd(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - } - break; - } - case 2: { - if (HostSupportsSVE) { - uqadd(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - } else { - uqadd(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - } - break; - } - case 4: { - if (HostSupportsSVE) { - uqadd(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - } else { - uqadd(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - } - break; - } - case 8: { - if (HostSupportsSVE) { - uqadd(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - } else { - uqadd(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { + uqadd(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); + } + else { + uqadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -346,42 +253,17 @@ DEF_OP(VUQSub) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); - switch (ElementSize) { - case 1: { - if (HostSupportsSVE) { - uqsub(Dst.Z().VnB(), Vector1.Z().VnB(), Vector2.Z().VnB()); - } else { - uqsub(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - } - break; - } - case 2: { - if (HostSupportsSVE) { - uqsub(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - } else { - uqsub(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - } - break; - } - case 4: { - if (HostSupportsSVE) { - uqsub(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - } else { - uqsub(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - } - break; - } - case 8: { - if (HostSupportsSVE) { - uqsub(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - } else { - uqsub(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { + uqsub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); + } + else { + uqsub(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -394,42 +276,17 @@ DEF_OP(VSQAdd) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); - switch (ElementSize) { - case 1: { - if (HostSupportsSVE) { - sqadd(Dst.Z().VnB(), Vector1.Z().VnB(), Vector2.Z().VnB()); - } else { - sqadd(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - } - break; - } - case 2: { - if (HostSupportsSVE) { - sqadd(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - } else { - sqadd(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - } - break; - } - case 4: { - if (HostSupportsSVE) { - sqadd(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - } else { - sqadd(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - } - break; - } - case 8: { - if (HostSupportsSVE) { - sqadd(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - } else { - sqadd(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { + sqadd(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); + } + else { + sqadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -442,42 +299,17 @@ DEF_OP(VSQSub) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); - switch (ElementSize) { - case 1: { - if (HostSupportsSVE) { - sqsub(Dst.Z().VnB(), Vector1.Z().VnB(), Vector2.Z().VnB()); - } else { - sqsub(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - } - break; - } - case 2: { - if (HostSupportsSVE) { - sqsub(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - } else { - sqsub(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - } - break; - } - case 4: { - if (HostSupportsSVE) { - sqsub(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - } else { - sqsub(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - } - break; - } - case 8: { - if (HostSupportsSVE) { - sqsub(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - } else { - sqsub(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { + sqsub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); + } + else { + sqsub(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -493,90 +325,35 @@ DEF_OP(VAddP) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Pred = PRED_TMP_32B.Merging(); // SVE ADDP is a destructive operation, so we need a temporary - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - movprfx(VTMP1.Z().VnD(), VectorLower.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + movprfx(VTMP1.Z(), VectorLower.Z()); // Unlike Adv. SIMD's version of ADDP, which acts like it concats the // upper vector onto the end of the lower vector and then performs // pairwise addition, the SVE version actually interleaves the // results of the pairwise addition (gross!), so we need to undo that. - switch (ElementSize) { - case 1: { - addp(VTMP1.Z().VnB(), Pred, VTMP1.Z().VnB(), VectorUpper.Z().VnB()); - uzp1(Dst.Z().VnB(), VTMP1.Z().VnB(), VTMP1.Z().VnB()); - uzp2(VTMP2.Z().VnB(), VTMP1.Z().VnB(), VTMP1.Z().VnB()); - break; - } - case 2: { - addp(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), VectorUpper.Z().VnH()); - uzp1(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP1.Z().VnH()); - uzp2(VTMP2.Z().VnH(), VTMP1.Z().VnH(), VTMP1.Z().VnH()); - break; - } - case 4: { - addp(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), VectorUpper.Z().VnS()); - uzp1(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP1.Z().VnS()); - uzp2(VTMP2.Z().VnS(), VTMP1.Z().VnS(), VTMP1.Z().VnS()); - break; - } - case 8: { - addp(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), VectorUpper.Z().VnD()); - uzp1(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - uzp2(VTMP2.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + addp(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), VectorUpper.Z()); + uzp1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP1.Z()); + uzp2(SubRegSize, VTMP2.Z(), VTMP1.Z(), VTMP1.Z()); // Merge upper half with lower half. - splice(Dst.Z().VnD(), PRED_TMP_16B, Dst.Z().VnD(), VTMP2.Z().VnD()); + splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP2.Z()); } else { if (IsScalar) { - switch (ElementSize) { - case 1: { - addp(Dst.V8B(), VectorLower.V8B(), VectorUpper.V8B()); - break; - } - case 2: { - addp(Dst.V4H(), VectorLower.V4H(), VectorUpper.V4H()); - break; - } - case 4: { - addp(Dst.V2S(), VectorLower.V2S(), VectorUpper.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + addp(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D()); } else { - switch (ElementSize) { - case 1: { - addp(Dst.V16B(), VectorLower.V16B(), VectorUpper.V16B()); - break; - } - case 2: { - addp(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); - break; - } - case 4: { - addp(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); - break; - } - case 8: { - addp(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + addp(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q()); } } } @@ -586,65 +363,39 @@ DEF_OP(VAddV) { const auto OpSize = IROp->Size; const auto ElementSize = Op->Header.ElementSize; - const auto Elements = OpSize / ElementSize; const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit) { // SVE doesn't have an equivalent ADDV instruction, so we make do // by performing two Adv. SIMD ADDV operations on the high and low // 128-bit lanes and then sum them up. const auto Mask = PRED_TMP_32B.Zeroing(); - const auto CompactPred = p0; + const auto CompactPred = ARMEmitter::PReg::p0; // Select all our upper elements to run ADDV over them. - not_(CompactPred.VnB(), Mask, PRED_TMP_16B.VnB()); - compact(VTMP1.Z().VnD(), CompactPred, Vector.Z().VnD()); + not_(CompactPred, Mask, PRED_TMP_16B); + compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, Vector.Z()); - switch (ElementSize) { - case 1: - addv(VTMP2.B(), Vector.V16B()); - addv(VTMP1.B(), VTMP1.V16B()); - add(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); - break; - case 2: - addv(VTMP2.H(), Vector.V8H()); - addv(VTMP1.H(), VTMP1.V8H()); - add(Dst.V8H(), VTMP1.V8H(), VTMP2.V8H()); - break; - case 4: - addv(VTMP2.S(), Vector.V4S()); - addv(VTMP1.S(), VTMP1.V4S()); - add(Dst.V4S(), VTMP1.V4S(), VTMP2.V4S()); - break; - case 8: - addp(VTMP2.D(), Vector.V2D()); - addp(VTMP1.D(), VTMP1.V2D()); - add(Dst.V2D(), VTMP1.V2D(), VTMP2.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + addv(SubRegSize.Vector, VTMP2.Q(), Vector.Q()); + addv(SubRegSize.Vector, VTMP1.Q(), VTMP1.Q()); + add(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), VTMP2.Q()); } else { - const auto OpSizeBits = OpSize * 8; - const auto ElementSizeBits = ElementSize * 8; - - switch (ElementSize) { - case 1: - case 2: - case 4: - addv(Dst.VCast(ElementSizeBits, 1), Vector.VCast(OpSizeBits, Elements)); - break; - case 8: - addp(Dst.VCast(OpSizeBits, 1), Vector.VCast(OpSizeBits, Elements)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; + if (ElementSize == 8) { + addp(SubRegSize.Scalar, Dst, Vector); + } + else { + addv(SubRegSize.Vector, Dst.Q(), Vector.Q()); } } } @@ -654,11 +405,17 @@ DEF_OP(VUMinV) { const auto OpSize = IROp->Size; const auto ElementSize = Op->Header.ElementSize; - const auto Elements = OpSize / ElementSize; const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { LOGMAN_THROW_AA_FMT(OpSize == 16 || OpSize == 32, "Unsupported vector length: {}", OpSize); @@ -666,35 +423,10 @@ DEF_OP(VUMinV) { const auto Pred = OpSize == 16 ? PRED_TMP_16B : PRED_TMP_32B; - switch (ElementSize) { - case 1: - uminv(Dst.B(), Pred, Vector.Z().VnB()); - break; - case 2: - uminv(Dst.H(), Pred, Vector.Z().VnH()); - break; - case 4: - uminv(Dst.S(), Pred, Vector.Z().VnS()); - break; - case 8: - uminv(Dst.D(), Pred, Vector.Z().VnD()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + uminv(SubRegSize, Dst.Z(), Pred, Vector.Z()); } else { // Vector - switch (ElementSize) { - case 1: - case 2: - case 4: - uminv(Dst.VCast(ElementSize * 8, 1), Vector.VCast(OpSize * 8, Elements)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uminv(SubRegSize, Dst.Q(), Vector.Q()); } } @@ -709,58 +441,23 @@ DEF_OP(VURAvg) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); // SVE URHADD is a destructive operation, so we need // a temporary for performing operations. - movprfx(VTMP1.Z().VnD(), Vector1.Z().VnD()); - - switch (ElementSize) { - case 1: { - urhadd(VTMP1.Z().VnB(), Mask, - VTMP1.Z().VnB(), Vector2.Z().VnB()); - break; - } - case 2: { - urhadd(VTMP1.Z().VnH(), Mask, - VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - urhadd(VTMP1.Z().VnS(), Mask, - VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - urhadd(VTMP1.Z().VnD(), Mask, - VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + movprfx(VTMP1.Z(), Vector1.Z()); + urhadd(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { - switch (ElementSize) { - case 1: { - urhadd(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - urhadd(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - urhadd(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + urhadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -769,58 +466,26 @@ DEF_OP(VAbs) { const auto OpSize = IROp->Size; const uint8_t ElementSize = Op->Header.ElementSize; - const uint8_t Elements = OpSize / Op->Header.ElementSize; const auto Dst = GetVReg(Node); const auto Src = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && OpSize == 32) { - switch (ElementSize) { - case 1: { - abs(Dst.Z().VnB(), PRED_TMP_32B.Merging(), Src.Z().VnB()); - break; - } - case 2: { - abs(Dst.Z().VnH(), PRED_TMP_32B.Merging(), Src.Z().VnH()); - break; - } - case 4: { - abs(Dst.Z().VnS(), PRED_TMP_32B.Merging(), Src.Z().VnS()); - break; - } - case 8: { - abs(Dst.Z().VnD(), PRED_TMP_32B.Merging(), Src.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + abs(SubRegSize, Dst.Z(), PRED_TMP_32B.Merging(), Src.Z()); } else { if (ElementSize == OpSize) { // Scalar - switch (ElementSize) { - case 8: { - abs(Dst.D(), Src.D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + abs(SubRegSize, Dst.D(), Src.D()); } else { // Vector - switch (ElementSize) { - case 1: - case 2: - case 4: - case 8: - abs(Dst.VCast(OpSize * 8, Elements), Src.VCast(OpSize * 8, Elements)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + abs(SubRegSize, Dst.Q(), Src.Q()); } } } @@ -835,49 +500,25 @@ DEF_OP(VPopcount) { const auto Dst = GetVReg(Node); const auto Src = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && !IsScalar) { const auto Pred = OpSize == 16 ? PRED_TMP_16B.Merging() : PRED_TMP_32B.Merging(); - switch (ElementSize) { - case 1: - cnt(Dst.Z().VnB(), Pred, Src.Z().VnB()); - break; - case 2: - cnt(Dst.Z().VnH(), Pred, Src.Z().VnH()); - break; - case 4: - cnt(Dst.Z().VnS(), Pred, Src.Z().VnS()); - break; - case 8: - cnt(Dst.Z().VnD(), Pred, Src.Z().VnD()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cnt(SubRegSize, Dst.Z(), Pred, Src.Z()); } else { if (IsScalar) { // Scalar - switch (ElementSize) { - case 1: { - cnt(Dst.V8B(), Src.V8B()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cnt(SubRegSize, Dst.D(), Src.D()); } else { - // Vector - switch (ElementSize) { - case 1: - cnt(Dst.V16B(), Src.V16B()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + // Scalar + cnt(SubRegSize, Dst.Q(), Src.Q()); } } } @@ -893,24 +534,14 @@ DEF_OP(VFAdd) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && !IsScalar) { - switch (ElementSize) { - case 2: { - fadd(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - fadd(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - fadd(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fadd(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -931,23 +562,7 @@ DEF_OP(VFAdd) { break; } } else { - switch (ElementSize) { - case 2: { - fadd(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - fadd(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - fadd(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -964,61 +579,32 @@ DEF_OP(VFAddP) { const bool Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + + if (HostSupportsSVE && Is256Bit) { const auto Pred = PRED_TMP_32B.Merging(); // SVE FADDP is a destructive operation, so we need a temporary - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - movprfx(VTMP1.Z().VnD(), VectorLower.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + movprfx(VTMP1.Z(), VectorLower.Z()); // Unlike Adv. SIMD's version of FADDP, which acts like it concats the // upper vector onto the end of the lower vector and then performs // pairwise addition, the SVE version actually interleaves the // results of the pairwise addition (gross!), so we need to undo that. - switch (ElementSize) { - case 2: { - faddp(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), VectorUpper.Z().VnH()); - uzp1(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP1.Z().VnH()); - uzp2(VTMP2.Z().VnH(), VTMP1.Z().VnH(), VTMP1.Z().VnH()); - break; - } - case 4: { - faddp(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), VectorUpper.Z().VnS()); - uzp1(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP1.Z().VnS()); - uzp2(VTMP2.Z().VnS(), VTMP1.Z().VnS(), VTMP1.Z().VnS()); - break; - } - case 8: { - faddp(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), VectorUpper.Z().VnD()); - uzp1(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - uzp2(VTMP2.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + faddp(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), VectorUpper.Z()); + uzp1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP1.Z()); + uzp2(SubRegSize, VTMP2.Z(), VTMP1.Z(), VTMP1.Z()); // Merge upper half with lower half. - splice(Dst.Z().VnD(), PRED_TMP_16B, Dst.Z().VnD(), VTMP2.Z().VnD()); + splice(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP2.Z()); } else { - switch (ElementSize) { - case 2: { - faddp(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); - break; - } - case 4: { - faddp(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); - break; - } - case 8: { - faddp(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + faddp(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q()); } } @@ -1033,24 +619,14 @@ DEF_OP(VFSub) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && !IsScalar) { - switch (ElementSize) { - case 2: { - fsub(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - fsub(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - fsub(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fsub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -1071,23 +647,7 @@ DEF_OP(VFSub) { break; } } else { - switch (ElementSize) { - case 2: { - fsub(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - fsub(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - fsub(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fsub(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -1103,24 +663,14 @@ DEF_OP(VFMul) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && !IsScalar) { - switch (ElementSize) { - case 2: { - fmul(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - fmul(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - fmul(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fmul(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -1141,23 +691,7 @@ DEF_OP(VFMul) { break; } } else { - switch (ElementSize) { - case 2: { - fmul(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - fmul(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - fmul(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fmul(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -1174,34 +708,19 @@ DEF_OP(VFDiv) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Merging(); // SVE VDIV is a destructive operation, so we need a temporary. - movprfx(VTMP1.Z().VnD(), Vector1.Z().VnD()); - - switch (ElementSize) { - case 2: { - fdiv(VTMP1.Z().VnH(), Mask, - VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - fdiv(VTMP1.Z().VnS(), Mask, - VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - fdiv(VTMP1.Z().VnD(), Mask, - VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + movprfx(VTMP1.Z(), Vector1.Z()); + fdiv(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -1222,23 +741,7 @@ DEF_OP(VFDiv) { break; } } else { - switch (ElementSize) { - case 2: { - fdiv(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - fdiv(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - fdiv(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fdiv(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -1255,6 +758,12 @@ DEF_OP(VFMin) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + // NOTE: We don't directly use FMIN here for any of the implementations, // because it has undesirable NaN handling behavior (it sets // entries either to the incoming NaN value*, or the default NaN @@ -1266,7 +775,7 @@ DEF_OP(VFMin) { if (HostSupportsSVE && !IsScalar) { const auto Mask = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B; - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // General idea: // 1. Compare greater than against the two vectors @@ -1276,54 +785,28 @@ DEF_OP(VFMin) { // predicate bits from the second vector into the // same temporary. // 5. Move temporary into the destination register and we're done. - - switch (ElementSize) { - case 2: { - fcmgt(ComparePred.VnH(), Mask.Zeroing(), - Vector2.Z().VnH(), Vector1.Z().VnH()); - not_(ComparePred.VnB(), Mask.Zeroing(), ComparePred.VnB()); - mov(VTMP1.Z().VnD(), Vector1.Z().VnD()); - mov(VTMP1.Z().VnH(), ComparePred.Merging(), Vector2.Z().VnH()); - break; - } - case 4: { - fcmgt(ComparePred.VnS(), Mask.Zeroing(), - Vector2.Z().VnS(), Vector1.Z().VnS()); - not_(ComparePred.VnB(), Mask.Zeroing(), ComparePred.VnB()); - mov(VTMP1.Z().VnD(), Vector1.Z().VnD()); - mov(VTMP1.Z().VnS(), ComparePred.Merging(), Vector2.Z().VnS()); - break; - } - case 8: { - fcmgt(ComparePred.VnD(), Mask.Zeroing(), - Vector2.Z().VnD(), Vector1.Z().VnD()); - not_(ComparePred.VnB(), Mask.Zeroing(), ComparePred.VnB()); - mov(VTMP1.Z().VnD(), Vector1.Z().VnD()); - mov(VTMP1.Z().VnD(), ComparePred.Merging(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + fcmgt(SubRegSize, ComparePred, Mask.Zeroing(), + Vector2.Z(), Vector1.Z()); + not_(ComparePred, Mask.Zeroing(), ComparePred); + mov(VTMP1.Z(), Vector1.Z()); + mov(SubRegSize, VTMP1.Z(), ComparePred.Merging(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { case 2: { fcmp(Vector1.H(), Vector2.H()); - fcsel(Dst.H(), Vector1.H(), Vector2.H(), Condition::mi); + fcsel(Dst.H(), Vector1.H(), Vector2.H(), ARMEmitter::Condition::CC_MI); break; } case 4: { fcmp(Vector1.S(), Vector2.S()); - fcsel(Dst.S(), Vector1.S(), Vector2.S(), Condition::mi); + fcsel(Dst.S(), Vector1.S(), Vector2.S(), ARMEmitter::Condition::CC_MI); break; } case 8: { fcmp(Vector1.D(), Vector2.D()); - fcsel(Dst.D(), Vector1.D(), Vector2.D(), Condition::mi); + fcsel(Dst.D(), Vector1.D(), Vector2.D(), ARMEmitter::Condition::CC_MI); break; } default: @@ -1331,32 +814,10 @@ DEF_OP(VFMin) { break; } } else { - switch (ElementSize) { - case 2: { - fcmgt(VTMP1.V8H(), Vector2.V8H(), Vector1.V8H()); - mov(VTMP2.V8H(), Vector1.V8H()); - bif(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V8H(), VTMP2.V8H()); - break; - } - case 4: { - fcmgt(VTMP1.V4S(), Vector2.V4S(), Vector1.V4S()); - mov(VTMP2.V4S(), Vector1.V4S()); - bif(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V4S(), VTMP2.V4S()); - break; - } - case 8: { - fcmgt(VTMP1.V2D(), Vector2.V2D(), Vector1.V2D()); - mov(VTMP2.V2D(), Vector1.V2D()); - bif(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V2D(), VTMP2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q()); + mov(VTMP2.Q(), Vector1.Q()); + bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q()); + mov(Dst.Q(), VTMP2.Q()); } } } @@ -1373,58 +834,40 @@ DEF_OP(VFMax) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + // NOTE: See VFMin implementation for reasons why we // don't just use FMAX/FMIN for these implementations. if (HostSupportsSVE && !IsScalar) { const auto Mask = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B; - const auto ComparePred = p0; - - switch (ElementSize) { - case 2: { - fcmgt(ComparePred.VnH(), Mask.Zeroing(), - Vector2.Z().VnH(), Vector1.Z().VnH()); - mov(VTMP1.Z().VnD(), Vector1.Z().VnD()); - mov(VTMP1.Z().VnH(), ComparePred.Merging(), Vector2.Z().VnH()); - break; - } - case 4: { - fcmgt(ComparePred.VnS(), Mask.Zeroing(), - Vector2.Z().VnS(), Vector1.Z().VnS()); - mov(VTMP1.Z().VnD(), Vector1.Z().VnD()); - mov(VTMP1.Z().VnS(), ComparePred.Merging(), Vector2.Z().VnS()); - break; - } - case 8: { - fcmgt(ComparePred.VnD(), Mask.Zeroing(), - Vector2.Z().VnD(), Vector1.Z().VnD()); - mov(VTMP1.Z().VnD(), Vector1.Z().VnD()); - mov(VTMP1.Z().VnD(), ComparePred.Merging(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + const auto ComparePred = ARMEmitter::PReg::p0; + fcmgt(SubRegSize, ComparePred, Mask.Zeroing(), + Vector2.Z(), Vector1.Z()); + mov(VTMP1.Z(), Vector1.Z()); + mov(SubRegSize, VTMP1.Z(), ComparePred.Merging(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { case 2: { fcmp(Vector1.H(), Vector2.H()); - fcsel(Dst.H(), Vector2.H(), Vector1.H(), Condition::mi); + fcsel(Dst.H(), Vector2.H(), Vector1.H(), ARMEmitter::Condition::CC_MI); break; } case 4: { fcmp(Vector1.S(), Vector2.S()); - fcsel(Dst.S(), Vector2.S(), Vector1.S(), Condition::mi); + fcsel(Dst.S(), Vector2.S(), Vector1.S(), ARMEmitter::Condition::CC_MI); break; } case 8: { fcmp(Vector1.D(), Vector2.D()); - fcsel(Dst.D(), Vector2.D(), Vector1.D(), Condition::mi); + fcsel(Dst.D(), Vector2.D(), Vector1.D(), ARMEmitter::Condition::CC_MI); break; } default: @@ -1432,32 +875,10 @@ DEF_OP(VFMax) { break; } } else { - switch (ElementSize) { - case 2: { - fcmgt(VTMP1.V8H(), Vector2.V8H(), Vector1.V8H()); - mov(VTMP2.V8H(), Vector1.V8H()); - bit(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V8H(), VTMP2.V8H()); - break; - } - case 4: { - fcmgt(VTMP1.V4S(), Vector2.V4S(), Vector1.V4S()); - mov(VTMP2.V4S(), Vector1.V4S()); - bit(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V4S(), VTMP2.V4S()); - break; - } - case 8: { - fcmgt(VTMP1.V2D(), Vector2.V2D(), Vector1.V2D()); - mov(VTMP2.V2D(), Vector1.V2D()); - bit(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V2D(), VTMP2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q()); + mov(VTMP2.Q(), Vector1.Q()); + bit(VTMP2.Q(), Vector2.Q(), VTMP1.Q()); + mov(Dst.Q(), VTMP2.Q()); } } } @@ -1473,47 +894,32 @@ DEF_OP(VFRecp) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit); + if (HostSupportsSVE && !IsScalar) { const auto Pred = Is256Bit ? PRED_TMP_32B.Merging() : PRED_TMP_16B.Merging(); - switch (ElementSize) { - case 2: { - fmov(VTMP1.Z().VnH(), 1.0); - fdiv(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), Vector.Z().VnH()); - break; - } - case 4: { - fmov(VTMP1.Z().VnS(), 1.0); - fdiv(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), Vector.Z().VnS()); - break; - } - case 8: { - fmov(VTMP1.Z().VnD(), 1.0); - fdiv(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), Vector.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + fmov(SubRegSize.Vector, VTMP1.Z(), 1.0); + fdiv(SubRegSize.Vector, VTMP1.Z(), Pred, VTMP1.Z(), Vector.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { + fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0f); switch (ElementSize) { case 2: { - fmov(VTMP1.H(), Float16{1.0}); fdiv(Dst.H(), VTMP1.H(), Vector.H()); break; } case 4: { - fmov(VTMP1.S(), 1.0f); fdiv(Dst.S(), VTMP1.S(), Vector.S()); break; } case 8: { - fmov(VTMP1.D(), 1.0); fdiv(Dst.D(), VTMP1.D(), Vector.D()); break; } @@ -1522,26 +928,8 @@ DEF_OP(VFRecp) { break; } } else { - switch (ElementSize) { - case 2: { - fmov(VTMP1.V8H(), Float16{1.0}); - fdiv(Dst.V8H(), VTMP1.V8H(), Vector.V8H()); - break; - } - case 4: { - fmov(VTMP1.V4S(), 1.0f); - fdiv(Dst.V4S(), VTMP1.V4S(), Vector.V4S()); - break; - } - case 8: { - fmov(VTMP1.V2D(), 1.0); - fdiv(Dst.V2D(), VTMP1.V2D(), Vector.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fmov(SubRegSize.Vector, VTMP1.Q(), 1.0f); + fdiv(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), Vector.Q()); } } } @@ -1557,27 +945,17 @@ DEF_OP(VFSqrt) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && !IsScalar) { const auto Pred = Is256Bit ? PRED_TMP_32B.Merging() : PRED_TMP_16B.Merging(); - switch (ElementSize) { - case 2: { - fsqrt(Dst.Z().VnH(), Pred, Vector.Z().VnH()); - break; - } - case 4: { - fsqrt(Dst.Z().VnS(), Pred, Vector.Z().VnS()); - break; - } - case 8: { - fsqrt(Dst.Z().VnD(), Pred, Vector.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fsqrt(SubRegSize, Dst.Z(), Pred, Vector.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -1598,23 +976,7 @@ DEF_OP(VFSqrt) { break; } } else { - switch (ElementSize) { - case 2: { - fsqrt(Dst.V8H(), Vector.V8H()); - break; - } - case 4: { - fsqrt(Dst.V4S(), Vector.V4S()); - break; - } - case 8: { - fsqrt(Dst.V2D(), Vector.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fsqrt(SubRegSize, Dst.Q(), Vector.Q()); } } } @@ -1630,51 +992,33 @@ DEF_OP(VFRSqrt) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Pred = PRED_TMP_32B.Merging(); - - switch (ElementSize) { - case 2: { - fmov(VTMP1.Z().VnH(), 1.0); - fsqrt(VTMP2.Z().VnH(), Pred, Vector.Z().VnH()); - fdiv(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - } - case 4: { - fmov(VTMP1.Z().VnS(), 1.0); - fsqrt(VTMP2.Z().VnS(), Pred, Vector.Z().VnS()); - fdiv(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - } - case 8: { - fmov(VTMP1.Z().VnD(), 1.0); - fsqrt(VTMP2.Z().VnD(), Pred, Vector.Z().VnD()); - fdiv(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), VTMP2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + fmov(SubRegSize.Vector, VTMP1.Z(), 1.0); + fsqrt(SubRegSize.Vector, VTMP2.Z(), Pred, Vector.Z()); + fdiv(SubRegSize.Vector, VTMP1.Z(), Pred, VTMP1.Z(), VTMP2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { + fmov(SubRegSize.Scalar, VTMP1.Q(), 1.0); switch (ElementSize) { case 2: { - fmov(VTMP1.H(), Float16{1.0}); fsqrt(VTMP2.H(), Vector.H()); fdiv(Dst.H(), VTMP1.H(), VTMP2.H()); break; } case 4: { - fmov(VTMP1.S(), 1.0f); fsqrt(VTMP2.S(), Vector.S()); fdiv(Dst.S(), VTMP1.S(), VTMP2.S()); break; } case 8: { - fmov(VTMP1.D(), 1.0); fsqrt(VTMP2.D(), Vector.D()); fdiv(Dst.D(), VTMP1.D(), VTMP2.D()); break; @@ -1684,29 +1028,9 @@ DEF_OP(VFRSqrt) { break; } } else { - switch (ElementSize) { - case 2: { - fmov(VTMP1.V8H(), Float16{1.0}); - fsqrt(VTMP2.V8H(), Vector.V8H()); - fdiv(Dst.V8H(), VTMP1.V8H(), VTMP2.V8H()); - break; - } - case 4: { - fmov(VTMP1.V4S(), 1.0f); - fsqrt(VTMP2.V4S(), Vector.V4S()); - fdiv(Dst.V4S(), VTMP1.V4S(), VTMP2.V4S()); - break; - } - case 8: { - fmov(VTMP1.V2D(), 1.0); - fsqrt(VTMP2.V2D(), Vector.V2D()); - fdiv(Dst.V2D(), VTMP1.V2D(), VTMP2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fmov(SubRegSize.Vector, VTMP1.Q(), 1.0); + fsqrt(SubRegSize.Vector, VTMP2.Q(), Vector.Q()); + fdiv(SubRegSize.Vector, Dst.Q(), VTMP1.Q(), VTMP2.Q()); } } } @@ -1719,47 +1043,22 @@ DEF_OP(VNeg) { const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; const auto Dst = GetVReg(Node); - const auto Vector = GetVReg(Op->Vector.ID()); + const auto Vector= GetVReg(Op->Vector.ID()); + + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; if (HostSupportsSVE) { const auto Pred = Is256Bit ? PRED_TMP_32B.Merging() : PRED_TMP_16B.Merging(); - switch (ElementSize) { - case 1: - neg(Dst.Z().VnB(), Pred, Vector.Z().VnB()); - break; - case 2: - neg(Dst.Z().VnH(), Pred, Vector.Z().VnH()); - break; - case 4: - neg(Dst.Z().VnS(), Pred, Vector.Z().VnS()); - break; - case 8: - neg(Dst.Z().VnD(), Pred, Vector.Z().VnD()); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported VNeg size: {}", ElementSize); - break; - } + neg(SubRegSize, Dst.Z(), Pred, Vector.Z()); } else { - switch (ElementSize) { - case 1: - neg(Dst.V16B(), Vector.V16B()); - break; - case 2: - neg(Dst.V8H(), Vector.V8H()); - break; - case 4: - neg(Dst.V4S(), Vector.V4S()); - break; - case 8: - neg(Dst.V2D(), Vector.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported VNeg size: {}", ElementSize); - break; - } + neg(SubRegSize, Dst.Q(), Vector.Q()); } } @@ -1773,39 +1072,19 @@ DEF_OP(VFNeg) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { const auto Pred = Is256Bit ? PRED_TMP_32B.Merging() : PRED_TMP_16B.Merging(); - switch (ElementSize) { - case 2: - fneg(Dst.Z().VnH(), Pred, Vector.Z().VnH()); - break; - case 4: - fneg(Dst.Z().VnS(), Pred, Vector.Z().VnS()); - break; - case 8: - fneg(Dst.Z().VnD(), Pred, Vector.Z().VnD()); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported VFNeg element size: {}", ElementSize); - break; - } + fneg(SubRegSize, Dst.Z(), Pred, Vector.Z()); } else { - switch (ElementSize) { - case 2: - fneg(Dst.V8H(), Vector.V8H()); - break; - case 4: - fneg(Dst.V4S(), Vector.V4S()); - break; - case 8: - fneg(Dst.V2D(), Vector.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported VFNeg element size: {}", ElementSize); - break; - } + fneg(SubRegSize, Dst.Q(), Vector.Q()); } } @@ -1818,9 +1097,9 @@ DEF_OP(VNot) { const auto Vector = GetVReg(Op->Vector.ID()); if (HostSupportsSVE && Is256Bit) { - not_(Dst.Z().VnB(), PRED_TMP_32B.Merging(), Vector.Z().VnB()); + not_(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), PRED_TMP_32B, Vector.Z()); } else { - mvn(Dst.V16B(), Vector.V16B()); + mvn(ARMEmitter::SubRegSize::i8Bit, Dst.Q(), Vector.Q()); } } @@ -1836,54 +1115,33 @@ DEF_OP(VUMin) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Pred = PRED_TMP_32B.Merging(); // SVE UMIN is a destructive operation so we need a temporary. - movprfx(VTMP1.Z().VnD(), Vector1.Z().VnD()); - - switch (ElementSize) { - case 1: { - umin(VTMP1.Z().VnB(), Pred, VTMP1.Z().VnB(), Vector2.Z().VnB()); - break; - } - case 2: { - umin(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - umin(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - umin(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + movprfx(VTMP1.Z(), Vector1.Z()); + umin(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { switch (ElementSize) { - case 1: { - umin(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - umin(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } + case 1: + case 2: case 4: { - umin(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); + umin(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); break; } case 8: { - cmhi(VTMP1.V2D(), Vector2.V2D(), Vector1.V2D()); - mov(VTMP2.V2D(), Vector1.V2D()); - bif(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V2D(), VTMP2.V2D()); + cmhi(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q()); + mov(VTMP2.Q(), Vector1.Q()); + bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q()); + mov(Dst.Q(), VTMP2.Q()); break; } default: @@ -1905,54 +1163,33 @@ DEF_OP(VSMin) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Pred = PRED_TMP_32B.Merging(); // SVE SMIN is a destructive operation, so we need a temporary. - movprfx(VTMP1.Z().VnD(), Vector1.Z().VnD()); - - switch (ElementSize) { - case 1: { - smin(VTMP1.Z().VnB(), Pred, VTMP1.Z().VnB(), Vector2.Z().VnB()); - break; - } - case 2: { - smin(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - smin(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - smin(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + movprfx(VTMP1.Z(), Vector1.Z()); + smin(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { switch (ElementSize) { - case 1: { - smin(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - smin(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } + case 1: + case 2: case 4: { - smin(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); + smin(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); break; } case 8: { - cmgt(VTMP1.V2D(), Vector2.V2D(), Vector1.V2D()); - mov(VTMP2.V2D(), Vector1.V2D()); - bif(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V2D(), VTMP2.V2D()); + cmgt(SubRegSize, VTMP1.Q(), Vector1.Q(), Vector2.Q()); + mov(VTMP2.Q(), Vector1.Q()); + bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q()); + mov(Dst.Q(), VTMP2.Q()); break; } default: @@ -1974,54 +1211,33 @@ DEF_OP(VUMax) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Pred = PRED_TMP_32B.Merging(); // SVE UMAX is a destructive operation, so we need a temporary. - movprfx(VTMP1.Z().VnD(), Vector1.Z().VnD()); - - switch (ElementSize) { - case 1: { - umax(VTMP1.Z().VnB(), Pred, VTMP1.Z().VnB(), Vector2.Z().VnB()); - break; - } - case 2: { - umax(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - umax(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - umax(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + movprfx(VTMP1.Z(), Vector1.Z()); + umax(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { switch (ElementSize) { - case 1: { - umax(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - umax(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } + case 1: + case 2: case 4: { - umax(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); + umax(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); break; } case 8: { - cmhi(VTMP1.V2D(), Vector2.V2D(), Vector1.V2D()); - mov(VTMP2.V2D(), Vector1.V2D()); - bit(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V2D(), VTMP2.V2D()); + cmhi(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q()); + mov(VTMP2.Q(), Vector1.Q()); + bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q()); + mov(Dst.Q(), VTMP2.Q()); break; } default: @@ -2043,54 +1259,33 @@ DEF_OP(VSMax) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Pred = PRED_TMP_32B.Merging(); // SVE SMAX is a destructive operation, so we need a temporary. - movprfx(VTMP1.Z().VnD(), Vector1.Z().VnD()); - - switch (ElementSize) { - case 1: { - smax(VTMP1.Z().VnB(), Pred, VTMP1.Z().VnB(), Vector2.Z().VnB()); - break; - } - case 2: { - smax(VTMP1.Z().VnH(), Pred, VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - smax(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - smax(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + movprfx(VTMP1.Z(), Vector1.Z()); + smax(SubRegSize, VTMP1.Z(), Pred, VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { switch (ElementSize) { - case 1: { - smax(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - smax(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } + case 1: + case 2: case 4: { - smax(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); + smax(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); break; } case 8: { - cmgt(VTMP1.V2D(), Vector2.V2D(), Vector1.V2D()); - mov(VTMP2.V2D(), Vector1.V2D()); - bit(VTMP2.V16B(), Vector2.V16B(), VTMP1.V16B()); - mov(Dst.V2D(), VTMP2.V2D()); + cmgt(SubRegSize, VTMP1.Q(), Vector2.Q(), Vector1.Q()); + mov(VTMP2.Q(), Vector1.Q()); + bif(VTMP2.Q(), Vector2.Q(), VTMP1.Q()); + mov(Dst.Q(), VTMP2.Q()); break; } default: @@ -2111,69 +1306,20 @@ DEF_OP(VZip) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 1: { - zip1(Dst.Z().VnB(), VectorLower.Z().VnB(), VectorUpper.Z().VnB()); - break; - } - case 2: { - zip1(Dst.Z().VnH(), VectorLower.Z().VnH(), VectorUpper.Z().VnH()); - break; - } - case 4: { - zip1(Dst.Z().VnS(), VectorLower.Z().VnS(), VectorUpper.Z().VnS()); - break; - } - case 8: { - zip1(Dst.Z().VnD(), VectorLower.Z().VnD(), VectorUpper.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + zip1(SubRegSize, Dst.Z(), VectorLower.Z(), VectorUpper.Z()); } else { if (OpSize == 8) { - switch (ElementSize) { - case 1: { - zip1(Dst.V8B(), VectorLower.V8B(), VectorUpper.V8B()); - break; - } - case 2: { - zip1(Dst.V4H(), VectorLower.V4H(), VectorUpper.V4H()); - break; - } - case 4: { - zip1(Dst.V2S(), VectorLower.V2S(), VectorUpper.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + zip1(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D()); } else { - switch (ElementSize) { - case 1: { - zip1(Dst.V16B(), VectorLower.V16B(), VectorUpper.V16B()); - break; - } - case 2: { - zip1(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); - break; - } - case 4: { - zip1(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); - break; - } - case 8: { - zip1(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + zip1(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q()); } } } @@ -2189,69 +1335,20 @@ DEF_OP(VZip2) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 1: { - zip2(Dst.Z().VnB(), VectorLower.Z().VnB(), VectorUpper.Z().VnB()); - break; - } - case 2: { - zip2(Dst.Z().VnH(), VectorLower.Z().VnH(), VectorUpper.Z().VnH()); - break; - } - case 4: { - zip2(Dst.Z().VnS(), VectorLower.Z().VnS(), VectorUpper.Z().VnS()); - break; - } - case 8: { - zip2(Dst.Z().VnD(), VectorLower.Z().VnD(), VectorUpper.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + zip2(SubRegSize, Dst.Z(), VectorLower.Z(), VectorUpper.Z()); } else { if (OpSize == 8) { - switch (ElementSize) { - case 1: { - zip2(Dst.V8B(), VectorLower.V8B(), VectorUpper.V8B()); - break; - } - case 2: { - zip2(Dst.V4H(), VectorLower.V4H(), VectorUpper.V4H()); - break; - } - case 4: { - zip2(Dst.V2S(), VectorLower.V2S(), VectorUpper.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + zip2(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D()); } else { - switch (ElementSize) { - case 1: { - zip2(Dst.V16B(), VectorLower.V16B(), VectorUpper.V16B()); - break; - } - case 2: { - zip2(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); - break; - } - case 4: { - zip2(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); - break; - } - case 8: { - zip2(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + zip2(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q()); } } } @@ -2267,69 +1364,20 @@ DEF_OP(VUnZip) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 1: { - uzp1(Dst.Z().VnB(), VectorLower.Z().VnB(), VectorUpper.Z().VnB()); - break; - } - case 2: { - uzp1(Dst.Z().VnH(), VectorLower.Z().VnH(), VectorUpper.Z().VnH()); - break; - } - case 4: { - uzp1(Dst.Z().VnS(), VectorLower.Z().VnS(), VectorUpper.Z().VnS()); - break; - } - case 8: { - uzp1(Dst.Z().VnD(), VectorLower.Z().VnD(), VectorUpper.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uzp1(SubRegSize, Dst.Z(), VectorLower.Z(), VectorUpper.Z()); } else { if (OpSize == 8) { - switch (ElementSize) { - case 1: { - uzp1(Dst.V8B(), VectorLower.V8B(), VectorUpper.V8B()); - break; - } - case 2: { - uzp1(Dst.V4H(), VectorLower.V4H(), VectorUpper.V4H()); - break; - } - case 4: { - uzp1(Dst.V2S(), VectorLower.V2S(), VectorUpper.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uzp1(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D()); } else { - switch (ElementSize) { - case 1: { - uzp1(Dst.V16B(), VectorLower.V16B(), VectorUpper.V16B()); - break; - } - case 2: { - uzp1(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); - break; - } - case 4: { - uzp1(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); - break; - } - case 8: { - uzp1(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uzp1(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q()); } } } @@ -2345,69 +1393,20 @@ DEF_OP(VUnZip2) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 1: { - uzp2(Dst.Z().VnB(), VectorLower.Z().VnB(), VectorUpper.Z().VnB()); - break; - } - case 2: { - uzp2(Dst.Z().VnH(), VectorLower.Z().VnH(), VectorUpper.Z().VnH()); - break; - } - case 4: { - uzp2(Dst.Z().VnS(), VectorLower.Z().VnS(), VectorUpper.Z().VnS()); - break; - } - case 8: { - uzp2(Dst.Z().VnD(), VectorLower.Z().VnD(), VectorUpper.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uzp2(SubRegSize, Dst.Z(), VectorLower.Z(), VectorUpper.Z()); } else { if (OpSize == 8) { - switch (ElementSize) { - case 1: { - uzp2(Dst.V8B(), VectorLower.V8B(), VectorUpper.V8B()); - break; - } - case 2: { - uzp2(Dst.V4H(), VectorLower.V4H(), VectorUpper.V4H()); - break; - } - case 4: { - uzp2(Dst.V2S(), VectorLower.V2S(), VectorUpper.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uzp2(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D()); } else { - switch (ElementSize) { - case 1: { - uzp2(Dst.V16B(), VectorLower.V16B(), VectorUpper.V16B()); - break; - } - case 2: { - uzp2(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); - break; - } - case 4: { - uzp2(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); - break; - } - case 8: { - uzp2(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uzp2(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q()); } } } @@ -2425,18 +1424,18 @@ DEF_OP(VBSL) { // NOTE: Slight parameter difference from ASIMD // ASIMD -> BSL Mask, True, False // SVE -> BSL True, True, False, Mask - mov(VTMP1.Z().VnD(), VectorTrue.Z().VnD()); - bsl(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VectorFalse.Z().VnD(), VectorMask.Z().VnD()); - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + mov(VTMP1.Z(), VectorTrue.Z()); + bsl(VTMP1.Z(), VTMP1.Z(), VectorFalse.Z(), VectorMask.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (OpSize == 8) { - mov(VTMP1.V8B(), VectorMask.V8B()); - bsl(VTMP1.V8B(), VectorTrue.V8B(), VectorFalse.V8B()); - mov(Dst.V8B(), VTMP1.V8B()); + mov(VTMP1.D(), VectorMask.D()); + bsl(VTMP1.D(), VectorTrue.D(), VectorFalse.D()); + mov(Dst.D(), VTMP1.D()); } else { - mov(VTMP1.V16B(), VectorMask.V16B()); - bsl(VTMP1.V16B(), VectorTrue.V16B(), VectorFalse.V16B()); - mov(Dst.V16B(), VTMP1.V16B()); + mov(VTMP1.Q(), VectorMask.Q()); + bsl(VTMP1.Q(), VectorTrue.Q(), VectorFalse.Q()); + mov(Dst.Q(), VTMP1.Q()); } } } @@ -2453,86 +1452,33 @@ DEF_OP(VCMPEQ) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure no junk is in the temp (important for ensuring - // non-equal entries remain as zero during the final bitwise OR). - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); + // non-equal entries remain as zero during the final bitwise OR). + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); // General idea is to compare for equality, not the equal vals // from one of the registers, then or both together to make the // relevant equal entries all 1s. - - switch (ElementSize) { - case 1: { - cmpeq(ComparePred.VnB(), Mask, Vector1.Z().VnB(), Vector2.Z().VnB()); - not_(VTMP1.Z().VnB(), ComparePred.Merging(), Vector1.Z().VnB()); - orr(VTMP1.Z().VnB(), ComparePred.Merging(), VTMP1.Z().VnB(), Vector1.Z().VnB()); - break; - } - case 2: { - cmpeq(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); - break; - } - case 4: { - cmpeq(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); - break; - } - case 8: { - cmpeq(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + cmpeq(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector1.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { - switch (ElementSize) { - case 4: { - cmeq(Dst.S(), Vector1.S(), Vector2.S()); - break; - } - case 8: { - cmeq(Dst.D(), Vector1.D(), Vector2.D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmeq(SubRegSize.Scalar, Dst, Vector1, Vector2); } else { - switch (ElementSize) { - case 1: { - cmeq(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - cmeq(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - cmeq(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - cmeq(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmeq(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -2548,82 +1494,32 @@ DEF_OP(VCMPEQZ) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure no junk is in the temp (important for ensuring // non-equal entries remain as zero). - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); // Unlike with VCMPEQ, we can skip needing to bitwise OR the // final results, since if our elements are equal to zero, // we just need to bitwise NOT them and they're already set // to all 1s. - switch (ElementSize) { - case 1: { - cmpeq(ComparePred.VnB(), Mask, Vector.Z().VnB(), 0); - not_(VTMP1.Z().VnB(), ComparePred.Merging(), Vector.Z().VnB()); - break; - } - case 2: { - cmpeq(ComparePred.VnH(), Mask, Vector.Z().VnH(), 0); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector.Z().VnH()); - break; - } - case 4: { - cmpeq(ComparePred.VnS(), Mask, Vector.Z().VnS(), 0); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector.Z().VnS()); - break; - } - case 8: { - cmpeq(ComparePred.VnD(), Mask, Vector.Z().VnD(), 0); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + cmpeq(SubRegSize.Vector, ComparePred, Mask, Vector.Z(), 0); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { - switch (ElementSize) { - case 4: { - cmeq(Dst.S(), Vector.S(), 0); - break; - } - case 8: { - cmeq(Dst.D(), Vector.D(), 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmeq(SubRegSize.Scalar, Dst, Vector); } else { - switch (ElementSize) { - case 1: { - cmeq(Dst.V16B(), Vector.V16B(), 0); - break; - } - case 2: { - cmeq(Dst.V8H(), Vector.V8H(), 0); - break; - } - case 4: { - cmeq(Dst.V4S(), Vector.V4S(), 0); - break; - } - case 8: { - cmeq(Dst.V2D(), Vector.V2D(), 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmeq(SubRegSize.Vector, Dst.Q(), Vector.Q()); } } } @@ -2640,86 +1536,33 @@ DEF_OP(VCMPGT) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure no junk is in the temp (important for ensuring // non greater-than values remain as zero). - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); // General idea is to compare for greater-than, bitwise NOT // the valid values, then ORR the NOTed values with the original // values to form entries that are all 1s. - - switch (ElementSize) { - case 1: { - cmpgt(ComparePred.VnB(), Mask, Vector1.Z().VnB(), Vector2.Z().VnB()); - not_(VTMP1.Z().VnB(), ComparePred.Merging(), Vector1.Z().VnB()); - orr(VTMP1.Z().VnB(), ComparePred.Merging(), VTMP1.Z().VnB(), Vector1.Z().VnB()); - break; - } - case 2: { - cmpgt(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); - break; - } - case 4: { - cmpgt(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); - break; - } - case 8: { - cmpgt(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + cmpgt(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector1.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { - switch (ElementSize) { - case 4: { - cmgt(Dst.S(), Vector1.S(), Vector2.S()); - break; - } - case 8: { - cmgt(Dst.D(), Vector1.D(), Vector2.D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmgt(SubRegSize.Scalar, Dst, Vector1, Vector2); } else { - switch (ElementSize) { - case 1: { - cmgt(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - cmgt(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - cmgt(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - cmgt(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmgt(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -2735,82 +1578,29 @@ DEF_OP(VCMPGTZ) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure no junk is in the temp (important for ensuring // non greater-than values remain as zero). - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - - switch (ElementSize) { - case 1: { - cmpgt(ComparePred.VnB(), Mask, Vector.Z().VnB(), 0); - not_(VTMP1.Z().VnB(), ComparePred.Merging(), Vector.Z().VnB()); - orr(VTMP1.Z().VnB(), ComparePred.Merging(), VTMP1.Z().VnB(), Vector.Z().VnB()); - break; - } - case 2: { - cmpgt(ComparePred.VnH(), Mask, Vector.Z().VnH(), 0); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector.Z().VnH()); - break; - } - case 4: { - cmpgt(ComparePred.VnS(), Mask, Vector.Z().VnS(), 0); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector.Z().VnS()); - break; - } - case 8: { - cmpgt(ComparePred.VnD(), Mask, Vector.Z().VnD(), 0); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + cmpgt(SubRegSize.Vector, ComparePred, Mask, Vector.Z(), 0); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { - switch (ElementSize) { - case 4: { - cmgt(Dst.S(), Vector.S(), 0); - break; - } - case 8: { - cmgt(Dst.D(), Vector.D(), 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmgt(SubRegSize.Scalar, Dst, Vector); } else { - switch (ElementSize) { - case 1: { - cmgt(Dst.V16B(), Vector.V16B(), 0); - break; - } - case 2: { - cmgt(Dst.V8H(), Vector.V8H(), 0); - break; - } - case 4: { - cmgt(Dst.V4S(), Vector.V4S(), 0); - break; - } - case 8: { - cmgt(Dst.V2D(), Vector.V2D(), 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmgt(SubRegSize.Vector, Dst.Q(), Vector.Q()); } } } @@ -2826,82 +1616,29 @@ DEF_OP(VCMPLTZ) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure no junk is in the temp (important for ensuring // non less-than values remain as zero). - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - - switch (ElementSize) { - case 1: { - cmplt(ComparePred.VnB(), Mask, Vector.Z().VnB(), 0); - not_(VTMP1.Z().VnB(), ComparePred.Merging(), Vector.Z().VnB()); - orr(VTMP1.Z().VnB(), ComparePred.Merging(), VTMP1.Z().VnB(), Vector.Z().VnB()); - break; - } - case 2: { - cmplt(ComparePred.VnH(), Mask, Vector.Z().VnH(), 0); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector.Z().VnH()); - break; - } - case 4: { - cmplt(ComparePred.VnS(), Mask, Vector.Z().VnS(), 0); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector.Z().VnS()); - break; - } - case 8: { - cmplt(ComparePred.VnD(), Mask, Vector.Z().VnD(), 0); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + cmplt(SubRegSize.Vector, ComparePred, Mask, Vector.Z(), 0); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { - switch (ElementSize) { - case 4: { - cmlt(Dst.S(), Vector.S(), 0); - break; - } - case 8: { - cmlt(Dst.D(), Vector.D(), 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmlt(SubRegSize.Scalar, Dst, Vector); } else { - switch (ElementSize) { - case 1: { - cmlt(Dst.V16B(), Vector.V16B(), 0); - break; - } - case 2: { - cmlt(Dst.V8H(), Vector.V8H(), 0); - break; - } - case 4: { - cmlt(Dst.V4S(), Vector.V4S(), 0); - break; - } - case 8: { - cmlt(Dst.V2D(), Vector.V2D(), 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + cmlt(SubRegSize.Vector, Dst.Q(), Vector.Q()); } } } @@ -2918,38 +1655,22 @@ DEF_OP(VFCMPEQ) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure we have no junk in the temporary. - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - - switch (ElementSize) { - case 2: { - fcmeq(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); - break; - } - case 4: { - fcmeq(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); - break; - } - case 8: { - fcmeq(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + fcmeq(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector1.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -2957,36 +1678,16 @@ DEF_OP(VFCMPEQ) { fcmeq(Dst.H(), Vector1.H(), Vector2.H()); break; } - case 4: { - fcmeq(Dst.S(), Vector1.S(), Vector2.S()); + case 4: + case 8: + fcmeq(SubRegSize.Scalar, Dst, Vector1, Vector2); break; - } - case 8: { - fcmeq(Dst.D(), Vector1.D(), Vector2.D()); - break; - } default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break; } } else { - switch (ElementSize) { - case 2: { - fcmeq(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - fcmeq(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - fcmeq(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmeq(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -3003,37 +1704,22 @@ DEF_OP(VFCMPNEQ) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure we have no junk in the temporary. - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - - switch (ElementSize) { - case 2: { - fcmne(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); - break; - } - case 4: { - fcmne(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); - break; - } - case 8: { - fcmne(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); - break; - } - default: - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + fcmne(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector1.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -3041,38 +1727,18 @@ DEF_OP(VFCMPNEQ) { fcmeq(Dst.H(), Vector1.H(), Vector2.H()); break; } - case 4: { - fcmeq(Dst.S(), Vector1.S(), Vector2.S()); + case 4: + case 8: + fcmeq(SubRegSize.Scalar, Dst, Vector1, Vector2); break; - } - case 8: { - fcmeq(Dst.D(), Vector1.D(), Vector2.D()); - break; - } default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break; } - mvn(Dst.V8B(), Dst.V8B()); + mvn(ARMEmitter::SubRegSize::i8Bit, Dst.D(), Dst.D()); } else { - switch (ElementSize) { - case 2: { - fcmeq(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - fcmeq(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - fcmeq(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - mvn(Dst.V16B(), Dst.V16B()); + fcmeq(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q()); + mvn(ARMEmitter::SubRegSize::i8Bit, Dst.Q(), Dst.Q()); } } } @@ -3089,38 +1755,22 @@ DEF_OP(VFCMPLT) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure we have no junk in the temporary. - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - - switch (ElementSize) { - case 2: { - fcmgt(ComparePred.VnH(), Mask, Vector2.Z().VnH(), Vector1.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector2.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - fcmgt(ComparePred.VnS(), Mask, Vector2.Z().VnS(), Vector1.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector2.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - fcmgt(ComparePred.VnD(), Mask, Vector2.Z().VnD(), Vector1.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector2.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + fcmgt(SubRegSize.Vector, ComparePred, Mask, Vector2.Z(), Vector1.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector2.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -3128,36 +1778,16 @@ DEF_OP(VFCMPLT) { fcmgt(Dst.H(), Vector2.H(), Vector1.H()); break; } - case 4: { - fcmgt(Dst.S(), Vector2.S(), Vector1.S()); + case 4: + case 8: + fcmgt(SubRegSize.Scalar, Dst, Vector2, Vector1); break; - } - case 8: { - fcmgt(Dst.D(), Vector2.D(), Vector1.D()); - break; - } default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break; } } else { - switch (ElementSize) { - case 2: { - fcmgt(Dst.V8H(), Vector2.V8H(), Vector1.V8H()); - break; - } - case 4: { - fcmgt(Dst.V4S(), Vector2.V4S(), Vector1.V4S()); - break; - } - case 8: { - fcmgt(Dst.V2D(), Vector2.V2D(), Vector1.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmgt(SubRegSize.Vector, Dst.Q(), Vector2.Q(), Vector1.Q()); } } } @@ -3174,38 +1804,22 @@ DEF_OP(VFCMPGT) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure there's no junk in the temporary. - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - - switch (ElementSize) { - case 2: { - fcmgt(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); - break; - } - case 4: { - fcmgt(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); - break; - } - case 8: { - fcmgt(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + fcmgt(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector1.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -3213,36 +1827,16 @@ DEF_OP(VFCMPGT) { fcmgt(Dst.H(), Vector1.H(), Vector2.H()); break; } - case 4: { - fcmgt(Dst.S(), Vector1.S(), Vector2.S()); + case 4: + case 8: + fcmgt(SubRegSize.Scalar, Dst, Vector1, Vector2); break; - } - case 8: { - fcmgt(Dst.D(), Vector1.D(), Vector2.D()); - break; - } default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break; } } else { - switch (ElementSize) { - case 2: { - fcmgt(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - fcmgt(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - fcmgt(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmgt(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q()); } } } @@ -3259,38 +1853,22 @@ DEF_OP(VFCMPLE) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure there's no junk in the temporary. - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); - - switch (ElementSize) { - case 2: { - fcmge(ComparePred.VnH(), Mask, Vector2.Z().VnH(), Vector1.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector2.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - fcmge(ComparePred.VnS(), Mask, Vector2.Z().VnS(), Vector1.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector2.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - fcmge(ComparePred.VnD(), Mask, Vector2.Z().VnD(), Vector1.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector2.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); + fcmge(SubRegSize.Vector, ComparePred, Mask, Vector2.Z(), Vector1.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector2.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { @@ -3298,36 +1876,16 @@ DEF_OP(VFCMPLE) { fcmge(Dst.H(), Vector2.H(), Vector1.H()); break; } - case 4: { - fcmge(Dst.S(), Vector2.S(), Vector1.S()); + case 4: + case 8: + fcmge(SubRegSize.Scalar, Dst, Vector2, Vector1); break; - } - case 8: { - fcmge(Dst.D(), Vector2.D(), Vector1.D()); - break; - } default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break; } } else { - switch (ElementSize) { - case 2: { - fcmge(Dst.V8H(), Vector2.V8H(), Vector1.V8H()); - break; - } - case 4: { - fcmge(Dst.V4S(), Vector2.V4S(), Vector1.V4S()); - break; - } - case 8: { - fcmge(Dst.V2D(), Vector2.V2D(), Vector1.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmge(SubRegSize.Vector, Dst.Q(), Vector2.Q(), Vector1.Q()); } } } @@ -3344,94 +1902,51 @@ DEF_OP(VFCMPORD) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Incorrect size"); + + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure there's no junk in the temporary. - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); // The idea is like comparing for unordered, but we just // invert the predicate from the comparison to instead // select all ordered elements in the vector. - - switch (ElementSize) { - case 2: { - fcmuo(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); - not_(ComparePred.VnB(), Mask, ComparePred.VnB()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); - break; - } - case 4: { - fcmuo(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); - not_(ComparePred.VnB(), Mask, ComparePred.VnB()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); - break; - } - case 8: { - fcmuo(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); - not_(ComparePred.VnB(), Mask, ComparePred.VnB()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + fcmuo(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z()); + not_(ComparePred, Mask, ComparePred); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector1.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { case 2: { fcmge(VTMP1.H(), Vector1.H(), Vector2.H()); fcmgt(VTMP2.H(), Vector2.H(), Vector1.H()); - orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B()); + orr(Dst.D(), VTMP1.D(), VTMP2.D()); break; } - case 4: { - fcmge(VTMP1.S(), Vector1.S(), Vector2.S()); - fcmgt(VTMP2.S(), Vector2.S(), Vector1.S()); - orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B()); + case 4: + case 8: + fcmge(SubRegSize.Scalar, VTMP1, Vector1, Vector2); + fcmgt(SubRegSize.Scalar, VTMP2, Vector2, Vector1); + orr(Dst.D(), VTMP1.D(), VTMP2.D()); break; - } - case 8: { - fcmge(VTMP1.D(), Vector1.D(), Vector2.D()); - fcmgt(VTMP2.D(), Vector2.D(), Vector1.D()); - orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B()); - break; - } default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break; } } else { - switch (ElementSize) { - case 2: { - fcmge(VTMP1.V8H(), Vector1.V8H(), Vector2.V8H()); - fcmgt(VTMP2.V8H(), Vector2.V8H(), Vector1.V8H()); - orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); - break; - } - case 4: { - fcmge(VTMP1.V4S(), Vector1.V4S(), Vector2.V4S()); - fcmgt(VTMP2.V4S(), Vector2.V4S(), Vector1.V4S()); - orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); - break; - } - case 8: { - fcmge(VTMP1.V2D(), Vector1.V2D(), Vector2.V2D()); - fcmgt(VTMP2.V2D(), Vector2.V2D(), Vector1.V2D()); - orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmge(SubRegSize.Vector, VTMP1.Q(), Vector1.Q(), Vector2.Q()); + fcmgt(SubRegSize.Vector, VTMP2.Q(), Vector2.Q(), Vector1.Q()); + orr(Dst.Q(), VTMP1.Q(), VTMP2.Q()); } } } @@ -3448,93 +1963,50 @@ DEF_OP(VFCMPUNO) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Incorrect size"); + + const auto SubRegSize = ARMEmitter::ToVectorSizePair( + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit); + if (HostSupportsSVE && Is256Bit && !IsScalar) { const auto Mask = PRED_TMP_32B.Zeroing(); - const auto ComparePred = p0; + const auto ComparePred = ARMEmitter::PReg::p0; // Ensure there's no junk in the temporary. - eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); + eor(VTMP1.Z(), VTMP1.Z(), VTMP1.Z()); - switch (ElementSize) { - case 2: { - fcmuo(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); - not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); - orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); - break; - } - case 4: { - fcmuo(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); - not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); - orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); - break; - } - case 8: { - fcmuo(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); - not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); - orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + fcmuo(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z()); + not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z()); + orr(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), VTMP1.Z(), Vector1.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { if (IsScalar) { switch (ElementSize) { case 2: { fcmge(VTMP1.H(), Vector1.H(), Vector2.H()); fcmgt(VTMP2.H(), Vector2.H(), Vector1.H()); - orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B()); - mvn(Dst.V8B(), Dst.V8B()); + orr(Dst.D(), VTMP1.D(), VTMP2.D()); + mvn(ARMEmitter::SubRegSize::i8Bit, Dst.D(), Dst.D()); break; } - case 4: { - fcmge(VTMP1.S(), Vector1.S(), Vector2.S()); - fcmgt(VTMP2.S(), Vector2.S(), Vector1.S()); - orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B()); - mvn(Dst.V8B(), Dst.V8B()); + case 4: + case 8: + fcmge(SubRegSize.Scalar, VTMP1, Vector1, Vector2); + fcmgt(SubRegSize.Scalar, VTMP2, Vector2, Vector1); + orr(Dst.D(), VTMP1.D(), VTMP2.D()); + mvn(ARMEmitter::SubRegSize::i8Bit, Dst.D(), Dst.D()); break; - } - case 8: { - fcmge(VTMP1.D(), Vector1.D(), Vector2.D()); - fcmgt(VTMP2.D(), Vector2.D(), Vector1.D()); - orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B()); - mvn(Dst.V8B(), Dst.V8B()); - break; - } default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break; } } else { - switch (ElementSize) { - case 2: { - fcmge(VTMP1.V8H(), Vector1.V8H(), Vector2.V8H()); - fcmgt(VTMP2.V8H(), Vector2.V8H(), Vector1.V8H()); - orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); - mvn(Dst.V16B(), Dst.V16B()); - break; - } - case 4: { - fcmge(VTMP1.V4S(), Vector1.V4S(), Vector2.V4S()); - fcmgt(VTMP2.V4S(), Vector2.V4S(), Vector1.V4S()); - orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); - mvn(Dst.V16B(), Dst.V16B()); - break; - } - case 8: { - fcmge(VTMP1.V2D(), Vector1.V2D(), Vector2.V2D()); - fcmgt(VTMP2.V2D(), Vector2.V2D(), Vector1.V2D()); - orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); - mvn(Dst.V16B(), Dst.V16B()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + fcmge(SubRegSize.Vector, VTMP1.Q(), Vector1.Q(), Vector2.Q()); + fcmgt(SubRegSize.Vector, VTMP2.Q(), Vector2.Q(), Vector1.Q()); + orr(Dst.Q(), VTMP1.Q(), VTMP2.Q()); + mvn(ARMEmitter::SubRegSize::i8Bit, Dst.Q(), Dst.Q()); } } } @@ -3562,66 +2034,23 @@ DEF_OP(VUShlS) { const auto ShiftScalar = GetVReg(Op->ShiftScalar.ID()); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); // NOTE: SVE LSL is a destructive operation. - - switch (ElementSize) { - case 1: { - dup(VTMP1.Z().VnB(), ShiftScalar.Z().VnB(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsl(Dst.Z().VnB(), Mask, Dst.Z().VnB(), VTMP1.Z().VnB()); - break; - } - case 2: { - dup(VTMP1.Z().VnH(), ShiftScalar.Z().VnH(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsl(Dst.Z().VnH(), Mask, Dst.Z().VnH(), VTMP1.Z().VnH()); - break; - } - case 4: { - dup(VTMP1.Z().VnS(), ShiftScalar.Z().VnS(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsl(Dst.Z().VnS(), Mask, Dst.Z().VnS(), VTMP1.Z().VnS()); - break; - } - case 8: { - dup(VTMP1.Z().VnD(), ShiftScalar.Z().VnD(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsl(Dst.Z().VnD(), Mask, Dst.Z().VnD(), VTMP1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + dup(SubRegSize, VTMP1.Z(), ShiftScalar.Z(), 0); + movprfx(Dst.Z(), Vector.Z()); + lsl(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z()); } else { - switch (ElementSize) { - case 1: { - dup(VTMP1.V16B(), ShiftScalar.V16B(), 0); - ushl(Dst.V16B(), Vector.V16B(), VTMP1.V16B()); - break; - } - case 2: { - dup(VTMP1.V8H(), ShiftScalar.V8H(), 0); - ushl(Dst.V8H(), Vector.V8H(), VTMP1.V8H()); - break; - } - case 4: { - dup(VTMP1.V4S(), ShiftScalar.V4S(), 0); - ushl(Dst.V4S(), Vector.V4S(), VTMP1.V4S()); - break; - } - case 8: { - dup(VTMP1.V2D(), ShiftScalar.V2D(), 0); - ushl(Dst.V2D(), Vector.V2D(), VTMP1.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + dup(SubRegSize, VTMP1.Q(), ShiftScalar.Q(), 0); + ushl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q()); } } @@ -3636,70 +2065,24 @@ DEF_OP(VUShrS) { const auto ShiftScalar = GetVReg(Op->ShiftScalar.ID()); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); // NOTE: SVE LSR is a destructive operation. - - switch (ElementSize) { - case 1: { - dup(VTMP1.Z().VnB(), ShiftScalar.Z().VnB(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsr(Dst.Z().VnB(), Mask, Dst.Z().VnB(), VTMP1.Z().VnB()); - break; - } - case 2: { - dup(VTMP1.Z().VnH(), ShiftScalar.Z().VnH(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsr(Dst.Z().VnH(), Mask, Dst.Z().VnH(), VTMP1.Z().VnH()); - break; - } - case 4: { - dup(VTMP1.Z().VnS(), ShiftScalar.Z().VnS(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsr(Dst.Z().VnS(), Mask, Dst.Z().VnS(), VTMP1.Z().VnS()); - break; - } - case 8: { - dup(VTMP1.Z().VnD(), ShiftScalar.Z().VnD(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - lsr(Dst.Z().VnD(), Mask, Dst.Z().VnD(), VTMP1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + dup(SubRegSize, VTMP1.Z(), ShiftScalar.Z(), 0); + movprfx(Dst.Z(), Vector.Z()); + lsr(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z()); } else { - switch (ElementSize) { - case 1: { - dup(VTMP1.V16B(), ShiftScalar.V16B(), 0); - neg(VTMP1.V16B(), VTMP1.V16B()); - ushl(Dst.V16B(), Vector.V16B(), VTMP1.V16B()); - break; - } - case 2: { - dup(VTMP1.V8H(), ShiftScalar.V8H(), 0); - neg(VTMP1.V8H(), VTMP1.V8H()); - ushl(Dst.V8H(), Vector.V8H(), VTMP1.V8H()); - break; - } - case 4: { - dup(VTMP1.V4S(), ShiftScalar.V4S(), 0); - neg(VTMP1.V4S(), VTMP1.V4S()); - ushl(Dst.V4S(), Vector.V4S(), VTMP1.V4S()); - break; - } - case 8: { - dup(VTMP1.V2D(), ShiftScalar.V2D(), 0); - neg(VTMP1.V2D(), VTMP1.V2D()); - ushl(Dst.V2D(), Vector.V2D(), VTMP1.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + dup(SubRegSize, VTMP1.Q(), ShiftScalar.Q(), 0); + neg(SubRegSize, VTMP1.Q(), VTMP1.Q()); + ushl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q()); } } @@ -3714,70 +2097,24 @@ DEF_OP(VSShrS) { const auto ShiftScalar = GetVReg(Op->ShiftScalar.ID()); const auto Vector = GetVReg(Op->Vector.ID()); - if (HostSupportsSVE && Is256Bit) { + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + + if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); // NOTE: SVE ASR is a destructive operation. - - switch (ElementSize) { - case 1: { - dup(VTMP1.Z().VnB(), ShiftScalar.Z().VnB(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - asr(Dst.Z().VnB(), Mask, Dst.Z().VnB(), VTMP1.Z().VnB()); - break; - } - case 2: { - dup(VTMP1.Z().VnH(), ShiftScalar.Z().VnH(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - asr(Dst.Z().VnH(), Mask, Dst.Z().VnH(), VTMP1.Z().VnH()); - break; - } - case 4: { - dup(VTMP1.Z().VnS(), ShiftScalar.Z().VnS(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - asr(Dst.Z().VnS(), Mask, Dst.Z().VnS(), VTMP1.Z().VnS()); - break; - } - case 8: { - dup(VTMP1.Z().VnD(), ShiftScalar.Z().VnD(), 0); - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - asr(Dst.Z().VnD(), Mask, Dst.Z().VnD(), VTMP1.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + dup(SubRegSize, VTMP1.Z(), ShiftScalar.Z(), 0); + movprfx(Dst.Z(), Vector.Z()); + asr(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP1.Z()); } else { - switch (ElementSize) { - case 1: { - dup(VTMP1.V16B(), ShiftScalar.V16B(), 0); - neg(VTMP1.V16B(), VTMP1.V16B()); - sshl(Dst.V16B(), Vector.V16B(), VTMP1.V16B()); - break; - } - case 2: { - dup(VTMP1.V8H(), ShiftScalar.V8H(), 0); - neg(VTMP1.V8H(), VTMP1.V8H()); - sshl(Dst.V8H(), Vector.V8H(), VTMP1.V8H()); - break; - } - case 4: { - dup(VTMP1.V4S(), ShiftScalar.V4S(), 0); - neg(VTMP1.V4S(), VTMP1.V4S()); - sshl(Dst.V4S(), Vector.V4S(), VTMP1.V4S()); - break; - } - case 8: { - dup(VTMP1.V2D(), ShiftScalar.V2D(), 0); - neg(VTMP1.V2D(), VTMP1.V2D()); - sshl(Dst.V2D(), Vector.V2D(), VTMP1.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + dup(SubRegSize, VTMP1.Q(), ShiftScalar.Q(), 0); + neg(SubRegSize, VTMP1.Q(), VTMP1.Q()); + sshl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q()); } } @@ -3801,116 +2138,79 @@ DEF_OP(VInsElement) { // 32-bit in size. We want to set up only the element corresponding // to the destination index, since we're going to copy over the equivalent // indexed element from the source vector. - auto Data = [ElementSize, DestIdx] { - using LiteralType = aarch64::Literal; - + auto Data = [ElementSize, DestIdx]() -> uint32_t { switch (ElementSize) { case 1: LOGMAN_THROW_AA_FMT(DestIdx <= 31, "DestIdx out of range: {}", DestIdx); - return LiteralType{1U << DestIdx}; + return 1U << DestIdx; case 2: LOGMAN_THROW_AA_FMT(DestIdx <= 15, "DestIdx out of range: {}", DestIdx); - return LiteralType{1U << (DestIdx * 2)}; + return 1U << (DestIdx * 2); case 4: LOGMAN_THROW_AA_FMT(DestIdx <= 7, "DestIdx out of range: {}", DestIdx); - return LiteralType{1U << (DestIdx * 4)}; + return 1U << (DestIdx * 4); case 8: LOGMAN_THROW_AA_FMT(DestIdx <= 3, "DestIdx out of range: {}", DestIdx); - return LiteralType{1U << (DestIdx * 8)}; + return 1U << (DestIdx * 8); case 16: LOGMAN_THROW_AA_FMT(DestIdx <= 1, "DestIdx out of range: {}", DestIdx); // Predicates can't be subdivided into the Q format, so we can just set up // the predicate to select the two adjacent doublewords. - return LiteralType{0x101U << (DestIdx * 16)}; + return 0x101U << (DestIdx * 16); default: - return LiteralType{UINT32_MAX}; + FEX_UNREACHABLE; + return UINT32_MAX; } }(); // Load our predicate register. - const auto Predicate = p0; - aarch64::Label DataLocation; + const auto Predicate = ARMEmitter::PReg::p0; + ARMEmitter::ForwardLabel DataLocation; adr(TMP1, &DataLocation); - ldr(Predicate, SVEMemOperand(TMP1)); + ldr(Predicate, TMP1); + + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; // Broadcast our source value across a temporary, // then combine with the destination. - switch (ElementSize) { - case 1: { - LOGMAN_THROW_AA_FMT(SrcIdx <= 31, "SrcIdx out of range: {}", SrcIdx); - dup(VTMP2.Z().VnB(), SrcVector.Z().VnB(), SrcIdx); - mov(Dst.Z().VnD(), Reg.Z().VnD()); - mov(Dst.Z().VnB(), Predicate.Merging(), VTMP2.Z().VnB()); - break; - } - case 2: { - LOGMAN_THROW_AA_FMT(SrcIdx <= 15, "SrcIdx out of range: {}", SrcIdx); - dup(VTMP2.Z().VnH(), SrcVector.Z().VnH(), SrcIdx); - mov(Dst.Z().VnD(), Reg.Z().VnD()); - mov(Dst.Z().VnH(), Predicate.Merging(), VTMP2.Z().VnH()); - break; - } - case 4: { - LOGMAN_THROW_AA_FMT(SrcIdx <= 7, "SrcIdx out of range: {}", SrcIdx); - dup(VTMP2.Z().VnS(), SrcVector.Z().VnS(), SrcIdx); - mov(Dst.Z().VnD(), Reg.Z().VnD()); - mov(Dst.Z().VnS(), Predicate.Merging(), VTMP2.Z().VnS()); - break; - } - case 8: { - LOGMAN_THROW_AA_FMT(SrcIdx <= 3, "SrcIdx out of range: {}", SrcIdx); - dup(VTMP2.Z().VnD(), SrcVector.Z().VnD(), SrcIdx); - mov(Dst.Z().VnD(), Reg.Z().VnD()); - mov(Dst.Z().VnD(), Predicate.Merging(), VTMP2.Z().VnD()); - break; - case 16: - LOGMAN_THROW_AA_FMT(SrcIdx <= 1, "SrcIdx out of range: {}", SrcIdx); - dup(VTMP2.Z().VnQ(), SrcVector.Z().VnQ(), SrcIdx); - mov(Dst.Z().VnD(), Reg.Z().VnD()); - mov(Dst.Z().VnD(), Predicate.Merging(), VTMP2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; + dup(SubRegSize, VTMP2.Z(), SrcVector.Z(), SrcIdx); + mov(Dst.Z(), Reg.Z()); + if (ElementSize == 16) { + mov(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Predicate, VTMP2.Z()); + } + else { + mov(SubRegSize, Dst.Z(), Predicate, VTMP2.Z()); } // Set up a label to jump over the data we inserted, so we don't try and execute it. - aarch64::Label PastConstant; + ARMEmitter::ForwardLabel PastConstant; b(&PastConstant); - bind(&DataLocation); - place(&Data); - bind(&PastConstant); - } else { - if (Dst.GetCode() != Reg.GetCode()) { - mov(VTMP1, Reg); + Bind(&DataLocation); + dc32(Data); + Bind(&PastConstant); + } + else { + if (Dst.Idx() != Reg.Idx()) { + mov(VTMP1.Q(), Reg.Q()); Reg = VTMP1; } - switch (ElementSize) { - case 1: { - mov(Reg.V16B(), DestIdx, SrcVector.V16B(), SrcIdx); - break; - } - case 2: { - mov(Reg.V8H(), DestIdx, SrcVector.V8H(), SrcIdx); - break; - } - case 4: { - mov(Reg.V4S(), DestIdx, SrcVector.V4S(), SrcIdx); - break; - } - case 8: { - mov(Reg.V2D(), DestIdx, SrcVector.V2D(), SrcIdx); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; - if (Dst.GetCode() != Reg.GetCode()) { - mov(Dst, Reg); + ins(SubRegSize, Reg.Q(), DestIdx, SrcVector.Q(), SrcIdx); + + if (Dst.Idx() != Reg.Idx()) { + mov(Dst.Q(), Reg.Q()); } } } @@ -3926,45 +2226,17 @@ DEF_OP(VDupElement) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 1: - dup(Dst.Z().VnB(), Vector.Z().VnB(), Index); - break; - case 2: - dup(Dst.Z().VnH(), Vector.Z().VnH(), Index); - break; - case 4: - dup(Dst.Z().VnS(), Vector.Z().VnS(), Index); - break; - case 8: - dup(Dst.Z().VnD(), Vector.Z().VnD(), Index); - break; - case 16: - dup(Dst.Z().VnQ(), Vector.Z().VnQ(), Index); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled VDupElement element size: {}", ElementSize); - break; - } + dup(SubRegSize, Dst.Z(), Vector.Z(), Index); } else { - switch (ElementSize) { - case 1: - dup(Dst.V16B(), Vector.V16B(), Index); - break; - case 2: - dup(Dst.V8H(), Vector.V8H(), Index); - break; - case 4: - dup(Dst.V4S(), Vector.V4S(), Index); - break; - case 8: - dup(Dst.V2D(), Vector.V2D(), Index); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled VDupElement element size: {}", ElementSize); - break; - } + dup(SubRegSize, Dst.Q(), Vector.Q(), Index); } } @@ -3987,21 +2259,21 @@ DEF_OP(VExtr) { // Upper bits are all now zero UpperBits = VTMP1; - eor(VTMP1.V16B(), VTMP1.V16B(), VTMP1.V16B()); + eor(VTMP1.Q(), VTMP1.Q(), VTMP1.Q()); Index -= OpSize; } const auto CopyFromByte = Index * ElementSize; if (HostSupportsSVE && Is256Bit) { - movprfx(VTMP2.Z().VnD(), LowerBits.Z().VnD()); - ext(VTMP2.Z().VnB(), VTMP2.Z().VnB(), UpperBits.Z().VnB(), CopyFromByte); - mov(Dst.Z().VnD(), VTMP2.Z().VnD()); + movprfx(VTMP2.Z(), LowerBits.Z()); + ext(VTMP2.Z(), VTMP2.Z(), UpperBits.Z(), CopyFromByte); + mov(Dst.Z(), VTMP2.Z()); } else { if (OpSize == 8) { - ext(Dst.V8B(), LowerBits.V8B(), UpperBits.V8B(), CopyFromByte); + ext(Dst.D(), LowerBits.D(), UpperBits.D(), CopyFromByte); } else { - ext(Dst.V16B(), LowerBits.V16B(), UpperBits.V16B(), CopyFromByte); + ext(Dst.Q(), LowerBits.Q(), UpperBits.Q(), CopyFromByte); } } } @@ -4017,58 +2289,24 @@ DEF_OP(VUShrI) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (BitShift >= (ElementSize * 8)) { - eor(Dst.V16B(), Dst.V16B(), Dst.V16B()); + eor(Dst.D(), Dst.D(), Dst.D()); } else { if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); // SVE LSR is destructive, so lets set up the destination. - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - - switch (ElementSize) { - case 1: { - lsr(Dst.Z().VnB(), Mask, Dst.Z().VnB(), BitShift); - break; - } - case 2: { - lsr(Dst.Z().VnH(), Mask, Dst.Z().VnH(), BitShift); - break; - } - case 4: { - lsr(Dst.Z().VnS(), Mask, Dst.Z().VnS(), BitShift); - break; - } - case 8: { - lsr(Dst.Z().VnD(), Mask, Dst.Z().VnD(), BitShift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + movprfx(Dst.Z(), Vector.Z()); + lsr(SubRegSize, Dst.Z(), Mask, Dst.Z(), BitShift); } else { - switch (ElementSize) { - case 1: { - ushr(Dst.V16B(), Vector.V16B(), BitShift); - break; - } - case 2: { - ushr(Dst.V8H(), Vector.V8H(), BitShift); - break; - } - case 4: { - ushr(Dst.V4S(), Vector.V4S(), BitShift); - break; - } - case 8: { - ushr(Dst.V2D(), Vector.V2D(), BitShift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + ushr(SubRegSize, Dst.Q(), Vector.Q(), BitShift); } } } @@ -4084,55 +2322,21 @@ DEF_OP(VSShrI) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); // SVE ASR is destructive, so lets set up the destination. - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); - - switch (ElementSize) { - case 1: { - asr(Dst.Z().VnB(), Mask, Dst.Z().VnB(), Shift); - break; - } - case 2: { - asr(Dst.Z().VnH(), Mask, Dst.Z().VnH(), Shift); - break; - } - case 4: { - asr(Dst.Z().VnS(), Mask, Dst.Z().VnS(), Shift); - break; - } - case 8: { - asr(Dst.Z().VnD(), Mask, Dst.Z().VnD(), Shift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + movprfx(Dst.Z(), Vector.Z()); + asr(SubRegSize, Dst.Z(), Mask, Dst.Z(), Shift); } else { - switch (ElementSize) { - case 1: { - sshr(Dst.V16B(), Vector.V16B(), Shift); - break; - } - case 2: { - sshr(Dst.V8H(), Vector.V8H(), Shift); - break; - } - case 4: { - sshr(Dst.V4S(), Vector.V4S(), Shift); - break; - } - case 8: { - sshr(Dst.V2D(), Vector.V2D(), Shift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sshr(SubRegSize, Dst.Q(), Vector.Q(), Shift); } } @@ -4147,58 +2351,26 @@ DEF_OP(VShlI) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + + if (BitShift >= (ElementSize * 8)) { - eor(Dst.V16B(), Dst.V16B(), Dst.V16B()); + eor(Dst.D(), Dst.D(), Dst.D()); } else { if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); // SVE LSL is destructive, so lets set up the destination. - movprfx(Dst.Z().VnD(), Vector.Z().VnD()); + movprfx(Dst.Z(), Vector.Z()); - switch (ElementSize) { - case 1: { - lsl(Dst.Z().VnB(), Mask, Dst.Z().VnB(), BitShift); - break; - } - case 2: { - lsl(Dst.Z().VnH(), Mask, Dst.Z().VnH(), BitShift); - break; - } - case 4: { - lsl(Dst.Z().VnS(), Mask, Dst.Z().VnS(), BitShift); - break; - } - case 8: { - lsl(Dst.Z().VnD(), Mask, Dst.Z().VnD(), BitShift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + lsl(SubRegSize, Dst.Z(), Mask, Dst.Z(), BitShift); } else { - switch (ElementSize) { - case 1: { - shl(Dst.V16B(), Vector.V16B(), BitShift); - break; - } - case 2: { - shl(Dst.V8H(), Vector.V8H(), BitShift); - break; - } - case 4: { - shl(Dst.V4S(), Vector.V4S(), BitShift); - break; - } - case 8: { - shl(Dst.V2D(), Vector.V2D(), BitShift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + shl(SubRegSize, Dst.Q(), Vector.Q(), BitShift); } } } @@ -4213,46 +2385,18 @@ DEF_OP(VUShrNI) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4, "Incorrect size"); + + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit; if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 1: { - shrnb(Dst.Z().VnB(), Vector.Z().VnH(), BitShift); - uzp1(Dst.Z().VnB(), Dst.Z().VnB(), Dst.Z().VnB()); - break; - } - case 2: { - shrnb(Dst.Z().VnH(), Vector.Z().VnS(), BitShift); - uzp1(Dst.Z().VnH(), Dst.Z().VnH(), Dst.Z().VnH()); - break; - } - case 4: { - shrnb(Dst.Z().VnS(), Vector.Z().VnD(), BitShift); - uzp1(Dst.Z().VnS(), Dst.Z().VnS(), Dst.Z().VnS()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + shrnb(SubRegSize, Dst.Z(), Vector.Z(), BitShift); + uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z()); } else { - switch (ElementSize) { - case 1: { - shrn(Dst.V8B(), Vector.V8H(), BitShift); - break; - } - case 2: { - shrn(Dst.V4H(), Vector.V4S(), BitShift); - break; - } - case 4: { - shrn(Dst.V2S(), Vector.V2D(), BitShift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + shrn(SubRegSize, Dst.D(), Vector.D(), BitShift); } } @@ -4268,58 +2412,25 @@ DEF_OP(VUShrNI2) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - mov(VTMP1.Z().VnD(), VectorLower.Z().VnD()); + mov(VTMP1.Z(), VectorLower.Z()); const auto Mask = PRED_TMP_16B; - switch (ElementSize) { - case 1: { - shrnb(VTMP2.Z().VnB(), VectorUpper.Z().VnH(), BitShift); - uzp1(VTMP2.Z().VnB(), VTMP2.Z().VnB(), VTMP2.Z().VnB()); - splice(VTMP1.Z().VnB(), Mask, VTMP1.Z().VnB(), VTMP2.Z().VnB()); - break; - } - case 2: { - shrnb(VTMP2.Z().VnH(), VectorUpper.Z().VnS(), BitShift); - uzp1(VTMP2.Z().VnH(), VTMP2.Z().VnH(), VTMP2.Z().VnH()); - splice(VTMP1.Z().VnH(), Mask, VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - } - case 4: { - shrnb(VTMP2.Z().VnS(), VectorUpper.Z().VnD(), BitShift); - uzp1(VTMP2.Z().VnS(), VTMP2.Z().VnS(), VTMP2.Z().VnS()); - splice(VTMP1.Z().VnS(), Mask, VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + shrnb(SubRegSize, VTMP2.Z(), VectorUpper.Z(), BitShift); + uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z()); + splice(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), VTMP2.Z()); + mov(Dst.Z(), VTMP1.Z()); } else { - mov(VTMP1, VectorLower); - - switch (ElementSize) { - case 1: { - shrn2(VTMP1.V16B(), VectorUpper.V8H(), BitShift); - break; - } - case 2: { - shrn2(VTMP1.V8H(), VectorUpper.V4S(), BitShift); - break; - } - case 4: { - shrn2(VTMP1.V4S(), VectorUpper.V2D(), BitShift); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - mov(Dst, VTMP1); + mov(VTMP1.Q(), VectorLower.Q()); + shrn2(SubRegSize, VTMP1.Q(), VectorUpper.Q(), BitShift); + mov(Dst.Q(), VTMP1.Q()); } } @@ -4332,37 +2443,17 @@ DEF_OP(VSXTL) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Incorrect size"); + + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: - sunpklo(Dst.Z().VnH(), Vector.Z().VnB()); - break; - case 4: - sunpklo(Dst.Z().VnS(), Vector.Z().VnH()); - break; - case 8: - sunpklo(Dst.Z().VnD(), Vector.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sunpklo(SubRegSize, Dst.Z(), Vector.Z()); } else { - switch (ElementSize) { - case 2: - sxtl(Dst.V8H(), Vector.V8B()); - break; - case 4: - sxtl(Dst.V4S(), Vector.V4H()); - break; - case 8: - sxtl(Dst.V2D(), Vector.V2S()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sxtl(SubRegSize, Dst.D(), Vector.D()); } } @@ -4375,37 +2466,17 @@ DEF_OP(VSXTL2) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Incorrect size"); + + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: - sunpkhi(Dst.Z().VnH(), Vector.Z().VnB()); - break; - case 4: - sunpkhi(Dst.Z().VnS(), Vector.Z().VnH()); - break; - case 8: - sunpkhi(Dst.Z().VnD(), Vector.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sunpkhi(SubRegSize, Dst.Z(), Vector.Z()); } else { - switch (ElementSize) { - case 2: - sxtl2(Dst.V8H(), Vector.V16B()); - break; - case 4: - sxtl2(Dst.V4S(), Vector.V8H()); - break; - case 8: - sxtl2(Dst.V2D(), Vector.V4S()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sxtl2(SubRegSize, Dst.Q(), Vector.Q()); } } @@ -4418,37 +2489,17 @@ DEF_OP(VUXTL) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Incorrect size"); + + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: - uunpklo(Dst.Z().VnH(), Vector.Z().VnB()); - break; - case 4: - uunpklo(Dst.Z().VnS(), Vector.Z().VnH()); - break; - case 8: - uunpklo(Dst.Z().VnD(), Vector.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uunpklo(SubRegSize, Dst.Z(), Vector.Z()); } else { - switch (ElementSize) { - case 2: - uxtl(Dst.V8H(), Vector.V8B()); - break; - case 4: - uxtl(Dst.V4S(), Vector.V4H()); - break; - case 8: - uxtl(Dst.V2D(), Vector.V2S()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uxtl(SubRegSize, Dst.D(), Vector.D()); } } @@ -4462,37 +2513,17 @@ DEF_OP(VUXTL2) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Incorrect size"); + + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: - uunpkhi(Dst.Z().VnH(), Vector.Z().VnB()); - break; - case 4: - uunpkhi(Dst.Z().VnS(), Vector.Z().VnH()); - break; - case 8: - uunpkhi(Dst.Z().VnD(), Vector.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uunpkhi(SubRegSize, Dst.Z(), Vector.Z()); } else { - switch (ElementSize) { - case 2: - uxtl2(Dst.V8H(), Vector.V16B()); - break; - case 4: - uxtl2(Dst.V4S(), Vector.V8H()); - break; - case 8: - uxtl2(Dst.V2D(), Vector.V4S()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + uxtl2(SubRegSize, Dst.D(), Vector.D()); } } @@ -4505,6 +2536,12 @@ DEF_OP(VSQXTN) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4, "Incorrect size"); + + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit; if (HostSupportsSVE && Is256Bit) { // Note that SVE SQXTNB and SQXTNT are a tad different @@ -4527,51 +2564,23 @@ DEF_OP(VSQXTN) { // e.g. consider this 64-bit (for brevity) vector with four 16-bit elements: // // ╔═══════════╗╔═══════════╗╔═══════════╗╔═══════════╗ - // ║ Value 3 ║║ Value 2 ║║ Value 1 ║║ Value 0 ║ + // ║ Value 3 ║║ Value 2 ║║ Value 1 ║║ Value 0 ║ // ╚═══════════╝╚═══════════╝╚═══════════╝╚═══════════╝ // // SQXTNB Dst.VnB, Src.VnH will result in: // // ╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗╔═════╗ - // ║ 0 ║║ V3 ║║ 0 ║║ V2 ║║ 0 ║║ V1 ║║ 0 ║║ V0 ║ + // ║ 0 ║║ V3 ║║ 0 ║║ V2 ║║ 0 ║║ V1 ║║ 0 ║║ V0 ║ // ╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝╚═════╝ // // This is kind of convenient, considering we only need // to use the bottom variant and then concatenate all the // even elements with SVE UZP1. - switch (ElementSize) { - case 1: - sqxtnb(Dst.Z().VnB(), Vector.Z().VnH()); - uzp1(Dst.Z().VnB(), Dst.Z().VnB(), Dst.Z().VnB()); - break; - case 2: - sqxtnb(Dst.Z().VnH(), Vector.Z().VnS()); - uzp1(Dst.Z().VnH(), Dst.Z().VnH(), Dst.Z().VnH()); - break; - case 4: - sqxtnb(Dst.Z().VnS(), Vector.Z().VnD()); - uzp1(Dst.Z().VnS(), Dst.Z().VnS(), Dst.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sqxtnb(SubRegSize, Dst.Z(), Vector.Z()); + uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z()); } else { - switch (ElementSize) { - case 1: - sqxtn(Dst.V8B(), Vector.V8H()); - break; - case 2: - sqxtn(Dst.V4H(), Vector.V4S()); - break; - case 4: - sqxtn(Dst.V2S(), Vector.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sqxtn(SubRegSize, Dst, Vector); } } @@ -4586,12 +2595,18 @@ DEF_OP(VSQXTN2) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { // Need to use the destructive variant of SPLICE, since // the constructive variant requires a register list, and // we can't guarantee VectorLower and VectorUpper will always // have consecutive indexes with one another. - mov(VTMP1.Z().VnD(), VectorLower.Z().VnD()); + mov(VTMP1.Z(), VectorLower.Z()); // We use the 16 byte mask due to how SPLICE works. We only // want to get at the first 16 bytes in the lower vector, so @@ -4600,67 +2615,22 @@ DEF_OP(VSQXTN2) { // previously copied lower 16 bytes. const auto Mask = PRED_TMP_16B; - switch (ElementSize) { - case 1: - sqxtnb(VTMP2.Z().VnB(), VectorUpper.Z().VnH()); - uzp1(VTMP2.Z().VnB(), VTMP2.Z().VnB(), VTMP2.Z().VnB()); - splice(VTMP1.Z().VnB(), Mask, VTMP1.Z().VnB(), VTMP2.Z().VnB()); - break; - case 2: - sqxtnb(VTMP2.Z().VnH(), VectorUpper.Z().VnS()); - uzp1(VTMP2.Z().VnH(), VTMP2.Z().VnH(), VTMP2.Z().VnH()); - splice(VTMP1.Z().VnH(), Mask, VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - case 4: - sqxtnb(VTMP2.Z().VnS(), VectorUpper.Z().VnD()); - uzp1(VTMP2.Z().VnS(), VTMP2.Z().VnS(), VTMP2.Z().VnS()); - splice(VTMP1.Z().VnS(), Mask, VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + sqxtnb(SubRegSize, VTMP2.Z(), VectorUpper.Z()); + uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z()); + splice(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), VTMP2.Z()); - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + mov(Dst.Z(), VTMP1.Z()); } else { - mov(VTMP1, VectorLower); + mov(VTMP1.Q(), VectorLower.Q()); if (OpSize == 8) { - switch (ElementSize) { - case 1: - sqxtn(VTMP2.V8B(), VectorUpper.V8H()); - ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); - break; - case 2: - sqxtn(VTMP2.V4H(), VectorUpper.V4S()); - ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); - break; - case 4: - sqxtn(VTMP2.V2S(), VectorUpper.V2D()); - ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + sqxtn(SubRegSize, VTMP2, VectorUpper); + ins(ARMEmitter::SubRegSize::i32Bit, VTMP1, 1, VTMP2, 0); } else { - switch (ElementSize) { - case 1: - sqxtn2(VTMP1.V16B(), VectorUpper.V8H()); - break; - case 2: - sqxtn2(VTMP1.V8H(), VectorUpper.V4S()); - break; - case 4: - sqxtn2(VTMP1.V4S(), VectorUpper.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + sqxtn2(SubRegSize, VTMP1, VectorUpper); } - mov(Dst, VTMP1); + mov(Dst.Q(), VTMP1.Q()); } } @@ -4674,39 +2644,17 @@ DEF_OP(VSQXTUN) { const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 1: - sqxtunb(Dst.Z().VnB(), Vector.Z().VnH()); - uzp1(Dst.Z().VnB(), Dst.Z().VnB(), Dst.Z().VnB()); - break; - case 2: - sqxtunb(Dst.Z().VnH(), Vector.Z().VnS()); - uzp1(Dst.Z().VnH(), Dst.Z().VnH(), Dst.Z().VnH()); - break; - case 4: - sqxtunb(Dst.Z().VnS(), Vector.Z().VnD()); - uzp1(Dst.Z().VnS(), Dst.Z().VnS(), Dst.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sqxtunb(SubRegSize, Dst.Z(), Vector.Z()); + uzp1(SubRegSize, Dst.Z(), Dst.Z(), Dst.Z()); } else { - switch (ElementSize) { - case 1: - sqxtun(Dst.V8B(), Vector.V8H()); - break; - case 2: - sqxtun(Dst.V4H(), Vector.V4S()); - break; - case 4: - sqxtun(Dst.V2S(), Vector.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + sqxtun(SubRegSize, Dst, Vector); } } @@ -4721,73 +2669,34 @@ DEF_OP(VSQXTUN2) { const auto VectorLower = GetVReg(Op->VectorLower.ID()); const auto VectorUpper = GetVReg(Op->VectorUpper.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { // NOTE: See VSQXTN2 implementation for an in-depth explanation // of everything going on here. - mov(VTMP1.Z().VnD(), VectorLower.Z().VnD()); + mov(VTMP1.Z(), VectorLower.Z()); const auto Mask = PRED_TMP_16B; - switch (ElementSize) { - case 1: - sqxtunb(VTMP2.Z().VnB(), VectorUpper.Z().VnH()); - uzp1(VTMP2.Z().VnB(), VTMP2.Z().VnB(), VTMP2.Z().VnB()); - splice(VTMP1.Z().VnB(), Mask, VTMP1.Z().VnB(), VTMP2.Z().VnB()); - break; - case 2: - sqxtunb(VTMP2.Z().VnH(), VectorUpper.Z().VnS()); - uzp1(VTMP2.Z().VnH(), VTMP2.Z().VnH(), VTMP2.Z().VnH()); - splice(VTMP1.Z().VnH(), Mask, VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - case 4: - sqxtunb(VTMP2.Z().VnS(), VectorUpper.Z().VnD()); - uzp1(VTMP2.Z().VnS(), VTMP2.Z().VnS(), VTMP2.Z().VnS()); - splice(VTMP1.Z().VnS(), Mask, VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + sqxtunb(SubRegSize, VTMP2.Z(), VectorUpper.Z()); + uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z()); + splice(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), VTMP2.Z()); - mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + mov(Dst.Z(), VTMP1.Z()); } else { - mov(VTMP1, VectorLower); + mov(VTMP1.Q(), VectorLower.Q()); if (OpSize == 8) { - switch (ElementSize) { - case 1: - sqxtun(VTMP2.V8B(), VectorUpper.V8H()); - ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); - break; - case 2: - sqxtun(VTMP2.V4H(), VectorUpper.V4S()); - ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); - break; - case 4: - sqxtun(VTMP2.V2S(), VectorUpper.V2D()); - ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + sqxtun(SubRegSize, VTMP2, VectorUpper); + ins(ARMEmitter::SubRegSize::i32Bit, VTMP1, 1, VTMP2, 0); } else { - switch (ElementSize) { - case 1: - sqxtun2(VTMP1.V16B(), VectorUpper.V8H()); - break; - case 2: - sqxtun2(VTMP1.V8H(), VectorUpper.V4S()); - break; - case 4: - sqxtun2(VTMP1.V4S(), VectorUpper.V2D()); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } + sqxtun2(SubRegSize, VTMP1, VectorUpper); } - mov(Dst, VTMP1); + mov(Dst.Q(), VTMP1.Q()); } } @@ -4800,50 +2709,17 @@ DEF_OP(VMul) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE) { - switch (ElementSize) { - case 1: { - mul(Dst.Z().VnB(), Vector1.Z().VnB(), Vector2.Z().VnB()); - break; - } - case 2: { - mul(Dst.Z().VnH(), Vector1.Z().VnH(), Vector2.Z().VnH()); - break; - } - case 4: { - mul(Dst.Z().VnS(), Vector1.Z().VnS(), Vector2.Z().VnS()); - break; - } - case 8: { - mul(Dst.Z().VnD(), Vector1.Z().VnD(), Vector2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + mul(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z()); } else { - switch (ElementSize) { - case 1: { - mul(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 2: { - mul(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 4: { - mul(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); - break; - } - case 8: { - mul(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } + mul(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -4858,48 +2734,18 @@ DEF_OP(VUMull) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: { - umullb(VTMP1.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - umullt(VTMP2.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - zip1(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - } - case 4: { - umullb(VTMP1.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - umullt(VTMP2.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - zip1(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - } - case 8: { - umullb(VTMP1.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - umullt(VTMP2.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - zip1(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + umullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z()); + umullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z()); + zip1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z()); } else { - switch (ElementSize) { - case 2: { - umull(Dst.V8H(), Vector1.V8B(), Vector2.V8B()); - break; - } - case 4: { - umull(Dst.V4S(), Vector1.V4H(), Vector2.V4H()); - break; - } - case 8: { - umull(Dst.V2D(), Vector1.V2S(), Vector2.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + umull(SubRegSize, Dst.D(), Vector1.D(), Vector2.D()); } } @@ -4914,48 +2760,18 @@ DEF_OP(VSMull) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: { - smullb(VTMP1.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - smullt(VTMP2.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - zip1(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - } - case 4: { - smullb(VTMP1.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - smullt(VTMP2.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - zip1(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - } - case 8: { - smullb(VTMP1.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - smullt(VTMP2.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - zip1(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + smullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z()); + smullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z()); + zip1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z()); } else { - switch (ElementSize) { - case 2: { - smull(Dst.V8H(), Vector1.V8B(), Vector2.V8B()); - break; - } - case 4: { - smull(Dst.V4S(), Vector1.V4H(), Vector2.V4H()); - break; - } - case 8: { - smull(Dst.V2D(), Vector1.V2S(), Vector2.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + smull(SubRegSize, Dst.D(), Vector1.D(), Vector2.D()); } } @@ -4970,48 +2786,18 @@ DEF_OP(VUMull2) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: { - umullb(VTMP1.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - umullt(VTMP2.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - zip2(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - } - case 4: { - umullb(VTMP1.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - umullt(VTMP2.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - zip2(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - } - case 8: { - umullb(VTMP1.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - umullt(VTMP2.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - zip2(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + umullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z()); + umullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z()); + zip2(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z()); } else { - switch (ElementSize) { - case 2: { - umull2(Dst.V8H(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 4: { - umull2(Dst.V4S(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 8: { - umull2(Dst.V2D(), Vector1.V4S(), Vector2.V4S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + umull2(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -5026,48 +2812,18 @@ DEF_OP(VSMull2) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { - switch (ElementSize) { - case 2: { - smullb(VTMP1.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - smullt(VTMP2.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - zip2(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - } - case 4: { - smullb(VTMP1.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - smullt(VTMP2.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - zip2(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - } - case 8: { - smullb(VTMP1.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - smullt(VTMP2.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - zip2(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + smullb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z()); + smullt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z()); + zip2(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z()); } else { - switch (ElementSize) { - case 2: { - smull2(Dst.V8H(), Vector1.V16B(), Vector2.V16B()); - break; - } - case 4: { - smull2(Dst.V4S(), Vector1.V8H(), Vector2.V8H()); - break; - } - case 8: { - smull2(Dst.V2D(), Vector1.V4S(), Vector2.V4S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + smull2(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); } } @@ -5082,53 +2838,23 @@ DEF_OP(VUABDL) { const auto Vector1 = GetVReg(Op->Vector1.ID()); const auto Vector2 = GetVReg(Op->Vector2.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size"); + const auto SubRegSize = + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : + ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { // To mimic the behavior of AdvSIMD UABDL, we need to get the // absolute difference of the even elements (UADBLB), get the // absolute difference of the odd elemenets (UABDLT), then // interleave the results in both vectors together. - - switch (ElementSize) { - case 2: { - uabdlb(VTMP1.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - uabdlt(VTMP2.Z().VnH(), Vector1.Z().VnB(), Vector2.Z().VnB()); - zip1(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); - break; - } - case 4: { - uabdlb(VTMP1.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - uabdlt(VTMP2.Z().VnS(), Vector1.Z().VnH(), Vector2.Z().VnH()); - zip1(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); - break; - } - case 8: { - uabdlb(VTMP1.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - uabdlt(VTMP2.Z().VnD(), Vector1.Z().VnS(), Vector2.Z().VnS()); - zip1(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - return; - } + + uabdlb(SubRegSize, VTMP1.Z(), Vector1.Z(), Vector2.Z()); + uabdlt(SubRegSize, VTMP2.Z(), Vector1.Z(), Vector2.Z()); + zip1(SubRegSize, Dst.Z(), VTMP1.Z(), VTMP2.Z()); } else { - switch (ElementSize) { - case 2: { - uabdl(Dst.V8H(), Vector1.V8B(), Vector2.V8B()); - break; - } - case 4: { - uabdl(Dst.V4S(), Vector1.V4H(), Vector2.V4H()); - break; - } - case 8: { - uabdl(Dst.V2D(), Vector1.V2S(), Vector2.V2S()); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize >> 1); - break; - } + uabdl(SubRegSize, Dst.D(), Vector1.D(), Vector2.D()); } } @@ -5142,18 +2868,18 @@ DEF_OP(VTBL1) { switch (OpSize) { case 8: { - tbl(Dst.V8B(), VectorTable.V16B(), VectorIndices.V8B()); + tbl(Dst.D(), VectorTable.D(), VectorIndices.D()); break; } case 16: { - tbl(Dst.V16B(), VectorTable.V16B(), VectorIndices.V16B()); + tbl(Dst.Q(), VectorTable.Q(), VectorIndices.Q()); break; } case 32: { LOGMAN_THROW_AA_FMT(HostSupportsSVE, "Host does not support SVE. Cannot perform 256-bit table lookup"); - tbl(Dst.Z().VnB(), VectorTable.Z().VnB(), VectorIndices.Z().VnB()); + tbl(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), VectorTable.Z(), VectorIndices.Z()); break; } default: @@ -5167,26 +2893,31 @@ DEF_OP(VRev64) { const auto OpSize = IROp->Size; const auto ElementSize = Op->Header.ElementSize; - const auto Elements = OpSize / ElementSize; const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; const auto Dst = GetVReg(Node); const auto Vector = GetVReg(Op->Vector.ID()); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4, "Invalid size"); + const auto SubRegSize = + ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : + ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : + ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit; + if (HostSupportsSVE && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); switch (ElementSize) { case 1: { - revb(Dst.Z().VnD(), Mask, Vector.Z().VnD()); + revb(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break; } case 2: { - revh(Dst.Z().VnD(), Mask, Vector.Z().VnD()); + revh(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break; } case 4: { - revw(Dst.Z().VnD(), Mask, Vector.Z().VnD()); + revw(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break; } default: @@ -5194,15 +2925,11 @@ DEF_OP(VRev64) { break; } } else { - switch (ElementSize) { - case 1: - case 2: - case 4: - rev64(Dst.VCast(OpSize * 8, Elements), Vector.VCast(OpSize * 8, Elements)); - break; - default: - LOGMAN_MSG_A_FMT("Invalid Element Size: {}", ElementSize); - break; + if (OpSize == 8) { + rev64(SubRegSize, Dst.D(), Vector.D()); + } + else { + rev64(SubRegSize, Dst.Q(), Vector.Q()); } } }