From c9712e45cb7baa1bd4c16048fd64735cb2c769aa Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Tue, 13 Jun 2023 19:45:22 -0700 Subject: [PATCH] Arm64: Fixes GPR pair allocation to get one pair back When executing a 32-bit application we were failing to allocate a single GPR pair. This meant we only have 7 pairs when we could have had 8. This was because r30 was ending up in the middle of the allocation arrays so we couldn't safely create a sequential pair of registers. Organize the register allocation arrays to be unique for each bitness being executed and then access them through spans instead. Also works around bug where the RA validation doesn't understand when pair indexes don't correlate directly to GPR indexes. So while the previous PR fixed the RA pass, it didn't fix the RA validation pass. Noticed this when pr57018 32-bit gcc test was run with the #2700 PR which improved the RA allocation a bit. --- .../Core/ArchHelpers/Arm64Emitter.cpp | 272 +++++++++++++----- .../Interface/Core/ArchHelpers/Arm64Emitter.h | 126 +------- .../Core/Dispatcher/Arm64Dispatcher.h | 12 +- .../Interface/Core/JIT/Arm64/ALUOps.cpp | 6 +- .../Source/Interface/Core/JIT/Arm64/JIT.cpp | 17 +- .../Interface/Core/JIT/Arm64/JITClass.h | 10 +- .../Interface/Core/JIT/Arm64/MemoryOps.cpp | 24 +- 7 files changed, 237 insertions(+), 230 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp index f7dc0de1f..10ca7596b 100644 --- a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp +++ b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp @@ -20,6 +20,131 @@ #include namespace FEXCore::CPU { +// Register x18 is unused in the current configuration. +// This is due to it being a platform register on wine platforms. +// TODO: Allow x18 register allocation in the future to gain one more register. + +namespace x64 { + // All but x19 and x29 are caller saved + constexpr std::array SRA = { + FEXCore::ARMEmitter::Reg::r4, FEXCore::ARMEmitter::Reg::r5, + FEXCore::ARMEmitter::Reg::r6, FEXCore::ARMEmitter::Reg::r7, + FEXCore::ARMEmitter::Reg::r8, FEXCore::ARMEmitter::Reg::r9, + FEXCore::ARMEmitter::Reg::r10, FEXCore::ARMEmitter::Reg::r11, + FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r13, + FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r15, + FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r17, + FEXCore::ARMEmitter::Reg::r19, FEXCore::ARMEmitter::Reg::r29 + }; + + constexpr std::array RA = { + // All these callee saved + FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21, + FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23, + FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25, + FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27, + FEXCore::ARMEmitter::Reg::r30, + }; + + constexpr std::array, 4> RAPair = {{ + {FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21}, + {FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23}, + {FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25}, + {FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27}, + }}; + + // All are caller saved + constexpr std::array SRAFPR = { + FEXCore::ARMEmitter::VReg::v16, FEXCore::ARMEmitter::VReg::v17, + FEXCore::ARMEmitter::VReg::v18, FEXCore::ARMEmitter::VReg::v19, + FEXCore::ARMEmitter::VReg::v20, FEXCore::ARMEmitter::VReg::v21, + FEXCore::ARMEmitter::VReg::v22, FEXCore::ARMEmitter::VReg::v23, + FEXCore::ARMEmitter::VReg::v24, FEXCore::ARMEmitter::VReg::v25, + FEXCore::ARMEmitter::VReg::v26, FEXCore::ARMEmitter::VReg::v27, + FEXCore::ARMEmitter::VReg::v28, FEXCore::ARMEmitter::VReg::v29, + FEXCore::ARMEmitter::VReg::v30, FEXCore::ARMEmitter::VReg::v31 + }; + + // v8..v15 = (lower 64bits) Callee saved + constexpr std::array RAFPR = { + // v0 ~ v3 are used as temps. + // FEXCore::ARMEmitter::VReg::v0, FEXCore::ARMEmitter::VReg::v1, + // FEXCore::ARMEmitter::VReg::v2, FEXCore::ARMEmitter::VReg::v3, + + FEXCore::ARMEmitter::VReg::v4, FEXCore::ARMEmitter::VReg::v5, + FEXCore::ARMEmitter::VReg::v6, FEXCore::ARMEmitter::VReg::v7, + FEXCore::ARMEmitter::VReg::v8, FEXCore::ARMEmitter::VReg::v9, + FEXCore::ARMEmitter::VReg::v10, FEXCore::ARMEmitter::VReg::v11, + FEXCore::ARMEmitter::VReg::v12, FEXCore::ARMEmitter::VReg::v13, + FEXCore::ARMEmitter::VReg::v14, FEXCore::ARMEmitter::VReg::v15, + }; +} + +namespace x32 { + // All but x19 and x29 are caller saved + constexpr std::array SRA = { + FEXCore::ARMEmitter::Reg::r4, FEXCore::ARMEmitter::Reg::r5, + FEXCore::ARMEmitter::Reg::r6, FEXCore::ARMEmitter::Reg::r7, + FEXCore::ARMEmitter::Reg::r8, FEXCore::ARMEmitter::Reg::r9, + FEXCore::ARMEmitter::Reg::r10, FEXCore::ARMEmitter::Reg::r11, + }; + + constexpr std::array RA = { + // All these callee saved + FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21, + FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23, + FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25, + FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27, + + // Registers only available on 32-bit + // All these are caller saved (except for r19). + FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r13, + FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r15, + FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r17, + FEXCore::ARMEmitter::Reg::r29, FEXCore::ARMEmitter::Reg::r30, + + FEXCore::ARMEmitter::Reg::r19, + }; + + constexpr std::array, 8> RAPair = {{ + {FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21}, + {FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23}, + {FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25}, + {FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27}, + + {FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r13}, + {FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r15}, + {FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r17}, + {FEXCore::ARMEmitter::Reg::r29, FEXCore::ARMEmitter::Reg::r30}, + }}; + + // All are caller saved + constexpr std::array SRAFPR = { + FEXCore::ARMEmitter::VReg::v16, FEXCore::ARMEmitter::VReg::v17, + FEXCore::ARMEmitter::VReg::v18, FEXCore::ARMEmitter::VReg::v19, + FEXCore::ARMEmitter::VReg::v20, FEXCore::ARMEmitter::VReg::v21, + FEXCore::ARMEmitter::VReg::v22, FEXCore::ARMEmitter::VReg::v23, + }; + + // v8..v15 = (lower 64bits) Callee saved + constexpr std::array RAFPR = { + // v0 ~ v3 are used as temps. + // FEXCore::ARMEmitter::VReg::v0, FEXCore::ARMEmitter::VReg::v1, + // FEXCore::ARMEmitter::VReg::v2, FEXCore::ARMEmitter::VReg::v3, + + FEXCore::ARMEmitter::VReg::v4, FEXCore::ARMEmitter::VReg::v5, + FEXCore::ARMEmitter::VReg::v6, FEXCore::ARMEmitter::VReg::v7, + FEXCore::ARMEmitter::VReg::v8, FEXCore::ARMEmitter::VReg::v9, + FEXCore::ARMEmitter::VReg::v10, FEXCore::ARMEmitter::VReg::v11, + FEXCore::ARMEmitter::VReg::v12, FEXCore::ARMEmitter::VReg::v13, + FEXCore::ARMEmitter::VReg::v14, FEXCore::ARMEmitter::VReg::v15, + + FEXCore::ARMEmitter::VReg::v24, FEXCore::ARMEmitter::VReg::v25, + FEXCore::ARMEmitter::VReg::v26, FEXCore::ARMEmitter::VReg::v27, + FEXCore::ARMEmitter::VReg::v28, FEXCore::ARMEmitter::VReg::v29, + FEXCore::ARMEmitter::VReg::v30, FEXCore::ARMEmitter::VReg::v31 + }; +} // We want vixl to not allocate a default buffer. Jit and dispatcher will manually create one. Arm64Emitter::Arm64Emitter(FEXCore::Context::ContextImpl *ctx, size_t size) @@ -29,22 +154,21 @@ Arm64Emitter::Arm64Emitter(FEXCore::Context::ContextImpl *ctx, size_t size) // Number of register available is dependent on what operating mode the proccess is in. if (EmitterCTX->Config.Is64BitMode()) { - ConfiguredGPRs = NumGPRs64; - ConfiguredSRAGPRs = NumSRAGPRs64; - ConfiguredGPRPairs = NumGPRPairs64; - ConfiguredFPRs = NumFPRs64; - ConfiguredSRAFPRs = NumSRAFPRs64; - ConfiguredDynamicGPRs = NumGPRs64 - NumGPRs64; // Will be zero, just to be consistent with 32-bit side - ConfiguredDynamicRegisterBase = nullptr; + StaticRegisters = x64::SRA; + GeneralRegisters = x64::RA; + GeneralPairRegisters = x64::RAPair; + StaticFPRegisters = x64::SRAFPR; + GeneralFPRegisters = x64::RAFPR; } else { - ConfiguredGPRs = NumGPRs32; - ConfiguredSRAGPRs = NumSRAGPRs32; - ConfiguredGPRPairs = NumGPRPairs32; - ConfiguredFPRs = NumFPRs32; - ConfiguredSRAFPRs = NumSRAFPRs32; - ConfiguredDynamicGPRs = NumGPRs32 - NumGPRs64; // Will be 8 - ConfiguredDynamicRegisterBase = &RA64[9]; + ConfiguredDynamicRegisterBase = std::span(x32::RA.begin() + 8, 8); + + StaticRegisters = x32::SRA; + GeneralRegisters = x32::RA; + GeneralPairRegisters = x32::RAPair; + + StaticFPRegisters = x32::SRAFPR; + GeneralFPRegisters = x32::RAFPR; } } @@ -224,9 +348,9 @@ void Arm64Emitter::SpillStaticRegs(FEXCore::ARMEmitter::Register TmpReg, bool FP return; } - for (size_t i = 0; i < ConfiguredSRAGPRs; i+=2) { - auto Reg1 = SRA64[i]; - auto Reg2 = SRA64[i+1]; + for (size_t i = 0; i < StaticRegisters.size(); i+=2) { + auto Reg1 = StaticRegisters[i]; + auto Reg2 = StaticRegisters[i+1]; if (((1U << Reg1.Idx()) & GPRSpillMask) && ((1U << Reg2.Idx()) & GPRSpillMask)) { stp(Reg1.X(), Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])); @@ -241,8 +365,8 @@ void Arm64Emitter::SpillStaticRegs(FEXCore::ARMEmitter::Register TmpReg, bool FP if (FPRs) { if (EmitterCTX->HostFeatures.SupportsAVX) { - for (size_t i = 0; i < ConfiguredSRAFPRs; i++) { - const auto Reg = SRAFPR[i]; + for (size_t i = 0; i < StaticFPRegisters.size(); i++) { + const auto Reg = StaticFPRegisters[i]; if (((1U << Reg.Idx()) & FPRSpillMask) != 0) { mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0])); @@ -254,18 +378,18 @@ void Arm64Emitter::SpillStaticRegs(FEXCore::ARMEmitter::Register TmpReg, bool FP // Optimize the common case where we can spill four registers per instruction // Load the sse offset in to the temporary register add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0])); - for (size_t i = 0; i < ConfiguredSRAFPRs; i += 4) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; - const auto Reg3 = SRAFPR[i + 2]; - const auto Reg4 = SRAFPR[i + 3]; + for (size_t i = 0; i < StaticFPRegisters.size(); i += 4) { + const auto Reg1 = StaticFPRegisters[i]; + const auto Reg2 = StaticFPRegisters[i + 1]; + const auto Reg3 = StaticFPRegisters[i + 2]; + const auto Reg4 = StaticFPRegisters[i + 3]; st1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64); } } else { - for (size_t i = 0; i < ConfiguredSRAFPRs; i += 2) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; + for (size_t i = 0; i < StaticFPRegisters.size(); i += 2) { + const auto Reg1 = StaticFPRegisters[i]; + const auto Reg2 = StaticFPRegisters[i + 1]; if (((1U << Reg1.Idx()) & FPRSpillMask) && ((1U << Reg2.Idx()) & FPRSpillMask)) { @@ -297,8 +421,8 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF ptrue(PRED_TMP_16B, ARMEmitter::PredicatePattern::SVE_VL16); ptrue(PRED_TMP_32B, ARMEmitter::PredicatePattern::SVE_VL32); - for (size_t i = 0; i < ConfiguredSRAFPRs; i++) { - const auto Reg = SRAFPR[i]; + for (size_t i = 0; i < StaticFPRegisters.size(); i++) { + const auto Reg = StaticFPRegisters[i]; if (((1U << Reg.Idx()) & FPRFillMask) != 0) { mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0])); ld1b(Reg.Z(), PRED_TMP_32B.Zeroing(), STATE.R(), TMP4.R()); @@ -308,22 +432,22 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF if (GPRFillMask && FPRFillMask == ~0U) { // Optimize the common case where we can fill four registers per instruction. // Use one of the filling static registers before we fill it. - auto TmpReg = SRA64[FindFirstSetBit(GPRFillMask)]; + auto TmpReg = StaticRegisters[FindFirstSetBit(GPRFillMask)]; // Load the sse offset in to the temporary register add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0])); - for (size_t i = 0; i < ConfiguredSRAFPRs; i += 4) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; - const auto Reg3 = SRAFPR[i + 2]; - const auto Reg4 = SRAFPR[i + 3]; + for (size_t i = 0; i < StaticFPRegisters.size(); i += 4) { + const auto Reg1 = StaticFPRegisters[i]; + const auto Reg2 = StaticFPRegisters[i + 1]; + const auto Reg3 = StaticFPRegisters[i + 2]; + const auto Reg4 = StaticFPRegisters[i + 3]; ld1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64); } } else { - for (size_t i = 0; i < ConfiguredSRAFPRs; i += 2) { - const auto Reg1 = SRAFPR[i]; - const auto Reg2 = SRAFPR[i + 1]; + for (size_t i = 0; i < StaticFPRegisters.size(); i += 2) { + const auto Reg1 = StaticFPRegisters[i]; + const auto Reg2 = StaticFPRegisters[i + 1]; if (((1U << Reg1.Idx()) & FPRFillMask) && ((1U << Reg2.Idx()) & FPRFillMask)) { @@ -340,9 +464,9 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF } } - for (size_t i = 0; i < ConfiguredSRAGPRs; i+=2) { - auto Reg1 = SRA64[i]; - auto Reg2 = SRA64[i+1]; + for (size_t i = 0; i < StaticRegisters.size(); i+=2) { + auto Reg1 = StaticRegisters[i]; + auto Reg2 = StaticRegisters[i+1]; if (((1U << Reg1.Idx()) & GPRFillMask) && ((1U << Reg2.Idx()) & GPRFillMask)) { ldp(Reg1.X(), Reg2.X(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])); @@ -358,10 +482,10 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF void Arm64Emitter::PushDynamicRegsAndLR(FEXCore::ARMEmitter::Register TmpReg) { const auto CanUseSVE = EmitterCTX->HostFeatures.SupportsAVX; - const auto GPRSize = (ConfiguredDynamicGPRs + 1) * Core::CPUState::GPR_REG_SIZE; + const auto GPRSize = (ConfiguredDynamicRegisterBase.size() + 1) * Core::CPUState::GPR_REG_SIZE; const auto FPRRegSize = CanUseSVE ? Core::CPUState::XMM_AVX_REG_SIZE : Core::CPUState::XMM_SSE_REG_SIZE; - const auto FPRSize = ConfiguredFPRs * FPRRegSize; + const auto FPRSize = GeneralFPRegisters.size() * FPRRegSize; const uint64_t SPOffset = AlignUp(GPRSize + FPRSize, 16); sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset); @@ -370,31 +494,29 @@ void Arm64Emitter::PushDynamicRegsAndLR(FEXCore::ARMEmitter::Register TmpReg) { add(ARMEmitter::Size::i64Bit, TmpReg, ARMEmitter::Reg::rsp, 0); if (CanUseSVE) { - for (size_t i = 0; i < ConfiguredFPRs; i += 4) { - const auto Reg1 = RAFPR[i]; - const auto Reg2 = RAFPR[i + 1]; - const auto Reg3 = RAFPR[i + 2]; - const auto Reg4 = RAFPR[i + 3]; + for (size_t i = 0; i < GeneralFPRegisters.size(); i += 4) { + const auto Reg1 = GeneralFPRegisters[i]; + const auto Reg2 = GeneralFPRegisters[i + 1]; + const auto Reg3 = GeneralFPRegisters[i + 2]; + const auto Reg4 = GeneralFPRegisters[i + 3]; st4b(Reg1.Z(), Reg2.Z(), Reg3.Z(), Reg4.Z(), PRED_TMP_32B, TmpReg, 0); add(ARMEmitter::Size::i64Bit, TmpReg, TmpReg, 32 * 4); } } else { - LOGMAN_THROW_AA_FMT(ConfiguredFPRs % 4 == 0, "Needs to have multiple of 4 FPRs for RA"); - for (size_t i = 0; i < ConfiguredFPRs; i += 4) { - const auto Reg1 = RAFPR[i]; - const auto Reg2 = RAFPR[i + 1]; - const auto Reg3 = RAFPR[i + 2]; - const auto Reg4 = RAFPR[i + 3]; + LOGMAN_THROW_A_FMT(GeneralFPRegisters.size() % 4 == 0, "Needs to have multiple of 4 FPRs for RA"); + for (size_t i = 0; i < GeneralFPRegisters.size(); i += 4) { + const auto Reg1 = GeneralFPRegisters[i]; + const auto Reg2 = GeneralFPRegisters[i + 1]; + const auto Reg3 = GeneralFPRegisters[i + 2]; + const auto Reg4 = GeneralFPRegisters[i + 3]; st1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), TmpReg, 64); } } - if (ConfiguredDynamicRegisterBase) { - for (size_t i = 0; i < ConfiguredDynamicGPRs; i += 2) { - const auto Reg1 = ConfiguredDynamicRegisterBase[i]; - const auto Reg2 = ConfiguredDynamicRegisterBase[i + 1]; - stp(Reg1.X(), Reg2.X(), TmpReg, 16); - } + for (size_t i = 0; i < ConfiguredDynamicRegisterBase.size(); i += 2) { + const auto Reg1 = ConfiguredDynamicRegisterBase[i]; + const auto Reg2 = ConfiguredDynamicRegisterBase[i + 1]; + stp(Reg1.X(), Reg2.X(), TmpReg, 16); } str(ARMEmitter::XReg::lr, TmpReg, 0); @@ -404,30 +526,28 @@ void Arm64Emitter::PopDynamicRegsAndLR() { const auto CanUseSVE = EmitterCTX->HostFeatures.SupportsAVX; if (CanUseSVE) { - for (size_t i = 0; i < ConfiguredFPRs; i += 4) { - const auto Reg1 = RAFPR[i]; - const auto Reg2 = RAFPR[i + 1]; - const auto Reg3 = RAFPR[i + 2]; - const auto Reg4 = RAFPR[i + 3]; + for (size_t i = 0; i < GeneralFPRegisters.size(); i += 4) { + const auto Reg1 = GeneralFPRegisters[i]; + const auto Reg2 = GeneralFPRegisters[i + 1]; + const auto Reg3 = GeneralFPRegisters[i + 2]; + const auto Reg4 = GeneralFPRegisters[i + 3]; ld4b(Reg1.Z(), Reg2.Z(), Reg3.Z(), Reg4.Z(), PRED_TMP_32B.Zeroing(), ARMEmitter::Reg::rsp); add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 32 * 4); } } else { - for (size_t i = 0; i < ConfiguredFPRs; i += 4) { - const auto Reg1 = RAFPR[i]; - const auto Reg2 = RAFPR[i + 1]; - const auto Reg3 = RAFPR[i + 2]; - const auto Reg4 = RAFPR[i + 3]; + for (size_t i = 0; i < GeneralFPRegisters.size(); i += 4) { + const auto Reg1 = GeneralFPRegisters[i]; + const auto Reg2 = GeneralFPRegisters[i + 1]; + const auto Reg3 = GeneralFPRegisters[i + 2]; + const auto Reg4 = GeneralFPRegisters[i + 3]; ld1(Reg1.Q(), Reg2.Q(), Reg3.Q(), Reg4.Q(), ARMEmitter::Reg::rsp, 64); } } - if (ConfiguredDynamicRegisterBase) { - for (size_t i = 0; i < ConfiguredDynamicGPRs; i += 2) { - const auto Reg1 = ConfiguredDynamicRegisterBase[i]; - const auto Reg2 = ConfiguredDynamicRegisterBase[i + 1]; - ldp(Reg1.X(), Reg2.X(), ARMEmitter::Reg::rsp, 16); - } + for (size_t i = 0; i < ConfiguredDynamicRegisterBase.size(); i += 2) { + const auto Reg1 = ConfiguredDynamicRegisterBase[i]; + const auto Reg2 = ConfiguredDynamicRegisterBase[i + 1]; + ldp(Reg1.X(), Reg2.X(), ARMEmitter::Reg::rsp, 16); } ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); diff --git a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h index f1a98f1d7..4a3a45cc6 100644 --- a/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h +++ b/External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h @@ -26,105 +26,9 @@ #include #include #include +#include namespace FEXCore::CPU { -// Register x18 is unused in the current configuration. -// This is due to it being a platform register on wine platforms. -// TODO: Allow x18 register allocation in the future to gain one more register. - -// All but x19 and x29 are caller saved -constexpr std::array SRA64 = { - FEXCore::ARMEmitter::Reg::r4, FEXCore::ARMEmitter::Reg::r5, - FEXCore::ARMEmitter::Reg::r6, FEXCore::ARMEmitter::Reg::r7, - FEXCore::ARMEmitter::Reg::r8, FEXCore::ARMEmitter::Reg::r9, - FEXCore::ARMEmitter::Reg::r10, FEXCore::ARMEmitter::Reg::r11, - // Registers that don't exist on 32-bit - FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r13, - FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r15, - FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r17, - FEXCore::ARMEmitter::Reg::r19, FEXCore::ARMEmitter::Reg::r29 -}; - -constexpr std::array RA64 = { - // All these callee saved - FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21, - FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23, - FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25, - FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27, - FEXCore::ARMEmitter::Reg::r30, - - // Registers only available on 32-bit - // All these are caller saved (except for r19). - FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r13, - FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r15, - FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r17, - FEXCore::ARMEmitter::Reg::r19, FEXCore::ARMEmitter::Reg::r29 -}; - -constexpr std::array, 4 + 3> RA64Pair = {{ - {FEXCore::ARMEmitter::Reg::r20, FEXCore::ARMEmitter::Reg::r21}, - {FEXCore::ARMEmitter::Reg::r22, FEXCore::ARMEmitter::Reg::r23}, - {FEXCore::ARMEmitter::Reg::r24, FEXCore::ARMEmitter::Reg::r25}, - {FEXCore::ARMEmitter::Reg::r26, FEXCore::ARMEmitter::Reg::r27}, - - // Registers only available on 32-bit - {FEXCore::ARMEmitter::Reg::r12, FEXCore::ARMEmitter::Reg::r13}, - {FEXCore::ARMEmitter::Reg::r14, FEXCore::ARMEmitter::Reg::r15}, - {FEXCore::ARMEmitter::Reg::r16, FEXCore::ARMEmitter::Reg::r17} -}}; - -/** - * @brief These are the intersection indexes for RA64Pair to RA64. - * - * Since RA64 sticks a register right in the middle of the allocation, this intersection offsets by one half way through. - * Keep these intersection indexes nearby the definitions so they can follow the previous definitions. - */ -constexpr std::array, RA64Pair.size()> RA64Pair_Intersections = {{ - {0, 1}, - {2, 3}, - {4, 5}, - {6, 7}, - - // Registers only available on 32-bit - {9, 10}, - {11, 12}, - {13, 14} -}}; - -// All are caller saved -constexpr std::array SRAFPR = { - FEXCore::ARMEmitter::VReg::v16, FEXCore::ARMEmitter::VReg::v17, - FEXCore::ARMEmitter::VReg::v18, FEXCore::ARMEmitter::VReg::v19, - FEXCore::ARMEmitter::VReg::v20, FEXCore::ARMEmitter::VReg::v21, - FEXCore::ARMEmitter::VReg::v22, FEXCore::ARMEmitter::VReg::v23, - - // Registers that don't exist on 32-bit - FEXCore::ARMEmitter::VReg::v24, FEXCore::ARMEmitter::VReg::v25, - FEXCore::ARMEmitter::VReg::v26, FEXCore::ARMEmitter::VReg::v27, - FEXCore::ARMEmitter::VReg::v28, FEXCore::ARMEmitter::VReg::v29, - FEXCore::ARMEmitter::VReg::v30, FEXCore::ARMEmitter::VReg::v31 -}; - -// v8..v15 = (lower 64bits) Callee saved -constexpr std::array RAFPR = { - // v0 ~ v3 are used as temps. - // FEXCore::ARMEmitter::VReg::v0, FEXCore::ARMEmitter::VReg::v1, - // FEXCore::ARMEmitter::VReg::v2, FEXCore::ARMEmitter::VReg::v3, - - FEXCore::ARMEmitter::VReg::v4, FEXCore::ARMEmitter::VReg::v5, - FEXCore::ARMEmitter::VReg::v6, FEXCore::ARMEmitter::VReg::v7, - FEXCore::ARMEmitter::VReg::v8, FEXCore::ARMEmitter::VReg::v9, - FEXCore::ARMEmitter::VReg::v10, FEXCore::ARMEmitter::VReg::v11, - FEXCore::ARMEmitter::VReg::v12, FEXCore::ARMEmitter::VReg::v13, - FEXCore::ARMEmitter::VReg::v14, FEXCore::ARMEmitter::VReg::v15, - - // Registers only available on 32-bit - FEXCore::ARMEmitter::VReg::v24, FEXCore::ARMEmitter::VReg::v25, - FEXCore::ARMEmitter::VReg::v26, FEXCore::ARMEmitter::VReg::v27, - FEXCore::ARMEmitter::VReg::v28, FEXCore::ARMEmitter::VReg::v29, - FEXCore::ARMEmitter::VReg::v30, FEXCore::ARMEmitter::VReg::v31 -}; - // Contains the address to the currently available CPU state constexpr auto STATE = FEXCore::ARMEmitter::XReg::x28; @@ -157,32 +61,16 @@ protected: FEXCore::Context::ContextImpl *EmitterCTX; vixl::aarch64::CPU CPU; - uint32_t ConfiguredGPRs; - uint32_t ConfiguredSRAGPRs; - uint32_t ConfiguredGPRPairs; - uint32_t ConfiguredFPRs; - uint32_t ConfiguredSRAFPRs; - uint32_t ConfiguredDynamicGPRs; - const FEXCore::ARMEmitter::Register *ConfiguredDynamicRegisterBase{}; + std::span ConfiguredDynamicRegisterBase{}; + std::span StaticRegisters{}; + std::span GeneralRegisters{}; + std::span> GeneralPairRegisters{}; + std::span StaticFPRegisters{}; + std::span GeneralFPRegisters{}; /** * @name Register Allocation * @{ */ - // 64-bit gets removal of additional pairs - constexpr static uint32_t NumGPRs64 = RA64.size() - 8; - constexpr static uint32_t NumSRAGPRs64 = SRA64.size(); - constexpr static uint32_t NumFPRs64 = RAFPR.size() - 8; - constexpr static uint32_t NumSRAFPRs64 = SRAFPR.size(); - constexpr static uint32_t NumGPRPairs64 = RA64Pair.size() - 3; - - // 32-bit gets full array of GPR registers - // SRA registers remove the additional 8 - constexpr static uint32_t NumGPRs32 = RA64.size(); - constexpr static uint32_t NumSRAGPRs32 = SRA64.size() - 8; - constexpr static uint32_t NumFPRs32 = RAFPR.size(); - constexpr static uint32_t NumSRAFPRs32 = SRAFPR.size() - 8; - constexpr static uint32_t NumGPRPairs32 = RA64Pair.size(); - constexpr static uint32_t RegisterClasses = 6; constexpr static uint64_t GPRBase = (0ULL << 32); diff --git a/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h b/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h index fe071f927..bd63b40fc 100644 --- a/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h +++ b/External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h @@ -31,22 +31,22 @@ class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter { void EmitDispatcher(); uint16_t GetSRAGPRCount() const override { - return SRA64.size(); + return StaticRegisters.size(); } uint16_t GetSRAFPRCount() const override { - return SRAFPR.size(); + return StaticFPRegisters.size(); } void GetSRAGPRMapping(uint8_t Mapping[16]) const override { - for (size_t i = 0; i < SRA64.size(); ++i) { - Mapping[i] = SRA64[i].Idx(); + for (size_t i = 0; i < StaticRegisters.size(); ++i) { + Mapping[i] = StaticRegisters[i].Idx(); } } void GetSRAFPRMapping(uint8_t Mapping[16]) const override { - for (size_t i = 0; i < SRAFPR.size(); ++i) { - Mapping[i] = SRAFPR[i].Idx(); + for (size_t i = 0; i < StaticFPRegisters.size(); ++i) { + Mapping[i] = StaticFPRegisters[i].Idx(); } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp index 37e2d3cff..40692c84f 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp @@ -477,9 +477,9 @@ DEF_OP(PDep) { const auto IndexReg = TMP4.R(); const auto ZeroReg = ARMEmitter::Reg::zr; - const auto InputReg = SRA64[0]; - const auto MaskReg = SRA64[1]; - const auto DestReg = SRA64[2]; + const auto InputReg = StaticRegisters[0]; + const auto MaskReg = StaticRegisters[1]; + const auto DestReg = StaticRegisters[2]; const auto SpillCode = 1U << InputReg.Idx() | 1U << MaskReg.Idx() | diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index 55fab3347..911127160 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -587,17 +587,16 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::In RAPass->AllocateRegisterSet(RegisterClasses); - RAPass->AddRegisters(FEXCore::IR::GPRClass, ConfiguredGPRs); - RAPass->AddRegisters(FEXCore::IR::GPRFixedClass, ConfiguredSRAGPRs); - RAPass->AddRegisters(FEXCore::IR::FPRClass, ConfiguredFPRs); - RAPass->AddRegisters(FEXCore::IR::FPRFixedClass, ConfiguredSRAFPRs); - RAPass->AddRegisters(FEXCore::IR::GPRPairClass, ConfiguredGPRPairs); + RAPass->AddRegisters(FEXCore::IR::GPRClass, GeneralRegisters.size()); + RAPass->AddRegisters(FEXCore::IR::GPRFixedClass, StaticRegisters.size()); + RAPass->AddRegisters(FEXCore::IR::FPRClass, GeneralFPRegisters.size()); + RAPass->AddRegisters(FEXCore::IR::FPRFixedClass, StaticFPRegisters.size()); + RAPass->AddRegisters(FEXCore::IR::GPRPairClass, GeneralPairRegisters.size()); RAPass->AddRegisters(FEXCore::IR::ComplexClass, 1); - for (uint32_t i = 0; i < ConfiguredGPRPairs; ++i) { - const auto Intersect = RA64Pair_Intersections[i]; - RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, Intersect.first, FEXCore::IR::GPRPairClass, i); - RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, Intersect.second, FEXCore::IR::GPRPairClass, i); + for (uint32_t i = 0; i < GeneralPairRegisters.size(); ++i) { + RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2, FEXCore::IR::GPRPairClass, i); + RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i); } { diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index 2079b8bab..65da24582 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -70,9 +70,9 @@ private: LOGMAN_THROW_AA_FMT(Reg.Class == IR::GPRFixedClass.Val || Reg.Class == IR::GPRClass.Val, "Unexpected Class: {}", Reg.Class); if (Reg.Class == IR::GPRFixedClass.Val) { - return SRA64[Reg.Reg]; + return StaticRegisters[Reg.Reg]; } else if (Reg.Class == IR::GPRClass.Val) { - return RA64[Reg.Reg]; + return GeneralRegisters[Reg.Reg]; } FEX_UNREACHABLE; @@ -84,9 +84,9 @@ private: LOGMAN_THROW_AA_FMT(Reg.Class == IR::FPRFixedClass.Val || Reg.Class == IR::FPRClass.Val, "Unexpected Class: {}", Reg.Class); if (Reg.Class == IR::FPRFixedClass.Val) { - return SRAFPR[Reg.Reg]; + return StaticFPRegisters[Reg.Reg]; } else if (Reg.Class == IR::FPRClass.Val) { - return RAFPR[Reg.Reg]; + return GeneralFPRegisters[Reg.Reg]; } FEX_UNREACHABLE; @@ -97,7 +97,7 @@ private: LOGMAN_THROW_AA_FMT(Reg.Class == IR::GPRPairClass.Val, "Unexpected Class: {}", Reg.Class); - return RA64Pair[Reg.Reg]; + return GeneralPairRegisters[Reg.Reg]; } [[nodiscard]] FEXCore::IR::RegisterClassType GetRegClass(IR::NodeID Node) const; diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp index f3176ea25..16aa21baf 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp @@ -132,7 +132,7 @@ DEF_OP(LoadRegister) { [[maybe_unused]] const auto regId = (Op->Offset / Core::CPUState::GPR_REG_SIZE) - 1; const auto regOffs = Op->Offset & 7; - LOGMAN_THROW_A_FMT(regId < SRA64.size(), "out of range regId"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "out of range regId"); switch (OpSize) { case 1: @@ -166,7 +166,7 @@ DEF_OP(LoadRegister) { [[maybe_unused]] const auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.xmm.avx.data[0][0])) / regSize; LOGMAN_THROW_A_FMT(HostSupportsSVE, "Unsupported code path!"); - LOGMAN_THROW_A_FMT(regId < SRAFPR.size(), "out of range regId"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "out of range regId"); const auto host = GetVReg(Node); @@ -214,7 +214,7 @@ DEF_OP(StoreRegister) { [[maybe_unused]] const auto regId = (Op->Offset / Core::CPUState::GPR_REG_SIZE) - 1; const auto regOffs = Op->Offset & 7; - LOGMAN_THROW_A_FMT(regId < SRA64.size(), "out of range regId"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "out of range regId"); const auto Src = GetReg(Op->Value.ID()); @@ -248,7 +248,7 @@ DEF_OP(StoreRegister) { [[maybe_unused]] const auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.xmm.avx.data[0][0])) / regSize; LOGMAN_THROW_A_FMT(HostSupportsSVE, "Unsupported code path!"); - LOGMAN_THROW_A_FMT(regId < SRAFPR.size(), "regId out of range"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "regId out of range"); const auto host = GetVReg(Op->Value.ID()); @@ -296,9 +296,9 @@ DEF_OP(LoadRegisterSRA) { const auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.gregs[0])) / Core::CPUState::GPR_REG_SIZE; const auto regOffs = Op->Offset & 7; - LOGMAN_THROW_A_FMT(regId < SRA64.size(), "out of range regId"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "out of range regId"); - const auto reg = SRA64[regId]; + const auto reg = StaticRegisters[regId]; switch (OpSize) { case 1: @@ -334,9 +334,9 @@ DEF_OP(LoadRegisterSRA) { : Core::CPUState::XMM_SSE_REG_SIZE; const auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.xmm.avx.data[0][0])) / regSize; - LOGMAN_THROW_A_FMT(regId < SRAFPR.size(), "out of range regId"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "out of range regId"); - const auto guest = SRAFPR[regId]; + const auto guest = StaticFPRegisters[regId]; const auto host = GetVReg(Node); if (HostSupportsSVE) { @@ -484,9 +484,9 @@ DEF_OP(StoreRegisterSRA) { const auto regId = (Op->Offset / Core::CPUState::GPR_REG_SIZE) - 1; const auto regOffs = Op->Offset & 7; - LOGMAN_THROW_A_FMT(regId < SRA64.size(), "out of range regId"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "out of range regId"); - const auto reg = SRA64[regId]; + const auto reg = StaticRegisters[regId]; const auto Src = GetReg(Op->Value.ID()); switch (OpSize) { @@ -520,9 +520,9 @@ DEF_OP(StoreRegisterSRA) { : Core::CPUState::XMM_SSE_REG_SIZE; const auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.xmm.avx.data[0][0])) / regSize; - LOGMAN_THROW_A_FMT(regId < SRAFPR.size(), "regId out of range"); + LOGMAN_THROW_A_FMT(regId < StaticFPRegisters.size(), "regId out of range"); - const auto guest = SRAFPR[regId]; + const auto guest = StaticFPRegisters[regId]; const auto host = GetVReg(Op->Value.ID()); if (HostSupportsSVE) {