diff --git a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp index 44d0819a1..4a7fcdb98 100644 --- a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp +++ b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +#include "Common/SoftFloat.h" #include "Interface/Context/Context.h" #include "Interface/Core/Dispatcher/Dispatcher.h" #include "Interface/Core/LookupCache.h" @@ -30,7 +31,7 @@ static void SleepThread(FEXCore::Context::ContextImpl* CTX, FEXCore::Core::CpuSt CTX->SyscallHandler->SleepThread(CTX, Frame); } -constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096 * 2; +constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096 * 4; Dispatcher::Dispatcher(FEXCore::Context::ContextImpl* ctx) : Arm64Emitter(ctx, FEXCore::Allocator::VirtualAlloc(MAX_DISPATCHER_CODE_SIZE, true), MAX_DISPATCHER_CODE_SIZE) @@ -541,6 +542,32 @@ void Dispatcher::EmitDispatcher() { LUREMHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM)); LREMHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM)); + // Interpreter fallbacks + { + constexpr static std::array ABIS {{ + FABI_F80_I16_F32_PTR, + FABI_F80_I16_F64_PTR, + FABI_F80_I16_I16_PTR, + FABI_F80_I16_I32_PTR, + FABI_F32_I16_F80_PTR, + FABI_F64_I16_F80_PTR, + FABI_F64_I16_F64_PTR, + FABI_F64_I16_F64_F64_PTR, + FABI_I16_I16_F80_PTR, + FABI_I32_I16_F80_PTR, + FABI_I64_I16_F80_PTR, + FABI_I64_I16_F80_F80_PTR, + FABI_F80_I16_F80_PTR, + FABI_F80_I16_F80_F80_PTR, + FABI_I32_I64_I64_V128_V128_I16, + FABI_I32_V128_V128_I16, + }}; + + for (auto ABI : ABIS) { + ABIPointers[ABI] = GenerateABICall(ABI); + } + } + Bind(&l_CTX); dc64(reinterpret_cast(CTX)); Bind(&l_Sleep); @@ -590,6 +617,404 @@ void Dispatcher::ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_ #endif +uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) { + auto Address = GetCursorAddress(); + constexpr static auto FallbackPointerReg = TMP4; + constexpr static auto ABI1 = ARMEmitter::XReg::x0; + constexpr static auto ABI2 = ARMEmitter::XReg::x1; + constexpr static auto ABI3 = ARMEmitter::XReg::x2; + + constexpr static auto VABI1 = ARMEmitter::VReg::v0; + constexpr static auto VABI2 = ARMEmitter::VReg::v1; + + auto FillF80Result = [&]() { + if (VTMP1 != VABI1) { + mov(VTMP1.Q(), VABI1.Q()); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + auto FillF64Result = [&]() { + if (!TMP_ABIARGS) { + fmov(VTMP1.D(), VABI1.D()); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + auto FillF32Result = [&]() { + if (!TMP_ABIARGS) { + fmov(VTMP1.S(), VABI1.S()); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + auto FillI64Result = [&]() { + if (!TMP_ABIARGS) { + mov(TMP1, ARMEmitter::XReg::x0); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + auto FillI32Result = [&]() { + if (!TMP_ABIARGS) { + mov(TMP1.W(), ARMEmitter::WReg::w0); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + auto FillI16Result = [&]() { + if (!TMP_ABIARGS) { + mov(TMP1, ARMEmitter::XReg::x0); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + switch (ABI) { + case FABI_F80_I16_F32_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + if (!TMP_ABIARGS) { + fmov(VABI1.S(), VTMP1.S()); + } + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF80Result(); + } break; + case FABI_F80_I16_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + if (!TMP_ABIARGS) { + fmov(VABI1.D(), VTMP1.D()); + } + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF80Result(); + } break; + case FABI_F80_I16_I16_PTR: + case FABI_F80_I16_I32_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // tmp2 (x1/x11): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + if (!TMP_ABIARGS) { + mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, TMP2); + } + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x2, STATE); + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF80Result(); + } break; + case FABI_F32_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + if (!TMP_ABIARGS) { + fmov(VABI1.D(), VTMP1.D()); + } + + mov(ARMEmitter::XReg::x1, STATE); + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF32Result(); + } break; + case FABI_F64_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + } + mov(ARMEmitter::XReg::x1, STATE); + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF64Result(); + } break; + case FABI_F64_I16_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + if (!TMP_ABIARGS) { + fmov(VABI1.D(), VTMP1.D()); + } + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF64Result(); + } break; + case FABI_F64_I16_F64_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + if (!TMP_ABIARGS) { + fmov(VABI1.D(), VTMP1.D()); + fmov(VABI2.D(), VTMP2.D()); + } + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF64Result(); + } break; + case FABI_I16_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + } + mov(ARMEmitter::XReg::x1, STATE); + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillI16Result(); + } break; + case FABI_I32_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + } + mov(ARMEmitter::XReg::x1, STATE); + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillI32Result(); + } break; + case FABI_I64_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + } + mov(ARMEmitter::XReg::x1, STATE); + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillI64Result(); + } break; + case FABI_I64_I16_F80_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + mov(VABI2.Q(), VTMP2.Q()); + } + mov(ARMEmitter::XReg::x1, STATE); + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillI64Result(); + } break; + case FABI_F80_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + } + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF80Result(); + } break; + case FABI_F80_I16_F80_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + mov(VABI2.Q(), VTMP2.Q()); + } + mov(ARMEmitter::XReg::x1, STATE); + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF80Result(); + } break; + case FABI_I32_I64_I64_V128_V128_I16: { + // Linux Reg/Win32 Reg: + // stack: FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + // tmp1 (x0/x10): source 1 + // tmp2 (x1/x11): source 2 + // tmp3 (x2/x12): source 3 + + const size_t OriginalSPOffset = SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP4, true); + + // Load the Fallback handler pointer from the stack. + ldr(FallbackPointerReg, ARMEmitter::XReg::rsp, OriginalSPOffset); + + if (!TMP_ABIARGS) { + mov(ABI1, TMP1); + mov(ABI2, TMP2); + mov(ABI3, TMP3); + mov(VABI1.Q(), VTMP1.Q()); + mov(VABI2.Q(), VTMP2.Q()); + } + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillI32Result(); + } break; + case FABI_I32_V128_V128_I16: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + // tmp1 (x0/x10): source 1 + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + mov(VABI2.Q(), VTMP2.Q()); + mov(ABI1, TMP1); + } + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillI32Result(); + } break; + case FABI_UNKNOWN: + default: +#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED + LOGMAN_MSG_A_FMT("Unhandled IR Fallback ABI: {}", ToUnderlying(ABI)); +#endif + break; + } + + // Return to JIT + ret(); + + return Address; +} + void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread) { // Setup dispatcher specific pointers that need to be accessed from JIT code { @@ -613,6 +1038,9 @@ void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread) AArch64.LDIVHandler = LDIVHandlerAddress; AArch64.LUREMHandler = LUREMHandlerAddress; AArch64.LREMHandler = LREMHandlerAddress; + + // Fill in the fallback handlers + InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers, &ABIPointers[0]); } } diff --git a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h index bef13815c..cdaad271b 100644 --- a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h +++ b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h @@ -3,7 +3,9 @@ #include "Interface/Core/ArchHelpers/Arm64Emitter.h" #include "Interface/Core/CPUBackend.h" +#include "Interface/Core/Interpreter/InterpreterOps.h" +#include #include #ifdef VIXL_SIMULATOR @@ -60,6 +62,7 @@ public: uint64_t IntCallbackReturnAddress {}; uint64_t PauseReturnInstruction {}; + std::array ABIPointers {}; /** @} */ @@ -119,6 +122,7 @@ private: uint64_t LREMHandlerAddress {}; void EmitDispatcher(); + uint64_t GenerateABICall(FallbackABI ABI); }; } // namespace FEXCore::CPU diff --git a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp index bbb7ffd1e..eee36c26d 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp +++ b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp @@ -12,72 +12,108 @@ namespace FEXCore::CPU { template static FallbackInfo GetFallbackInfo(R (*fn)(Args...), FEXCore::Core::FallbackHandlerIndex HandlerIndex) { - return {FABI_UNKNOWN, (void*)fn, HandlerIndex, false}; + return {FABI_UNKNOWN, HandlerIndex, false}; } template<> FallbackInfo GetFallbackInfo(double (*fn)(uint16_t, double, FEXCore::Core::CpuStateFrame*), FEXCore::Core::FallbackHandlerIndex HandlerIndex) { - return {FABI_F64_I16_F64_PTR, (void*)fn, HandlerIndex, false}; + return {FABI_F64_I16_F64_PTR, HandlerIndex, false}; } template<> FallbackInfo GetFallbackInfo(double (*fn)(uint16_t, double, double, FEXCore::Core::CpuStateFrame*), FEXCore::Core::FallbackHandlerIndex HandlerIndex) { - return {FABI_F64_I16_F64_F64_PTR, (void*)fn, HandlerIndex, false}; + return {FABI_F64_I16_F64_F64_PTR, HandlerIndex, false}; } -void InterpreterOps::FillFallbackIndexPointers(uint64_t* Info) { - Info[Core::OPINDEX_F80CVTTO_4] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4); - Info[Core::OPINDEX_F80CVTTO_8] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8); - Info[Core::OPINDEX_F80CVT_4] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4); - Info[Core::OPINDEX_F80CVT_8] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8); - Info[Core::OPINDEX_F80CVTINT_2] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2); - Info[Core::OPINDEX_F80CVTINT_4] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4); - Info[Core::OPINDEX_F80CVTINT_8] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8); - Info[Core::OPINDEX_F80CVTINT_TRUNC2] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2t); - Info[Core::OPINDEX_F80CVTINT_TRUNC4] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4t); - Info[Core::OPINDEX_F80CVTINT_TRUNC8] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8t); - Info[Core::OPINDEX_F80CMP] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80CVTTOINT_2] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2); - Info[Core::OPINDEX_F80CVTTOINT_4] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4); +void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint64_t* ABIHandlers) { + Info[Core::OPINDEX_F80CVTTO_4] = {ABIHandlers[FABI_F80_I16_F32_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)}; + Info[Core::OPINDEX_F80CVTTO_8] = {ABIHandlers[FABI_F80_I16_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8)}; + Info[Core::OPINDEX_F80CVT_4] = {ABIHandlers[FABI_F32_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)}; + Info[Core::OPINDEX_F80CVT_8] = {ABIHandlers[FABI_F64_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8)}; + Info[Core::OPINDEX_F80CVTINT_2] = {ABIHandlers[FABI_I16_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2)}; + Info[Core::OPINDEX_F80CVTINT_4] = {ABIHandlers[FABI_I32_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)}; + Info[Core::OPINDEX_F80CVTINT_8] = {ABIHandlers[FABI_I64_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8)}; + Info[Core::OPINDEX_F80CVTINT_TRUNC2] = {ABIHandlers[FABI_I16_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2t)}; + Info[Core::OPINDEX_F80CVTINT_TRUNC4] = {ABIHandlers[FABI_I32_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4t)}; + Info[Core::OPINDEX_F80CVTINT_TRUNC8] = {ABIHandlers[FABI_I64_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle8t)}; + Info[Core::OPINDEX_F80CMP] = {ABIHandlers[FABI_I64_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80CVTTOINT_2] = {ABIHandlers[FABI_F80_I16_I16_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle2)}; + Info[Core::OPINDEX_F80CVTTOINT_4] = {ABIHandlers[FABI_F80_I16_I32_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle4)}; // Unary - Info[Core::OPINDEX_F80ROUND] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80F2XM1] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80TAN] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80SQRT] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80SIN] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80COS] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80XTRACT_EXP] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80XTRACT_SIG] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80BCDSTORE] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80BCDLOAD] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); + Info[Core::OPINDEX_F80ROUND] = {ABIHandlers[FABI_F80_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80F2XM1] = {ABIHandlers[FABI_F80_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80TAN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80SQRT] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80SIN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80COS] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80XTRACT_EXP] = {ABIHandlers[FABI_F80_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80XTRACT_SIG] = {ABIHandlers[FABI_F80_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80BCDSTORE] = {ABIHandlers[FABI_F80_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80BCDLOAD] = {ABIHandlers[FABI_F80_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; // Binary - Info[Core::OPINDEX_F80ADD] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80SUB] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80MUL] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80DIV] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80FYL2X] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80ATAN] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80FPREM1] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80FPREM] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F80SCALE] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); + Info[Core::OPINDEX_F80ADD] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80SUB] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80MUL] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80DIV] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80FYL2X] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80ATAN] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80FPREM1] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80FPREM] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80SCALE] = {ABIHandlers[FABI_F80_I16_F80_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; - // Double Precision - Info[Core::OPINDEX_F64SIN] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64COS] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64TAN] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64ATAN] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64F2XM1] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64FYL2X] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64FPREM] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64FPREM1] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_F64SCALE] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); + // Double Precision Unary + Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_I16_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + + // Double Precision Binary + Info[Core::OPINDEX_F64ATAN] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64FPREM] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64FPREM1] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64FYL2X] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64SCALE] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; // SSE4.2 string instructions - Info[Core::OPINDEX_VPCMPESTRX] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); - Info[Core::OPINDEX_VPCMPISTRX] = reinterpret_cast(&FEXCore::CPU::OpHandlers::handle); + Info[Core::OPINDEX_VPCMPESTRX] = {ABIHandlers[FABI_I32_I64_I64_V128_V128_I16], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_VPCMPISTRX] = {ABIHandlers[FABI_I32_V128_V128_I16], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; } bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::IROp_Header* IROp, FallbackInfo* Info) { @@ -88,11 +124,11 @@ bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::I switch (Op->SrcSize) { case IR::OpSize::i32Bit: { - *Info = {FABI_F80_I16_F32_PTR, (void*)&FEXCore::CPU::OpHandlers::handle4, Core::OPINDEX_F80CVTTO_4, SupportsPreserveAllABI}; + *Info = {FABI_F80_I16_F32_PTR, Core::OPINDEX_F80CVTTO_4, SupportsPreserveAllABI}; return true; } case IR::OpSize::i64Bit: { - *Info = {FABI_F80_I16_F64_PTR, (void*)&FEXCore::CPU::OpHandlers::handle8, Core::OPINDEX_F80CVTTO_8, SupportsPreserveAllABI}; + *Info = {FABI_F80_I16_F64_PTR, Core::OPINDEX_F80CVTTO_8, SupportsPreserveAllABI}; return true; } default: LogMan::Msg::DFmt("Unhandled size: {}", OpSize); @@ -102,11 +138,11 @@ bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::I case IR::OP_F80CVT: { switch (OpSize) { case IR::OpSize::i32Bit: { - *Info = {FABI_F32_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle4, Core::OPINDEX_F80CVT_4, SupportsPreserveAllABI}; + *Info = {FABI_F32_I16_F80_PTR, Core::OPINDEX_F80CVT_4, SupportsPreserveAllABI}; return true; } case IR::OpSize::i64Bit: { - *Info = {FABI_F64_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle8, Core::OPINDEX_F80CVT_8, SupportsPreserveAllABI}; + *Info = {FABI_F64_I16_F80_PTR, Core::OPINDEX_F80CVT_8, SupportsPreserveAllABI}; return true; } default: LogMan::Msg::DFmt("Unhandled size: {}", OpSize); @@ -119,31 +155,25 @@ bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::I switch (OpSize) { case IR::OpSize::i16Bit: { if (Op->Truncate) { - *Info = {FABI_I16_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle2t, Core::OPINDEX_F80CVTINT_TRUNC2, - SupportsPreserveAllABI}; + *Info = {FABI_I16_I16_F80_PTR, Core::OPINDEX_F80CVTINT_TRUNC2, SupportsPreserveAllABI}; } else { - *Info = {FABI_I16_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle2, Core::OPINDEX_F80CVTINT_2, - SupportsPreserveAllABI}; + *Info = {FABI_I16_I16_F80_PTR, Core::OPINDEX_F80CVTINT_2, SupportsPreserveAllABI}; } return true; } case IR::OpSize::i32Bit: { if (Op->Truncate) { - *Info = {FABI_I32_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle4t, Core::OPINDEX_F80CVTINT_TRUNC4, - SupportsPreserveAllABI}; + *Info = {FABI_I32_I16_F80_PTR, Core::OPINDEX_F80CVTINT_TRUNC4, SupportsPreserveAllABI}; } else { - *Info = {FABI_I32_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle4, Core::OPINDEX_F80CVTINT_4, - SupportsPreserveAllABI}; + *Info = {FABI_I32_I16_F80_PTR, Core::OPINDEX_F80CVTINT_4, SupportsPreserveAllABI}; } return true; } case IR::OpSize::i64Bit: { if (Op->Truncate) { - *Info = {FABI_I64_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle8t, Core::OPINDEX_F80CVTINT_TRUNC8, - SupportsPreserveAllABI}; + *Info = {FABI_I64_I16_F80_PTR, Core::OPINDEX_F80CVTINT_TRUNC8, SupportsPreserveAllABI}; } else { - *Info = {FABI_I64_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle8, Core::OPINDEX_F80CVTINT_8, - SupportsPreserveAllABI}; + *Info = {FABI_I64_I16_F80_PTR, Core::OPINDEX_F80CVTINT_8, SupportsPreserveAllABI}; } return true; } @@ -152,8 +182,7 @@ bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::I break; } case IR::OP_F80CMP: { - *Info = {FABI_I64_I16_F80_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle, - (Core::FallbackHandlerIndex)(Core::OPINDEX_F80CMP), SupportsPreserveAllABI}; + *Info = {FABI_I64_I16_F80_F80_PTR, (Core::FallbackHandlerIndex)(Core::OPINDEX_F80CMP), SupportsPreserveAllABI}; return true; } @@ -162,13 +191,11 @@ bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::I switch (Op->SrcSize) { case IR::OpSize::i16Bit: { - *Info = {FABI_F80_I16_I16_PTR, (void*)&FEXCore::CPU::OpHandlers::handle2, Core::OPINDEX_F80CVTTOINT_2, - SupportsPreserveAllABI}; + *Info = {FABI_F80_I16_I16_PTR, Core::OPINDEX_F80CVTTOINT_2, SupportsPreserveAllABI}; return true; } case IR::OpSize::i32Bit: { - *Info = {FABI_F80_I16_I32_PTR, (void*)&FEXCore::CPU::OpHandlers::handle4, Core::OPINDEX_F80CVTTOINT_4, - SupportsPreserveAllABI}; + *Info = {FABI_F80_I16_I32_PTR, Core::OPINDEX_F80CVTTOINT_4, SupportsPreserveAllABI}; return true; } default: LogMan::Msg::DFmt("Unhandled size: {}", OpSize); @@ -176,16 +203,16 @@ bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::I break; } -#define COMMON_UNARY_X87_OP(OP) \ - case IR::OP_F80##OP: { \ - *Info = {FABI_F80_I16_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle, Core::OPINDEX_F80##OP, SupportsPreserveAllABI}; \ - return true; \ +#define COMMON_UNARY_X87_OP(OP) \ + case IR::OP_F80##OP: { \ + *Info = {FABI_F80_I16_F80_PTR, Core::OPINDEX_F80##OP, SupportsPreserveAllABI}; \ + return true; \ } -#define COMMON_BINARY_X87_OP(OP) \ - case IR::OP_F80##OP: { \ - *Info = {FABI_F80_I16_F80_F80_PTR, (void*)&FEXCore::CPU::OpHandlers::handle, Core::OPINDEX_F80##OP, SupportsPreserveAllABI}; \ - return true; \ +#define COMMON_BINARY_X87_OP(OP) \ + case IR::OP_F80##OP: { \ + *Info = {FABI_F80_I16_F80_F80_PTR, Core::OPINDEX_F80##OP, SupportsPreserveAllABI}; \ + return true; \ } #define COMMON_F64_OP(OP) \ @@ -231,13 +258,8 @@ bool InterpreterOps::GetFallbackHandler(bool SupportsPreserveAllABI, const IR::I COMMON_F64_OP(SCALE) // SSE4.2 Fallbacks - case IR::OP_VPCMPESTRX: - *Info = {FABI_I32_I64_I64_V128_V128_I16, (void*)&FEXCore::CPU::OpHandlers::handle, Core::OPINDEX_VPCMPESTRX, - SupportsPreserveAllABI}; - return true; - case IR::OP_VPCMPISTRX: - *Info = {FABI_I32_V128_V128_I16, (void*)&FEXCore::CPU::OpHandlers::handle, Core::OPINDEX_VPCMPISTRX, SupportsPreserveAllABI}; - return true; + case IR::OP_VPCMPESTRX: *Info = {FABI_I32_I64_I64_V128_V128_I16, Core::OPINDEX_VPCMPESTRX, SupportsPreserveAllABI}; return true; + case IR::OP_VPCMPISTRX: *Info = {FABI_I32_V128_V128_I16, Core::OPINDEX_VPCMPISTRX, SupportsPreserveAllABI}; return true; default: break; } diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index c82872439..c063d2ca9 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -13,7 +13,6 @@ struct IROp_Header; namespace FEXCore::CPU { enum FallbackABI { - FABI_UNKNOWN, FABI_F80_I16_F32_PTR, FABI_F80_I16_F64_PTR, FABI_F80_I16_I16_PTR, @@ -30,18 +29,17 @@ enum FallbackABI { FABI_F80_I16_F80_F80_PTR, FABI_I32_I64_I64_V128_V128_I16, FABI_I32_V128_V128_I16, + FABI_UNKNOWN, }; - struct FallbackInfo { FallbackABI ABI; - void* fn; FEXCore::Core::FallbackHandlerIndex HandlerIndex; bool SupportsPreserveAllABI; }; class InterpreterOps { public: - static void FillFallbackIndexPointers(uint64_t* Info); + static void FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint64_t* ABIHandlers); static bool GetFallbackHandler(bool SupportsPreserveAllABI, const IR::IROp_Header* IROp, FallbackInfo* Info); }; } // namespace FEXCore::CPU diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index b9160814a..e514e06ca 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -88,392 +88,345 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) { #endif } else { auto FillF80Result = [&]() { - if (!TMP_ABIARGS) { - mov(VTMP1.Q(), ARMEmitter::VReg::v0.Q()); - } - - FillForABICall(Info.SupportsPreserveAllABI, true); - const auto Dst = GetVReg(Node); mov(Dst.Q(), VTMP1.Q()); }; auto FillF64Result = [&]() { - if (!TMP_ABIARGS) { - mov(VTMP1.D(), ARMEmitter::DReg::d0); - } - FillForABICall(Info.SupportsPreserveAllABI, true); - const auto Dst = GetVReg(Node); - mov(Dst.D(), VTMP1.D()); + fmov(Dst.D(), VTMP1.D()); + }; + + auto FillF32Result = [&]() { + const auto Dst = GetVReg(Node); + fmov(Dst.S(), VTMP1.S()); + }; + + auto FillI64Result = [&]() { + const auto Dst = GetReg(Node); + mov(Dst.X(), TMP1); }; auto FillI32Result = [&]() { - if (!TMP_ABIARGS) { - mov(TMP1.W(), ARMEmitter::WReg::w0); - } - FillForABICall(Info.SupportsPreserveAllABI, true); + const auto Dst = GetReg(Node); + mov(Dst.W(), TMP1.W()); + }; + auto FillI16Result = [&]() { const auto Dst = GetReg(Node); mov(Dst.W(), TMP1.W()); }; switch (Info.ABI) { case FABI_F80_I16_F32_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); - fmov(ARMEmitter::SReg::s0, Src1.S()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::XReg::x1, STATE); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + fmov(VTMP1.S(), Src1.S()); + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF80Result(); } break; case FABI_F80_I16_F64_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); - mov(ARMEmitter::DReg::d0, Src1.D()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::XReg::x1, STATE); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + mov(VTMP1.D(), Src1.D()); + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF80Result(); } break; case FABI_F80_I16_I16_PTR: case FABI_F80_I16_I32_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // tmp2 (x1/x11): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetReg(IROp->Args[0].ID()); + + // Need to sign or zero extend this for the dispatcher handler. if (Info.ABI == FABI_F80_I16_I16_PTR) { - sxth(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r1, Src1); + sxth(ARMEmitter::Size::i32Bit, TMP2, Src1); } else { - mov(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r1, Src1); - } - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::XReg::x2, STATE); - ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); - } else { - blr(ARMEmitter::Reg::r3); + mov(ARMEmitter::Size::i32Bit, TMP2, Src1); } + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF80Result(); } break; case FABI_F32_I16_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(VTMP1.Q(), Src1.Q()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); - mov(ARMEmitter::XReg::x1, STATE); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } - - if (!TMP_ABIARGS) { - fmov(VTMP1.S(), ARMEmitter::SReg::s0); - } - FillForABICall(Info.SupportsPreserveAllABI, true); - - const auto Dst = GetVReg(Node); - fmov(Dst.S(), VTMP1.S()); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillF32Result(); } break; case FABI_F64_I16_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(VTMP1.Q(), Src1.Q()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::XReg::x1, STATE); - - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF64Result(); } break; case FABI_F64_I16_F64_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(VTMP1.D(), Src1.D()); - mov(ARMEmitter::DReg::d0, Src1.D()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::XReg::x1, STATE); - - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF64Result(); } break; case FABI_F64_I16_F64_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); + const auto Src1 = GetVReg(IROp->Args[0].ID()); const auto Src2 = GetVReg(IROp->Args[1].ID()); mov(VTMP1.D(), Src1.D()); mov(VTMP2.D(), Src2.D()); - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); - - if (!TMP_ABIARGS) { - mov(ARMEmitter::DReg::d0, VTMP1.D()); - mov(ARMEmitter::DReg::d1, VTMP2.D()); - } - - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::XReg::x1, STATE); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF64Result(); } break; case FABI_I16_I16_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(VTMP1.Q(), Src1.Q()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::XReg::x1, STATE); + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } - - if (!TMP_ABIARGS) { - mov(TMP1, ARMEmitter::XReg::x0); - } - FillForABICall(Info.SupportsPreserveAllABI, true); - - const auto Dst = GetReg(Node); - sxth(ARMEmitter::Size::i64Bit, Dst, TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillI16Result(); } break; + case FABI_I32_I16_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(VTMP1.Q(), Src1.Q()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::XReg::x1, STATE); - - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillI32Result(); } break; + case FABI_I64_I16_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): source + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(VTMP1.Q(), Src1.Q()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::XReg::x1, STATE); + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } - - if (!TMP_ABIARGS) { - mov(TMP1, ARMEmitter::XReg::x0); - } - FillForABICall(Info.SupportsPreserveAllABI, true); - - const auto Dst = GetReg(Node); - mov(ARMEmitter::Size::i64Bit, Dst, TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillI64Result(); } break; + case FABI_I64_I16_F80_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); const auto Src2 = GetVReg(IROp->Args[1].ID()); + mov(VTMP1.Q(), Src1.Q()); + mov(VTMP2.Q(), Src2.Q()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - if (!TMP_ABIARGS) { - mov(VTMP1.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - mov(ARMEmitter::VReg::v0.Q(), VTMP1.Q()); - } else { - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - } - mov(ARMEmitter::XReg::x1, STATE); + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } - - if (!TMP_ABIARGS) { - mov(TMP1, ARMEmitter::XReg::x0); - } - FillForABICall(Info.SupportsPreserveAllABI, true); - - const auto Dst = GetReg(Node); - mov(ARMEmitter::Size::i64Bit, Dst, TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillI64Result(); } break; + case FABI_F80_I16_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); + mov(VTMP1.Q(), Src1.Q()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - mov(ARMEmitter::XReg::x1, STATE); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF80Result(); } break; + case FABI_F80_I16_F80_F80_PTR: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(IROp->Args[0].ID()); const auto Src2 = GetVReg(IROp->Args[1].ID()); - ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); - if (!TMP_ABIARGS) { - mov(VTMP1.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - mov(ARMEmitter::VReg::v0.Q(), VTMP1.Q()); - } else { - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - } + mov(VTMP1.Q(), Src1.Q()); + mov(VTMP2.Q(), Src2.Q()); - mov(ARMEmitter::XReg::x1, STATE); - ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall( - ARMEmitter::Reg::r2); - } else { - blr(ARMEmitter::Reg::r2); - } + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF80Result(); } break; + case FABI_I32_I64_I64_V128_V128_I16: { + // Linux Reg/Win32 Reg: + // stack: FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + // tmp1 (x0/x10): source 1 + // tmp2 (x1/x11): source 2 + // tmp3 (x2/x12): source 3 const auto Op = IROp->C(); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + + stp(TMP1, ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); + const auto SrcRAX = GetReg(Op->RAX.ID()); const auto SrcRDX = GetReg(Op->RDX.ID()); + const auto Control = Op->Control; mov(TMP1, SrcRAX.X()); mov(TMP2, SrcRDX.X()); - - SpillForABICall(Info.SupportsPreserveAllABI, TMP3, true); - - const auto Control = Op->Control; + movz(ARMEmitter::Size::i32Bit, TMP3, Control); const auto Src1 = GetVReg(Op->LHS.ID()); const auto Src2 = GetVReg(Op->RHS.ID()); - if (!TMP_ABIARGS) { - mov(ARMEmitter::XReg::x0, TMP1); - mov(ARMEmitter::XReg::x1, TMP2); - } + mov(VTMP1.Q(), Src1.Q()); + mov(VTMP2.Q(), Src2.Q()); - if (!TMP_ABIARGS) { - mov(VTMP1.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - mov(ARMEmitter::VReg::v0.Q(), VTMP1.Q()); - } else { - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - } - - movz(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r2, Control); - - ldr(ARMEmitter::XReg::x3, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r3); - } else { - blr(ARMEmitter::Reg::r3); - } + blr(TMP4); + ldp(ARMEmitter::XReg::zr, ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillI32Result(); } break; case FABI_I32_V128_V128_I16: { - SpillForABICall(Info.SupportsPreserveAllABI, TMP1, true); - + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v17): vector source 2 + // tmp1 (x0/x10): source 1 const auto Op = IROp->C(); + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); const auto Src1 = GetVReg(Op->LHS.ID()); const auto Src2 = GetVReg(Op->RHS.ID()); const auto Control = Op->Control; - if (!TMP_ABIARGS) { - mov(VTMP1.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - mov(ARMEmitter::VReg::v0.Q(), VTMP1.Q()); - } else { - mov(ARMEmitter::VReg::v0.Q(), Src1.Q()); - mov(ARMEmitter::VReg::v1.Q(), Src2.Q()); - } + mov(VTMP1.Q(), Src1.Q()); + mov(VTMP2.Q(), Src2.Q()); + movz(ARMEmitter::Size::i32Bit, TMP1, Control); - movz(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Control); - - ldr(ARMEmitter::XReg::x1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])); - if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { - GenerateIndirectRuntimeCall(ARMEmitter::Reg::r1); - } else { - blr(ARMEmitter::Reg::r1); - } + ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP2); + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillI32Result(); } break; case FABI_UNKNOWN: @@ -588,9 +541,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In } Common.ExitFunctionLink = reinterpret_cast(&Context::ContextImpl::ThreadExitFunctionLink); - // Fill in the fallback handlers - InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers); - // Platform Specific auto& AArch64 = ThreadState->CurrentFrame->Pointers.AArch64; diff --git a/FEXCore/include/FEXCore/Core/CoreState.h b/FEXCore/include/FEXCore/Core/CoreState.h index e2da0a500..598d8abe1 100644 --- a/FEXCore/include/FEXCore/Core/CoreState.h +++ b/FEXCore/include/FEXCore/Core/CoreState.h @@ -244,6 +244,11 @@ enum FallbackHandlerIndex { OPINDEX_MAX, }; +struct FallbackABIInfo { + uint64_t ABIHandler; + uint64_t Func; +}; + struct JITPointers { struct { @@ -262,7 +267,7 @@ struct JITPointers { // Handles returning/calling ARM64EC code from the JIT, expects the target PC in TMP3 uint64_t ExitFunctionEC {}; - uint64_t FallbackHandlerPointers[FallbackHandlerIndex::OPINDEX_MAX]; + FallbackABIInfo FallbackHandlerPointers[FallbackHandlerIndex::OPINDEX_MAX]; uint64_t NamedVectorConstantPointers[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX]; uint64_t IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_MAX]; uint64_t TelemetryValueAddresses[FEXCore::Telemetry::TYPE_LAST];