diff --git a/FEXCore/Source/Common/VectorRegType.h b/FEXCore/Source/Common/VectorRegType.h index 8265832b6..f4209d48d 100644 --- a/FEXCore/Source/Common/VectorRegType.h +++ b/FEXCore/Source/Common/VectorRegType.h @@ -3,14 +3,32 @@ #ifdef _M_X86_64 #include +#include #endif namespace FEXCore { +struct VectorScalarF64Pair { + double val[2]; +}; + #ifdef _M_ARM_64 // Can't use uint8x16_t directly from arm_neon.h here. // Overrides softfloat-3e's defines which causes problems. using VectorRegType = __attribute__((neon_vector_type(16))) uint8_t; +struct VectorRegPairType { + VectorRegType val[2]; +}; + +static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) { + return VectorRegPairType {low, high}; +} + #elif defined(_M_X86_64) using VectorRegType = __m128i; +using VectorRegPairType = __m256i; + +static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) { + return _mm256_set_m128i(high, low); +} #endif } // namespace FEXCore diff --git a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp index a545e15f2..34b059700 100644 --- a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp +++ b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp @@ -558,6 +558,8 @@ void Dispatcher::EmitDispatcher() { FABI_I64_I16_F80_F80_PTR, FABI_F80_I16_F80_PTR, FABI_F80_I16_F80_F80_PTR, + FABI_F80x2_I16_F80_PTR, + FABI_F64x2_I16_F64_PTR, FABI_I32_I64_I64_V128_V128_I16, FABI_I32_V128_V128_I16, }}; @@ -626,6 +628,22 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) { constexpr static auto VABI1 = ARMEmitter::VReg::v0; constexpr static auto VABI2 = ARMEmitter::VReg::v1; + auto FillF80x2Result = [&]() { + if (!TMP_ABIARGS) { + mov(VTMP1.Q(), VABI1.Q()); + mov(VTMP2.Q(), VABI2.Q()); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + auto FillF64x2Result = [&]() { + if (!TMP_ABIARGS) { + fmov(VTMP1.D(), VABI1.D()); + fmov(VTMP2.D(), VABI2.D()); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + auto FillF80Result = [&]() { if (VTMP1 != VABI1) { mov(VTMP1.Q(), VABI1.Q()); @@ -946,6 +964,52 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) { FillF80Result(); } break; + case FABI_F80x2_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v16): vector source 2 + + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + } + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + // GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF80x2Result(); + } break; + case FABI_F64x2_I16_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v16): vector source 2 + + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!TMP_ABIARGS) { + fmov(VABI1.D(), VTMP1.D()); + } + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + // GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF64x2Result(); + } break; case FABI_I32_I64_I64_V128_V128_I16: { // Linux Reg/Win32 Reg: // stack: FallbackHandler diff --git a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h index d30b1d448..e35fb56b8 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h +++ b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h @@ -202,6 +202,15 @@ struct OpHandlers { } }; +template<> +struct OpHandlers { + FEXCORE_PRESERVE_ALL_ATTR static VectorRegPairType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) { + FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1); + softfloat_state State = SoftFloatStateFromFCW(FCW, true); + return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State, Src1), X80SoftFloat::FCOS(&State, Src1)); + } +}; + template<> struct OpHandlers { FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) { @@ -315,6 +324,21 @@ struct OpHandlers { } }; +template<> +struct OpHandlers { + FEXCORE_PRESERVE_ALL_ATTR static VectorScalarF64Pair handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) { + FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1); + double sin, cos; +#ifdef _WIN32 + sin = ::sin(src); + cos = ::cos(src); +#else + sincos(src, &sin, &cos); +#endif + return VectorScalarF64Pair {sin, cos}; + } +}; + template<> struct OpHandlers { FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) { diff --git a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp index 3f50867ce..21a29a8dc 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp +++ b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp @@ -50,6 +50,8 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint Info[Core::OPINDEX_F80SQRT] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80SIN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80COS] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80SINCOS] = {ABIHandlers[FABI_F80x2_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80XTRACT_EXP] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80XTRACT_SIG] = {ABIHandlers[FABI_F80_I16_F80_PTR], @@ -82,6 +84,8 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint // Double Precision Unary Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64SINCOS] = {ABIHandlers[FABI_F64x2_I16_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; @@ -198,6 +202,12 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf return true; \ } +#define COMMON_UNARYPAIR_X87_OP(OP) \ + case IR::OP_F80##OP: { \ + *Info = {FABI_F80x2_I16_F80_PTR, Core::OPINDEX_F80##OP}; \ + return true; \ + } + #define COMMON_BINARY_X87_OP(OP) \ case IR::OP_F80##OP: { \ *Info = {FABI_F80_I16_F80_F80_PTR, Core::OPINDEX_F80##OP}; \ @@ -215,6 +225,12 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf *Info = {FABI_F64_I16_F64_PTR, Core::OPINDEX_F64##OP}; \ return true; \ } +#define COMMON_UNARYPAIR_F64_OP(OP) \ + case IR::OP_F64##OP: { \ + *Info = {FABI_F64x2_I16_F64_PTR, Core::OPINDEX_F64##OP}; \ + return true; \ + } + #define COMMON_BINARY_F64_OP(OP) \ case IR::OP_F64##OP: { \ *Info = {FABI_F64_I16_F64_F64_PTR, Core::OPINDEX_F64##OP}; \ @@ -228,6 +244,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf COMMON_UNARY_X87_OP(SQRT) COMMON_UNARY_X87_OP(SIN) COMMON_UNARY_X87_OP(COS) + COMMON_UNARYPAIR_X87_OP(SINCOS) COMMON_UNARY_X87_OP(XTRACT_EXP) COMMON_UNARY_X87_OP(XTRACT_SIG) COMMON_UNARY_X87_OP(BCDSTORE) @@ -249,6 +266,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf COMMON_UNARY_F64_OP(TAN) COMMON_UNARY_F64_OP(SIN) COMMON_UNARY_F64_OP(COS) + COMMON_UNARYPAIR_F64_OP(SINCOS) // Double Precision Binary COMMON_BINARY_F64_OP(FYL2X) diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index 7df77c8e0..08bb44e36 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -27,6 +27,8 @@ enum FallbackABI { FABI_I64_I16_F80_F80_PTR, FABI_F80_I16_F80_PTR, FABI_F80_I16_F80_F80_PTR, + FABI_F80x2_I16_F80_PTR, + FABI_F64x2_I16_F64_PTR, FABI_I32_I64_I64_V128_V128_I16, FABI_I32_V128_V128_I16, FABI_UNKNOWN, diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 79d8303ee..25ea1fb43 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -84,6 +84,16 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { LOGMAN_MSG_A_FMT("Unhandled IR Op: {}", FEXCore::IR::GetName(IROp->Op)); #endif } else { + auto FillF80x2Result = [&](auto DstLo, auto DstHi) { + mov(DstLo.Q(), VTMP1.Q()); + mov(DstHi.Q(), VTMP2.Q()); + }; + + auto FillF64x2Result = [&](auto DstLo, auto DstHi) { + fmov(DstLo.D(), VTMP1.D()); + fmov(DstHi.D(), VTMP2.D()); + }; + auto FillF80Result = [&]() { const auto Dst = GetVReg(Node); mov(Dst.Q(), VTMP1.Q()); @@ -229,6 +239,30 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF64Result(); } break; + case FABI_F64x2_I16_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source + // vtmp2 (v1/v16): vector source +#ifdef VIXL_SIMULATOR + LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently"); +#endif + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); + + const auto Src1 = GetVReg(IROp->Args[0]); + const auto DstLo = GetVReg(IROp->Args[1]); + const auto DstHi = GetVReg(IROp->Args[2]); + + fmov(VTMP1.D(), Src1.D()); + + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillF64x2Result(DstLo, DstHi); + } break; case FABI_F64_I16_F64_F64_PTR: { // Linux Reg/Win32 Reg: @@ -345,6 +379,31 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { FillF80Result(); } break; + case FABI_F80x2_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v16): vector source 2 +#ifdef VIXL_SIMULATOR + LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently"); +#endif + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); + + const auto Src1 = GetVReg(IROp->Args[0]); + const auto DstLo = GetVReg(IROp->Args[1]); + const auto DstHi = GetVReg(IROp->Args[2]); + + mov(VTMP1.Q(), Src1.Q()); + + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillF80x2Result(DstLo, DstHi); + } break; + case FABI_F80_I16_F80_F80_PTR: { // Linux Reg/Win32 Reg: // tmp4 (x4/x13): FallbackHandler diff --git a/FEXCore/Source/Interface/IR/IR.json b/FEXCore/Source/Interface/IR/IR.json index f85b84d20..97da525c7 100644 --- a/FEXCore/Source/Interface/IR/IR.json +++ b/FEXCore/Source/Interface/IR/IR.json @@ -2788,6 +2788,11 @@ "FPR = F64COS FPR:$Src": { "DestSize": "OpSize::i64Bit", "JITDispatch": false + }, + "FPR:$Sin, FPR:$Cos = F64SINCOS FPR:$Src": { + "DestSize": "OpSize::i64Bit", + "HasSideEffects": true, + "JITDispatch": false } }, "F80": { @@ -3127,6 +3132,11 @@ "DestSize": "OpSize::i128Bit", "JITDispatch": false }, + "FPR:$Sin, FPR:$Cos = F80SINCOS FPR:$X80Src": { + "DestSize": "OpSize::i128Bit", + "HasSideEffects": true, + "JITDispatch": false + }, "F80SINCOSStack": { "X87": true, "HasSideEffects": true diff --git a/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp b/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp index 1e8124463..cda2bbc98 100644 --- a/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp +++ b/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp @@ -161,6 +161,7 @@ private: const FEXCore::HostFeatures& Features; const OpSize GPROpSize; bool ReducedPrecisionMode; + FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS); // Helpers Ref RotateRight8(uint32_t V, Ref Amount); @@ -774,12 +775,26 @@ void X87StackOptimization::Run(IREmitter* Emit) { Ref SinValue {}; Ref CosValue {}; - if (ReducedPrecisionMode) { - SinValue = IREmit->_F64SIN(St0); - CosValue = IREmit->_F64COS(St0); - } else { - SinValue = IREmit->_F80SIN(St0); - CosValue = IREmit->_F80COS(St0); + +#ifdef VIXL_SIMULATOR + if (DisableVixlIndirectCalls() == 0) { + if (ReducedPrecisionMode) { + SinValue = IREmit->_F64SIN(St0); + CosValue = IREmit->_F64COS(St0); + } else { + SinValue = IREmit->_F80SIN(St0); + CosValue = IREmit->_F80COS(St0); + } + } else +#endif + { + SinValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit); + CosValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit); + if (ReducedPrecisionMode) { + IREmit->_F64SINCOS(St0, SinValue, CosValue); + } else { + IREmit->_F80SINCOS(St0, SinValue, CosValue); + } } // Push values diff --git a/FEXCore/include/FEXCore/Core/CoreState.h b/FEXCore/include/FEXCore/Core/CoreState.h index 598d8abe1..42ce766cd 100644 --- a/FEXCore/include/FEXCore/Core/CoreState.h +++ b/FEXCore/include/FEXCore/Core/CoreState.h @@ -209,6 +209,7 @@ enum FallbackHandlerIndex { OPINDEX_F80SQRT, OPINDEX_F80SIN, OPINDEX_F80COS, + OPINDEX_F80SINCOS, OPINDEX_F80XTRACT_EXP, OPINDEX_F80XTRACT_SIG, OPINDEX_F80BCDSTORE, @@ -228,6 +229,7 @@ enum FallbackHandlerIndex { // Double Precision OPINDEX_F64SIN, OPINDEX_F64COS, + OPINDEX_F64SINCOS, OPINDEX_F64TAN, OPINDEX_F64ATAN, OPINDEX_F64F2XM1,