From 6ebbd91245168ee0363802d3034847751a17b077 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Thu, 3 Jul 2025 15:13:25 -0700 Subject: [PATCH] JIT: Optimize x87 FSINCOS Turns out Bayonetta hammers SINCOS, our splitting the operation is actually harming the performance of games that heavily use FSINCOS. We instead can actually combine the operation which improves performance. Not enough to get the game running full speed consistently on my Radxa, but good numbers in my microbenchmark. ``` Test, Total Cycles, Total Runs, Cycles Average, Internal Loops, Average cycles per internal, per/second 64-bit: Before: FSIN, 2691031290, 50000, 53820.6, 1000, 53.8206, 18580.237319 FCOS, 2719397120, 50000, 54387.9, 1000, 54.3879, 18386.428239 FSINCOS, 5586917530, 50000, 111738, 1000, 111.738, 8949.478801 After: FSIN, 2669959250, 50000, 53399.2, 1000, 53.3992, 18726.877573 FCOS, 2740942260, 50000, 54818.8, 1000, 54.8188, 18241.901965 FSINCOS, 3189472870, 50000, 63789.5, 1000, 63.7895, 15676.571659 80-bit: Before: FSIN, 24702939380, 50000, 494059, 1000, 494.059, 2024.050629 FCOS, 19127131020, 50000, 382543, 1000, 382.543, 2614.087808 FSINCOS, 40386785260, 50000, 807736, 1000, 807.736, 1238.028719 After: FSIN, 24869980710, 50000, 497400, 1000, 497.4, 2010.455922 FCOS, 19131849590, 50000, 382637, 1000, 382.637, 2613.443084 FSINCOS, 38329985570, 50000, 766600, 1000, 766.6, 1304.461749 Improvement 64-bit: 1.75x Improvement 80-bit: 1.05x ``` Only a minor improvement at 80-bit precision since cephes doesn't provide a combined sincos operation, but the f64 implementation is significantly improved, allowing 75% more operations per second. Disabled in the simulator because we can't easily support pairs of vector registers being returned. --- FEXCore/Source/Common/VectorRegType.h | 18 ++++++ .../Interface/Core/Dispatcher/Dispatcher.cpp | 64 +++++++++++++++++++ .../Core/Interpreter/Fallbacks/F80Fallbacks.h | 24 +++++++ .../Fallbacks/InterpreterFallbacks.cpp | 18 ++++++ .../Core/Interpreter/InterpreterOps.h | 2 + FEXCore/Source/Interface/Core/JIT/JIT.cpp | 59 +++++++++++++++++ FEXCore/Source/Interface/IR/IR.json | 10 +++ .../IR/Passes/x87StackOptimizationPass.cpp | 27 ++++++-- FEXCore/include/FEXCore/Core/CoreState.h | 2 + 9 files changed, 218 insertions(+), 6 deletions(-) diff --git a/FEXCore/Source/Common/VectorRegType.h b/FEXCore/Source/Common/VectorRegType.h index 8265832b6..f4209d48d 100644 --- a/FEXCore/Source/Common/VectorRegType.h +++ b/FEXCore/Source/Common/VectorRegType.h @@ -3,14 +3,32 @@ #ifdef _M_X86_64 #include +#include #endif namespace FEXCore { +struct VectorScalarF64Pair { + double val[2]; +}; + #ifdef _M_ARM_64 // Can't use uint8x16_t directly from arm_neon.h here. // Overrides softfloat-3e's defines which causes problems. using VectorRegType = __attribute__((neon_vector_type(16))) uint8_t; +struct VectorRegPairType { + VectorRegType val[2]; +}; + +static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) { + return VectorRegPairType {low, high}; +} + #elif defined(_M_X86_64) using VectorRegType = __m128i; +using VectorRegPairType = __m256i; + +static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) { + return _mm256_set_m128i(high, low); +} #endif } // namespace FEXCore diff --git a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp index a545e15f2..34b059700 100644 --- a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp +++ b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp @@ -558,6 +558,8 @@ void Dispatcher::EmitDispatcher() { FABI_I64_I16_F80_F80_PTR, FABI_F80_I16_F80_PTR, FABI_F80_I16_F80_F80_PTR, + FABI_F80x2_I16_F80_PTR, + FABI_F64x2_I16_F64_PTR, FABI_I32_I64_I64_V128_V128_I16, FABI_I32_V128_V128_I16, }}; @@ -626,6 +628,22 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) { constexpr static auto VABI1 = ARMEmitter::VReg::v0; constexpr static auto VABI2 = ARMEmitter::VReg::v1; + auto FillF80x2Result = [&]() { + if (!TMP_ABIARGS) { + mov(VTMP1.Q(), VABI1.Q()); + mov(VTMP2.Q(), VABI2.Q()); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + + auto FillF64x2Result = [&]() { + if (!TMP_ABIARGS) { + fmov(VTMP1.D(), VABI1.D()); + fmov(VTMP2.D(), VABI2.D()); + } + FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true); + }; + auto FillF80Result = [&]() { if (VTMP1 != VABI1) { mov(VTMP1.Q(), VABI1.Q()); @@ -946,6 +964,52 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) { FillF80Result(); } break; + case FABI_F80x2_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v16): vector source 2 + + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!TMP_ABIARGS) { + mov(VABI1.Q(), VTMP1.Q()); + } + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + // GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF80x2Result(); + } break; + case FABI_F64x2_I16_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v16): vector source 2 + + SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true); + + ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW)); + mov(ARMEmitter::XReg::x1, STATE); + if (!TMP_ABIARGS) { + fmov(VABI1.D(), VTMP1.D()); + } + + if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] { + // GenerateIndirectRuntimeCall(FallbackPointerReg); + } else { + blr(FallbackPointerReg); + } + + FillF64x2Result(); + } break; case FABI_I32_I64_I64_V128_V128_I16: { // Linux Reg/Win32 Reg: // stack: FallbackHandler diff --git a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h index d30b1d448..e35fb56b8 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h +++ b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h @@ -202,6 +202,15 @@ struct OpHandlers { } }; +template<> +struct OpHandlers { + FEXCORE_PRESERVE_ALL_ATTR static VectorRegPairType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) { + FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1); + softfloat_state State = SoftFloatStateFromFCW(FCW, true); + return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State, Src1), X80SoftFloat::FCOS(&State, Src1)); + } +}; + template<> struct OpHandlers { FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) { @@ -315,6 +324,21 @@ struct OpHandlers { } }; +template<> +struct OpHandlers { + FEXCORE_PRESERVE_ALL_ATTR static VectorScalarF64Pair handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) { + FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1); + double sin, cos; +#ifdef _WIN32 + sin = ::sin(src); + cos = ::cos(src); +#else + sincos(src, &sin, &cos); +#endif + return VectorScalarF64Pair {sin, cos}; + } +}; + template<> struct OpHandlers { FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) { diff --git a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp index 3f50867ce..21a29a8dc 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp +++ b/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp @@ -50,6 +50,8 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint Info[Core::OPINDEX_F80SQRT] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80SIN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80COS] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F80SINCOS] = {ABIHandlers[FABI_F80x2_I16_F80_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80XTRACT_EXP] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F80XTRACT_SIG] = {ABIHandlers[FABI_F80_I16_F80_PTR], @@ -82,6 +84,8 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint // Double Precision Unary Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; + Info[Core::OPINDEX_F64SINCOS] = {ABIHandlers[FABI_F64x2_I16_F64_PTR], + reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast(&FEXCore::CPU::OpHandlers::handle)}; @@ -198,6 +202,12 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf return true; \ } +#define COMMON_UNARYPAIR_X87_OP(OP) \ + case IR::OP_F80##OP: { \ + *Info = {FABI_F80x2_I16_F80_PTR, Core::OPINDEX_F80##OP}; \ + return true; \ + } + #define COMMON_BINARY_X87_OP(OP) \ case IR::OP_F80##OP: { \ *Info = {FABI_F80_I16_F80_F80_PTR, Core::OPINDEX_F80##OP}; \ @@ -215,6 +225,12 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf *Info = {FABI_F64_I16_F64_PTR, Core::OPINDEX_F64##OP}; \ return true; \ } +#define COMMON_UNARYPAIR_F64_OP(OP) \ + case IR::OP_F64##OP: { \ + *Info = {FABI_F64x2_I16_F64_PTR, Core::OPINDEX_F64##OP}; \ + return true; \ + } + #define COMMON_BINARY_F64_OP(OP) \ case IR::OP_F64##OP: { \ *Info = {FABI_F64_I16_F64_F64_PTR, Core::OPINDEX_F64##OP}; \ @@ -228,6 +244,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf COMMON_UNARY_X87_OP(SQRT) COMMON_UNARY_X87_OP(SIN) COMMON_UNARY_X87_OP(COS) + COMMON_UNARYPAIR_X87_OP(SINCOS) COMMON_UNARY_X87_OP(XTRACT_EXP) COMMON_UNARY_X87_OP(XTRACT_SIG) COMMON_UNARY_X87_OP(BCDSTORE) @@ -249,6 +266,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf COMMON_UNARY_F64_OP(TAN) COMMON_UNARY_F64_OP(SIN) COMMON_UNARY_F64_OP(COS) + COMMON_UNARYPAIR_F64_OP(SINCOS) // Double Precision Binary COMMON_BINARY_F64_OP(FYL2X) diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index 7df77c8e0..08bb44e36 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -27,6 +27,8 @@ enum FallbackABI { FABI_I64_I16_F80_F80_PTR, FABI_F80_I16_F80_PTR, FABI_F80_I16_F80_F80_PTR, + FABI_F80x2_I16_F80_PTR, + FABI_F64x2_I16_F64_PTR, FABI_I32_I64_I64_V128_V128_I16, FABI_I32_V128_V128_I16, FABI_UNKNOWN, diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 79d8303ee..25ea1fb43 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -84,6 +84,16 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { LOGMAN_MSG_A_FMT("Unhandled IR Op: {}", FEXCore::IR::GetName(IROp->Op)); #endif } else { + auto FillF80x2Result = [&](auto DstLo, auto DstHi) { + mov(DstLo.Q(), VTMP1.Q()); + mov(DstHi.Q(), VTMP2.Q()); + }; + + auto FillF64x2Result = [&](auto DstLo, auto DstHi) { + fmov(DstLo.D(), VTMP1.D()); + fmov(DstHi.D(), VTMP2.D()); + }; + auto FillF80Result = [&]() { const auto Dst = GetVReg(Node); mov(Dst.Q(), VTMP1.Q()); @@ -229,6 +239,30 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); FillF64Result(); } break; + case FABI_F64x2_I16_F64_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source + // vtmp2 (v1/v16): vector source +#ifdef VIXL_SIMULATOR + LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently"); +#endif + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); + + const auto Src1 = GetVReg(IROp->Args[0]); + const auto DstLo = GetVReg(IROp->Args[1]); + const auto DstHi = GetVReg(IROp->Args[2]); + + fmov(VTMP1.D(), Src1.D()); + + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillF64x2Result(DstLo, DstHi); + } break; case FABI_F64_I16_F64_F64_PTR: { // Linux Reg/Win32 Reg: @@ -345,6 +379,31 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { FillF80Result(); } break; + case FABI_F80x2_I16_F80_PTR: { + // Linux Reg/Win32 Reg: + // tmp4 (x4/x13): FallbackHandler + // x30: return + // vtmp1 (v0/v16): vector source 1 + // vtmp2 (v1/v16): vector source 2 +#ifdef VIXL_SIMULATOR + LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently"); +#endif + str(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16); + + const auto Src1 = GetVReg(IROp->Args[0]); + const auto DstLo = GetVReg(IROp->Args[1]); + const auto DstHi = GetVReg(IROp->Args[2]); + + mov(VTMP1.Q(), Src1.Q()); + + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler)); + ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func)); + blr(TMP1); + + ldr(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16); + FillF80x2Result(DstLo, DstHi); + } break; + case FABI_F80_I16_F80_F80_PTR: { // Linux Reg/Win32 Reg: // tmp4 (x4/x13): FallbackHandler diff --git a/FEXCore/Source/Interface/IR/IR.json b/FEXCore/Source/Interface/IR/IR.json index f85b84d20..97da525c7 100644 --- a/FEXCore/Source/Interface/IR/IR.json +++ b/FEXCore/Source/Interface/IR/IR.json @@ -2788,6 +2788,11 @@ "FPR = F64COS FPR:$Src": { "DestSize": "OpSize::i64Bit", "JITDispatch": false + }, + "FPR:$Sin, FPR:$Cos = F64SINCOS FPR:$Src": { + "DestSize": "OpSize::i64Bit", + "HasSideEffects": true, + "JITDispatch": false } }, "F80": { @@ -3127,6 +3132,11 @@ "DestSize": "OpSize::i128Bit", "JITDispatch": false }, + "FPR:$Sin, FPR:$Cos = F80SINCOS FPR:$X80Src": { + "DestSize": "OpSize::i128Bit", + "HasSideEffects": true, + "JITDispatch": false + }, "F80SINCOSStack": { "X87": true, "HasSideEffects": true diff --git a/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp b/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp index 1e8124463..cda2bbc98 100644 --- a/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp +++ b/FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp @@ -161,6 +161,7 @@ private: const FEXCore::HostFeatures& Features; const OpSize GPROpSize; bool ReducedPrecisionMode; + FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS); // Helpers Ref RotateRight8(uint32_t V, Ref Amount); @@ -774,12 +775,26 @@ void X87StackOptimization::Run(IREmitter* Emit) { Ref SinValue {}; Ref CosValue {}; - if (ReducedPrecisionMode) { - SinValue = IREmit->_F64SIN(St0); - CosValue = IREmit->_F64COS(St0); - } else { - SinValue = IREmit->_F80SIN(St0); - CosValue = IREmit->_F80COS(St0); + +#ifdef VIXL_SIMULATOR + if (DisableVixlIndirectCalls() == 0) { + if (ReducedPrecisionMode) { + SinValue = IREmit->_F64SIN(St0); + CosValue = IREmit->_F64COS(St0); + } else { + SinValue = IREmit->_F80SIN(St0); + CosValue = IREmit->_F80COS(St0); + } + } else +#endif + { + SinValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit); + CosValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit); + if (ReducedPrecisionMode) { + IREmit->_F64SINCOS(St0, SinValue, CosValue); + } else { + IREmit->_F80SINCOS(St0, SinValue, CosValue); + } } // Push values diff --git a/FEXCore/include/FEXCore/Core/CoreState.h b/FEXCore/include/FEXCore/Core/CoreState.h index 598d8abe1..42ce766cd 100644 --- a/FEXCore/include/FEXCore/Core/CoreState.h +++ b/FEXCore/include/FEXCore/Core/CoreState.h @@ -209,6 +209,7 @@ enum FallbackHandlerIndex { OPINDEX_F80SQRT, OPINDEX_F80SIN, OPINDEX_F80COS, + OPINDEX_F80SINCOS, OPINDEX_F80XTRACT_EXP, OPINDEX_F80XTRACT_SIG, OPINDEX_F80BCDSTORE, @@ -228,6 +229,7 @@ enum FallbackHandlerIndex { // Double Precision OPINDEX_F64SIN, OPINDEX_F64COS, + OPINDEX_F64SINCOS, OPINDEX_F64TAN, OPINDEX_F64ATAN, OPINDEX_F64F2XM1,