JIT: Optimize x87 FSINCOS

Turns out Bayonetta hammers SINCOS, our splitting the operation is actually harming the performance of games that heavily use FSINCOS. We instead can actually combine the operation which improves performance. Not enough to get the game running full speed consistently on my Radxa, but good numbers in my microbenchmark. ``` Test, Total Cycles, Total Runs, Cycles Average, Internal Loops, Average cycles per internal, per/second 64-bit: Before: FSIN, 2691031290, 50000, 53820.6, 1000, 53.8206, 18580.237319 FCOS, 2719397120, 50000, 54387.9, 1000, 54.3879, 18386.428239 FSINCOS, 5586917530, 50000, 111738, 1000, 111.738, 8949.478801 After: FSIN, 2669959250, 50000, 53399.2, 1000, 53.3992, 18726.877573 FCOS, 2740942260, 50000, 54818.8, 1000, 54.8188, 18241.901965 FSINCOS, 3189472870, 50000, 63789.5, 1000, 63.7895, 15676.571659 80-bit: Before: FSIN, 24702939380, 50000, 494059, 1000, 494.059, 2024.050629 FCOS, 19127131020, 50000, 382543, 1000, 382.543, 2614.087808 FSINCOS, 40386785260, 50000, 807736, 1000, 807.736, 1238.028719 After: FSIN, 24869980710, 50000, 497400, 1000, 497.4, 2010.455922 FCOS, 19131849590, 50000, 382637, 1000, 382.637, 2613.443084 FSINCOS, 38329985570, 50000, 766600, 1000, 766.6, 1304.461749 Improvement 64-bit: 1.75x Improvement 80-bit: 1.05x ``` Only a minor improvement at 80-bit precision since cephes doesn't provide a combined sincos operation, but the f64 implementation is significantly improved, allowing 75% more operations per second. Disabled in the simulator because we can't easily support pairs of vector registers being returned.
2026-10-06 11:00:19 +02:00 · 2025-07-03 18:05:51 -07:00
1 parent fa0a54deb9
commit 6ebbd91245
@@ -3,14 +3,32 @@

 #ifdef _M_X86_64
 #include <xmmintrin.h>
+#include <immintrin.h>
 #endif

 namespace FEXCore {
+struct VectorScalarF64Pair {
+  double val[2];
+};
+
 #ifdef _M_ARM_64
 // Can't use uint8x16_t directly from arm_neon.h here.
 // Overrides softfloat-3e's defines which causes problems.
 using VectorRegType = __attribute__((neon_vector_type(16))) uint8_t;
+struct VectorRegPairType {
+  VectorRegType val[2];
+};
+
+static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) {
+  return VectorRegPairType {low, high};
+}
+
 #elif defined(_M_X86_64)
 using VectorRegType = __m128i;
+using VectorRegPairType = __m256i;
+
+static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) {
+  return _mm256_set_m128i(high, low);
+}
 #endif
 } // namespace FEXCore
@@ -558,6 +558,8 @@ void Dispatcher::EmitDispatcher() {
      FABI_I64_I16_F80_F80_PTR,
      FABI_F80_I16_F80_PTR,
      FABI_F80_I16_F80_F80_PTR,
+      FABI_F80x2_I16_F80_PTR,
+      FABI_F64x2_I16_F64_PTR,
      FABI_I32_I64_I64_V128_V128_I16,
      FABI_I32_V128_V128_I16,
    }};
@@ -626,6 +628,22 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
  constexpr static auto VABI1 = ARMEmitter::VReg::v0;
  constexpr static auto VABI2 = ARMEmitter::VReg::v1;

+  auto FillF80x2Result = [&]() {
+    if (!TMP_ABIARGS) {
+      mov(VTMP1.Q(), VABI1.Q());
+      mov(VTMP2.Q(), VABI2.Q());
+    }
+    FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);
+  };
+
+  auto FillF64x2Result = [&]() {
+    if (!TMP_ABIARGS) {
+      fmov(VTMP1.D(), VABI1.D());
+      fmov(VTMP2.D(), VABI2.D());
+    }
+    FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);
+  };
+
  auto FillF80Result = [&]() {
    if (VTMP1 != VABI1) {
      mov(VTMP1.Q(), VABI1.Q());
@@ -946,6 +964,52 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {

    FillF80Result();
  } break;
+  case FABI_F80x2_I16_F80_PTR: {
+    // Linux Reg/Win32 Reg:
+    // tmp4 (x4/x13): FallbackHandler
+    // x30: return
+    // vtmp1 (v0/v16): vector source 1
+    // vtmp2 (v1/v16): vector source 2
+
+    SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);
+
+    ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
+    mov(ARMEmitter::XReg::x1, STATE);
+    if (!TMP_ABIARGS) {
+      mov(VABI1.Q(), VTMP1.Q());
+    }
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      // GenerateIndirectRuntimeCall<FEXCore::VectorRegPairType, uint16_t, FEXCore::VectorRegType, uint64_t>(FallbackPointerReg);
+    } else {
+      blr(FallbackPointerReg);
+    }
+
+    FillF80x2Result();
+  } break;
+  case FABI_F64x2_I16_F64_PTR: {
+    // Linux Reg/Win32 Reg:
+    // tmp4 (x4/x13): FallbackHandler
+    // x30: return
+    // vtmp1 (v0/v16): vector source 1
+    // vtmp2 (v1/v16): vector source 2
+
+    SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);
+
+    ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
+    mov(ARMEmitter::XReg::x1, STATE);
+    if (!TMP_ABIARGS) {
+      fmov(VABI1.D(), VTMP1.D());
+    }
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      // GenerateIndirectRuntimeCall<FEXCore::VectorScalarF64Pair, uint16_t, FEXCore::VectorRegType, uint64_t>(FallbackPointerReg);
+    } else {
+      blr(FallbackPointerReg);
+    }
+
+    FillF64x2Result();
+  } break;
  case FABI_I32_I64_I64_V128_V128_I16: {
    // Linux Reg/Win32 Reg:
    // stack: FallbackHandler
@@ -202,6 +202,15 @@ struct OpHandlers<IR::OP_F80COS> {
  }
 };

+template<>
+struct OpHandlers<IR::OP_F80SINCOS> {
+  FEXCORE_PRESERVE_ALL_ATTR static VectorRegPairType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
+    FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
+    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
+    return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State, Src1), X80SoftFloat::FCOS(&State, Src1));
+  }
+};
+
 template<>
 struct OpHandlers<IR::OP_F80XTRACT_EXP> {
  FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
@@ -315,6 +324,21 @@ struct OpHandlers<IR::OP_F64COS> {
  }
 };

+template<>
+struct OpHandlers<IR::OP_F64SINCOS> {
+  FEXCORE_PRESERVE_ALL_ATTR static VectorScalarF64Pair handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
+    FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
+    double sin, cos;
+#ifdef _WIN32
+    sin = ::sin(src);
+    cos = ::cos(src);
+#else
+    sincos(src, &sin, &cos);
+#endif
+    return VectorScalarF64Pair {sin, cos};
+  }
+};
+
 template<>
 struct OpHandlers<IR::OP_F64TAN> {
  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
@@ -50,6 +50,8 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint
  Info[Core::OPINDEX_F80SQRT] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SQRT>::handle)};
  Info[Core::OPINDEX_F80SIN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SIN>::handle)};
  Info[Core::OPINDEX_F80COS] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80COS>::handle)};
+  Info[Core::OPINDEX_F80SINCOS] = {ABIHandlers[FABI_F80x2_I16_F80_PTR],
+                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SINCOS>::handle)};
  Info[Core::OPINDEX_F80XTRACT_EXP] = {ABIHandlers[FABI_F80_I16_F80_PTR],
                                       reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80XTRACT_EXP>::handle)};
  Info[Core::OPINDEX_F80XTRACT_SIG] = {ABIHandlers[FABI_F80_I16_F80_PTR],
@@ -82,6 +84,8 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint
  // Double Precision Unary
  Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SIN>::handle)};
  Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64COS>::handle)};
+  Info[Core::OPINDEX_F64SINCOS] = {ABIHandlers[FABI_F64x2_I16_F64_PTR],
+                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SINCOS>::handle)};
  Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64TAN>::handle)};
  Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_I16_F64_PTR],
                                  reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64F2XM1>::handle)};
@@ -198,6 +202,12 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
    return true;                                           \
  }

+#define COMMON_UNARYPAIR_X87_OP(OP)                          \
+  case IR::OP_F80##OP: {                                     \
+    *Info = {FABI_F80x2_I16_F80_PTR, Core::OPINDEX_F80##OP}; \
+    return true;                                             \
+  }
+
 #define COMMON_BINARY_X87_OP(OP)                               \
  case IR::OP_F80##OP: {                                       \
    *Info = {FABI_F80_I16_F80_F80_PTR, Core::OPINDEX_F80##OP}; \
@@ -215,6 +225,12 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
    *Info = {FABI_F64_I16_F64_PTR, Core::OPINDEX_F64##OP}; \
    return true;                                           \
  }
+#define COMMON_UNARYPAIR_F64_OP(OP)                          \
+  case IR::OP_F64##OP: {                                     \
+    *Info = {FABI_F64x2_I16_F64_PTR, Core::OPINDEX_F64##OP}; \
+    return true;                                             \
+  }
+
 #define COMMON_BINARY_F64_OP(OP)                               \
  case IR::OP_F64##OP: {                                       \
    *Info = {FABI_F64_I16_F64_F64_PTR, Core::OPINDEX_F64##OP}; \
@@ -228,6 +244,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
    COMMON_UNARY_X87_OP(SQRT)
    COMMON_UNARY_X87_OP(SIN)
    COMMON_UNARY_X87_OP(COS)
+    COMMON_UNARYPAIR_X87_OP(SINCOS)
    COMMON_UNARY_X87_OP(XTRACT_EXP)
    COMMON_UNARY_X87_OP(XTRACT_SIG)
    COMMON_UNARY_X87_OP(BCDSTORE)
@@ -249,6 +266,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
    COMMON_UNARY_F64_OP(TAN)
    COMMON_UNARY_F64_OP(SIN)
    COMMON_UNARY_F64_OP(COS)
+    COMMON_UNARYPAIR_F64_OP(SINCOS)

    // Double Precision Binary
    COMMON_BINARY_F64_OP(FYL2X)
@@ -27,6 +27,8 @@ enum FallbackABI {
  FABI_I64_I16_F80_F80_PTR,
  FABI_F80_I16_F80_PTR,
  FABI_F80_I16_F80_F80_PTR,
+  FABI_F80x2_I16_F80_PTR,
+  FABI_F64x2_I16_F64_PTR,
  FABI_I32_I64_I64_V128_V128_I16,
  FABI_I32_V128_V128_I16,
  FABI_UNKNOWN,
@@ -84,6 +84,16 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
    LOGMAN_MSG_A_FMT("Unhandled IR Op: {}", FEXCore::IR::GetName(IROp->Op));
 #endif
  } else {
+    auto FillF80x2Result = [&](auto DstLo, auto DstHi) {
+      mov(DstLo.Q(), VTMP1.Q());
+      mov(DstHi.Q(), VTMP2.Q());
+    };
+
+    auto FillF64x2Result = [&](auto DstLo, auto DstHi) {
+      fmov(DstLo.D(), VTMP1.D());
+      fmov(DstHi.D(), VTMP2.D());
+    };
+
    auto FillF80Result = [&]() {
      const auto Dst = GetVReg(Node);
      mov(Dst.Q(), VTMP1.Q());
@@ -229,6 +239,30 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
      FillF64Result();
    } break;
+    case FABI_F64x2_I16_F64_PTR: {
+      // Linux Reg/Win32 Reg:
+      // tmp4 (x4/x13): FallbackHandler
+      // x30: return
+      // vtmp1 (v0/v16): vector source
+      // vtmp2 (v1/v16): vector source
+#ifdef VIXL_SIMULATOR
+      LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently");
+#endif
+      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
+
+      const auto Src1 = GetVReg(IROp->Args[0]);
+      const auto DstLo = GetVReg(IROp->Args[1]);
+      const auto DstHi = GetVReg(IROp->Args[2]);
+
+      fmov(VTMP1.D(), Src1.D());
+
+      ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
+      ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
+      blr(TMP1);
+
+      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      FillF64x2Result(DstLo, DstHi);
+    } break;

    case FABI_F64_I16_F64_F64_PTR: {
      // Linux Reg/Win32 Reg:
@@ -345,6 +379,31 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
      FillF80Result();
    } break;

+    case FABI_F80x2_I16_F80_PTR: {
+      // Linux Reg/Win32 Reg:
+      // tmp4 (x4/x13): FallbackHandler
+      // x30: return
+      // vtmp1 (v0/v16): vector source 1
+      // vtmp2 (v1/v16): vector source 2
+#ifdef VIXL_SIMULATOR
+      LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently");
+#endif
+      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
+
+      const auto Src1 = GetVReg(IROp->Args[0]);
+      const auto DstLo = GetVReg(IROp->Args[1]);
+      const auto DstHi = GetVReg(IROp->Args[2]);
+
+      mov(VTMP1.Q(), Src1.Q());
+
+      ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
+      ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
+      blr(TMP1);
+
+      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      FillF80x2Result(DstLo, DstHi);
+    } break;
+
    case FABI_F80_I16_F80_F80_PTR: {
      // Linux Reg/Win32 Reg:
      // tmp4 (x4/x13): FallbackHandler
@@ -2788,6 +2788,11 @@
      "FPR = F64COS FPR:$Src": {
        "DestSize": "OpSize::i64Bit",
        "JITDispatch": false
+      },
+      "FPR:$Sin, FPR:$Cos = F64SINCOS FPR:$Src": {
+        "DestSize": "OpSize::i64Bit",
+        "HasSideEffects": true,
+        "JITDispatch": false
      }
    },
    "F80": {
@@ -3127,6 +3132,11 @@
        "DestSize": "OpSize::i128Bit",
        "JITDispatch": false
      },
+      "FPR:$Sin, FPR:$Cos = F80SINCOS FPR:$X80Src": {
+        "DestSize": "OpSize::i128Bit",
+        "HasSideEffects": true,
+        "JITDispatch": false
+      },
      "F80SINCOSStack": {
        "X87": true,
        "HasSideEffects": true
@@ -161,6 +161,7 @@ private:
  const FEXCore::HostFeatures& Features;
  const OpSize GPROpSize;
  bool ReducedPrecisionMode;
+  FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS);

  // Helpers
  Ref RotateRight8(uint32_t V, Ref Amount);
@@ -774,12 +775,26 @@ void X87StackOptimization::Run(IREmitter* Emit) {

        Ref SinValue {};
        Ref CosValue {};
-        if (ReducedPrecisionMode) {
-          SinValue = IREmit->_F64SIN(St0);
-          CosValue = IREmit->_F64COS(St0);
-        } else {
-          SinValue = IREmit->_F80SIN(St0);
-          CosValue = IREmit->_F80COS(St0);
+
+#ifdef VIXL_SIMULATOR
+        if (DisableVixlIndirectCalls() == 0) {
+          if (ReducedPrecisionMode) {
+            SinValue = IREmit->_F64SIN(St0);
+            CosValue = IREmit->_F64COS(St0);
+          } else {
+            SinValue = IREmit->_F80SIN(St0);
+            CosValue = IREmit->_F80COS(St0);
+          }
+        } else
+#endif
+        {
+          SinValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit);
+          CosValue = IREmit->_AllocateFPR(OpSize::i128Bit, OpSize::i128Bit);
+          if (ReducedPrecisionMode) {
+            IREmit->_F64SINCOS(St0, SinValue, CosValue);
+          } else {
+            IREmit->_F80SINCOS(St0, SinValue, CosValue);
+          }
        }

        // Push values
@@ -209,6 +209,7 @@ enum FallbackHandlerIndex {
  OPINDEX_F80SQRT,
  OPINDEX_F80SIN,
  OPINDEX_F80COS,
+  OPINDEX_F80SINCOS,
  OPINDEX_F80XTRACT_EXP,
  OPINDEX_F80XTRACT_SIG,
  OPINDEX_F80BCDSTORE,
@@ -228,6 +229,7 @@ enum FallbackHandlerIndex {
  // Double Precision
  OPINDEX_F64SIN,
  OPINDEX_F64COS,
+  OPINDEX_F64SINCOS,
  OPINDEX_F64TAN,
  OPINDEX_F64ATAN,
  OPINDEX_F64F2XM1,