diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index 627a97e70..110857fb7 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -4606,7 +4606,7 @@ OrderedNode *OpDispatchBuilder::AppendSegmentOffset(OrderedNode *Value, uint32_t return Value; } -OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData, bool ForceLoad) { +OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData, bool ForceLoad, MemoryAccessType AccessType) { LOGMAN_THROW_A_FMT(Operand.IsGPR() || Operand.IsLiteral() || Operand.IsGPRDirect() || @@ -4617,7 +4617,6 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass OrderedNode *Src {nullptr}; bool LoadableType = false; - bool StackAccess = false; const uint8_t GPRSize = CTX->GetGPRSize(); const uint32_t AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize; @@ -4646,7 +4645,9 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass else if (Operand.IsGPRDirect()) { Src = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR])); LoadableType = true; - StackAccess = Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP; + if (Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { + AccessType = MemoryAccessType::ACCESS_NONTSO; + } } else if (Operand.IsGPRIndirect()) { auto GPR = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR])); @@ -4655,7 +4656,9 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass Src = _Add(GPR, Constant); LoadableType = true; - StackAccess = Operand.Data.GPRIndirect.GPR == FEXCore::X86State::REG_RSP; + if (Operand.Data.GPRIndirect.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { + AccessType = MemoryAccessType::ACCESS_NONTSO; + } } else if (Operand.IsRIPRelative()) { if (CTX->Config.Is64BitMode) { @@ -4677,7 +4680,9 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass auto Constant = _Constant(GPRSize * 8, Operand.Data.SIB.Scale); Tmp = _Mul(Tmp, Constant); } - StackAccess |= Operand.Data.SIB.Index == FEXCore::X86State::REG_RSP; + if (Operand.Data.SIB.Index == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { + AccessType = MemoryAccessType::ACCESS_NONTSO; + } } if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) { @@ -4689,7 +4694,10 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass else { Tmp = GPR; } - StackAccess |= Operand.Data.SIB.Base == FEXCore::X86State::REG_RSP; + + if (Operand.Data.SIB.Base == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { + AccessType = MemoryAccessType::ACCESS_NONTSO; + } } if (Operand.Data.SIB.Offset) { @@ -4724,7 +4732,7 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass if ((LoadableType && LoadData) || ForceLoad) { Src = AppendSegmentOffset(Src, Flags); - if (StackAccess) { + if (AccessType == MemoryAccessType::ACCESS_NONTSO || AccessType == MemoryAccessType::ACCESS_STREAM) { Src = _LoadMem(Class, OpSize, Src, Align == -1 ? OpSize : Align); } else { @@ -4739,12 +4747,12 @@ OrderedNode *OpDispatchBuilder::GetRelocatedPC(FEXCore::X86Tables::DecodedOp con return _EntrypointOffset(Op->PC + Op->InstSize + Offset - Entry, GPRSize); } -OrderedNode *OpDispatchBuilder::LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData, bool ForceLoad) { +OrderedNode *OpDispatchBuilder::LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData, bool ForceLoad, MemoryAccessType AccessType) { const uint8_t OpSize = GetSrcSize(Op); - return LoadSource_WithOpSize(Class, Op, Operand, OpSize, Flags, Align, LoadData, ForceLoad); + return LoadSource_WithOpSize(Class, Op, Operand, OpSize, Flags, Align, LoadData, ForceLoad, AccessType); } -void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, uint8_t OpSize, int8_t Align) { +void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, uint8_t OpSize, int8_t Align, MemoryAccessType AccessType) { LOGMAN_THROW_A_FMT(Operand.IsGPR() || Operand.IsLiteral() || Operand.IsGPRDirect() || @@ -4757,7 +4765,6 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl // 32bit ops ZEXT the result to 64bit OrderedNode *MemStoreDst {nullptr}; bool MemStore = false; - bool StackAccess = false; const uint8_t GPRSize = CTX->GetGPRSize(); const uint32_t AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize; @@ -4791,7 +4798,9 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl else if (Operand.IsGPRDirect()) { MemStoreDst = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR])); MemStore = true; - StackAccess = Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP; + if (Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { + AccessType = MemoryAccessType::ACCESS_NONTSO; + } } else if (Operand.IsGPRIndirect()) { auto GPR = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR])); @@ -4799,7 +4808,9 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl MemStoreDst = _Add(GPR, Constant); MemStore = true; - StackAccess = Operand.Data.GPRIndirect.GPR == FEXCore::X86State::REG_RSP; + if (Operand.Data.GPRIndirect.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { + AccessType = MemoryAccessType::ACCESS_NONTSO; + } } else if (Operand.IsRIPRelative()) { if (CTX->Config.Is64BitMode) { @@ -4869,7 +4880,7 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl auto DestAddr = _Add(MemStoreDst, _Constant(8)); _StoreMem(GPRClass, 2, DestAddr, Upper, std::min(Align, 8)); } else { - if (StackAccess) { + if (AccessType == MemoryAccessType::ACCESS_NONTSO || AccessType == MemoryAccessType::ACCESS_STREAM) { _StoreMem(Class, OpSize, MemStoreDst, Src, Align == -1 ? OpSize : Align); } else { @@ -4879,12 +4890,12 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl } } -void OpDispatchBuilder::StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align) { - StoreResult_WithOpSize(Class, Op, Operand, Src, GetDstSize(Op), Align); +void OpDispatchBuilder::StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align, MemoryAccessType AccessType) { + StoreResult_WithOpSize(Class, Op, Operand, Src, GetDstSize(Op), Align, AccessType); } -void OpDispatchBuilder::StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align) { - StoreResult(Class, Op, Op->Dest, Src, Align); +void OpDispatchBuilder::StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align, MemoryAccessType AccessType) { + StoreResult(Class, Op, Op->Dest, Src, Align, AccessType); } OpDispatchBuilder::OpDispatchBuilder(FEXCore::Context::Context *ctx) @@ -4917,6 +4928,11 @@ void OpDispatchBuilder::MOVGPROp(OpcodeArgs) { StoreResult(GPRClass, Op, Src, 1); } +void OpDispatchBuilder::MOVGPRNTOp(OpcodeArgs) { + OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, 1); + StoreResult(GPRClass, Op, Src, 1, MemoryAccessType::ACCESS_STREAM); +} + void OpDispatchBuilder::ALUOp(OpcodeArgs) { bool RequiresMask = false; FEXCore::IR::IROps IROp; @@ -5454,7 +5470,7 @@ void InstallOpcodeHandlers(Context::OperatingMode Mode) { {0xBD, 1, &OpDispatchBuilder::BSROp}, // BSF {0xBE, 2, &OpDispatchBuilder::MOVSXOp}, {0xC0, 2, &OpDispatchBuilder::XADDOp}, - {0xC3, 1, &OpDispatchBuilder::MOVGPROp<0>}, + {0xC3, 1, &OpDispatchBuilder::MOVGPRNTOp}, {0xC4, 1, &OpDispatchBuilder::PINSROp<2>}, {0xC5, 1, &OpDispatchBuilder::PExtrOp<2>}, {0xC8, 8, &OpDispatchBuilder::BSWAPOp}, @@ -5468,7 +5484,7 @@ void InstallOpcodeHandlers(Context::OperatingMode Mode) { {0x17, 1, &OpDispatchBuilder::MOVUPSOp}, {0x28, 2, &OpDispatchBuilder::MOVUPSOp}, {0x2A, 1, &OpDispatchBuilder::MMX_To_XMM_Vector_CVT_Int_To_Float<4, false>}, - {0x2B, 1, &OpDispatchBuilder::MOVAPSOp}, + {0x2B, 1, &OpDispatchBuilder::MOVVectorNTOp}, {0x2C, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<4, false, false>}, {0x2D, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<4, false, true>}, {0x2E, 2, &OpDispatchBuilder::UCOMISxOp<4>}, @@ -5530,7 +5546,7 @@ void InstallOpcodeHandlers(Context::OperatingMode Mode) { {0xE3, 1, &OpDispatchBuilder::PAVGOp<2>}, {0xE4, 1, &OpDispatchBuilder::PMULHW}, {0xE5, 1, &OpDispatchBuilder::PMULHW}, - {0xE7, 1, &OpDispatchBuilder::MOVUPSOp}, + {0xE7, 1, &OpDispatchBuilder::MOVVectorNTOp}, {0xE8, 1, &OpDispatchBuilder::PSUBSOp<1, true>}, {0xE9, 1, &OpDispatchBuilder::PSUBSOp<2, true>}, {0xEA, 1, &OpDispatchBuilder::VectorALUOp}, @@ -5758,7 +5774,7 @@ void InstallOpcodeHandlers(Context::OperatingMode Mode) { {0x19, 7, &OpDispatchBuilder::NOPOp}, {0x28, 2, &OpDispatchBuilder::MOVAPSOp}, {0x2A, 1, &OpDispatchBuilder::MMX_To_XMM_Vector_CVT_Int_To_Float<4, true>}, - {0x2B, 1, &OpDispatchBuilder::MOVAPSOp}, + {0x2B, 1, &OpDispatchBuilder::MOVVectorNTOp}, {0x2C, 1, &OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int<8, false>}, {0x2D, 1, &OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int<8, true>}, {0x2E, 2, &OpDispatchBuilder::UCOMISxOp<8>}, @@ -5832,7 +5848,7 @@ void InstallOpcodeHandlers(Context::OperatingMode Mode) { {0xE4, 1, &OpDispatchBuilder::PMULHW}, {0xE5, 1, &OpDispatchBuilder::PMULHW}, {0xE6, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<8, true, false>}, - {0xE7, 1, &OpDispatchBuilder::MOVVectorOp}, + {0xE7, 1, &OpDispatchBuilder::MOVVectorNTOp}, {0xE8, 1, &OpDispatchBuilder::VectorALUOp}, {0xE9, 1, &OpDispatchBuilder::VectorALUOp}, {0xEA, 1, &OpDispatchBuilder::VectorALUOp}, @@ -6475,7 +6491,7 @@ constexpr uint16_t PF_F2 = 3; {OPD(PF_38_66, 0x25), 1, &OpDispatchBuilder::ExtendVectorElements<4, 8, true>}, {OPD(PF_38_66, 0x28), 1, &OpDispatchBuilder::PMULLOp<4, true>}, {OPD(PF_38_66, 0x29), 1, &OpDispatchBuilder::VectorALUOp}, - {OPD(PF_38_66, 0x2A), 1, &OpDispatchBuilder::MOVAPSOp}, + {OPD(PF_38_66, 0x2A), 1, &OpDispatchBuilder::MOVVectorNTOp}, {OPD(PF_38_66, 0x2B), 1, &OpDispatchBuilder::PACKUSOp<4>}, {OPD(PF_38_66, 0x30), 1, &OpDispatchBuilder::ExtendVectorElements<1, 2, false>}, {OPD(PF_38_66, 0x31), 1, &OpDispatchBuilder::ExtendVectorElements<1, 4, false>}, diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index d9c3d230e..31c0c8c98 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -162,7 +162,9 @@ public: void UnhandledOp(OpcodeArgs); template void MOVGPROp(OpcodeArgs); + void MOVGPRNTOp(OpcodeArgs); void MOVVectorOp(OpcodeArgs); + void MOVVectorNTOp(OpcodeArgs); void ALUOp(OpcodeArgs); void INTOp(OpcodeArgs); void SyscallOp(OpcodeArgs); @@ -632,12 +634,22 @@ private: OrderedNode *AppendSegmentOffset(OrderedNode *Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false); + enum class MemoryAccessType { + // Choose TSO or Non-TSO depending on access type + ACCESS_DEFAULT, + // TSO access behaviour + ACCESS_TSO, + // Non-TSO access behaviour + ACCESS_NONTSO, + // Non-temporal streaming + ACCESS_STREAM, + }; OrderedNode *GetRelocatedPC(FEXCore::X86Tables::DecodedOp const& Op, int64_t Offset = 0); - OrderedNode *LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false); - OrderedNode *LoadSource_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false); - void StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, uint8_t OpSize, int8_t Align); - void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align); - void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align); + OrderedNode *LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT); + OrderedNode *LoadSource_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT); + void StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, uint8_t OpSize, int8_t Align, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT); + void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT); + void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT); [[nodiscard]] static uint32_t GPROffset(X86State::X86Reg reg) { LOGMAN_THROW_A_FMT(reg <= X86State::X86Reg::REG_R15, "Invalid reg used"); diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp index 74b47be97..f90681746 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp @@ -28,6 +28,11 @@ void OpDispatchBuilder::MOVVectorOp(OpcodeArgs) { StoreResult(FPRClass, Op, Src, 1); } +void OpDispatchBuilder::MOVVectorNTOp(OpcodeArgs) { + OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, 1, true, false, MemoryAccessType::ACCESS_STREAM); + StoreResult(FPRClass, Op, Src, 1, MemoryAccessType::ACCESS_STREAM); +} + void OpDispatchBuilder::MOVAPSOp(OpcodeArgs) { OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); StoreResult(FPRClass, Op, Src, -1); @@ -1194,7 +1199,8 @@ void OpDispatchBuilder::MASKMOVOp(OpcodeArgs) { { auto DestByte = _Bfe(8, 8 * Select, DestElement); auto MemLocation = _Add(MemDest, _Constant(Element * 8 + Select)); - _StoreMemAutoTSO(GPRClass, 1, MemLocation, DestByte, 1); + // MASKMOVDQU/MASKMOVQ is explicitly weakly-ordered on its store + _StoreMem(GPRClass, 1, MemLocation, DestByte, 1); } auto Jump = _Jump(); auto NextJumpTarget = CreateNewCodeBlockAfter(StoreBlock);