diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index 76aae3128..d4f267b83 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -95,6 +95,7 @@ set (SRCS Interface/Core/ObjectCache/JobHandling.cpp Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp Interface/Core/ObjectCache/ObjectCacheService.cpp + Interface/Core/OpcodeDispatcher/AVX_128.cpp Interface/Core/OpcodeDispatcher/Crypto.cpp Interface/Core/OpcodeDispatcher/Flags.cpp Interface/Core/OpcodeDispatcher/Vector.cpp diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index 84213b9b8..7c4b4285e 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -5415,6 +5415,8 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() { if (CTX->HostFeatures.SupportsSVE256) { InstallToTable(FEXCore::X86Tables::VEXTableOps, AVXTable); InstallToTable(FEXCore::X86Tables::VEXTableGroupOps, VEXTableGroupOps); + } else if (CTX->HostFeatures.SupportsAVX) { + InstallAVX128Handlers(); } if (CTX->HostFeatures.SupportsPMULL_128Bit) { diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index f904990a8..a1e9e84e0 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -973,6 +973,35 @@ public: void UnimplementedOp(OpcodeArgs); void PermissionRestrictedOp(OpcodeArgs); + // AVX 128-bit operations + Ref AVX128_LoadXMMRegister(uint32_t XMM, bool High); + void AVX128_StoreXMMRegister(uint32_t XMM, const Ref Src, bool High); + + struct RefPair { + Ref Low, High; + }; + RefPair AVX128_LoadSource_WithOpSize(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags, + bool NeedsHigh, MemoryAccessType AccessType = MemoryAccessType::DEFAULT); + + void AVX128_StoreResult_WithOpSize(FEXCore::X86Tables::DecodedOp Op, const FEXCore::X86Tables::DecodedOperand& Operand, const RefPair Src, + MemoryAccessType AccessType = MemoryAccessType::DEFAULT); + void InstallAVX128Handlers(); + void AVX128_VMOVScalarImpl(OpcodeArgs, size_t ElementSize); + void AVX128_VectorALUImpl(OpcodeArgs, IROps IROp, size_t ElementSize); + void AVX128_VectorUnaryImpl(OpcodeArgs, IROps IROp, size_t ElementSize); + + void AVX128_VMOVAPS(OpcodeArgs); + void AVX128_VMOVSD(OpcodeArgs); + void AVX128_VMOVSS(OpcodeArgs); + + template + void AVX128_VectorALU(OpcodeArgs); + template + void AVX128_VectorUnary(OpcodeArgs); + + void AVX128_VZERO(OpcodeArgs); + // End of AVX 128-bit implementation + void InvalidOp(OpcodeArgs); void SetPackedRFLAG(bool Lower8, Ref Src); diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp new file mode 100644 index 000000000..67f3ddbf9 --- /dev/null +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: MIT +/* +$info$ +tags: frontend|x86-to-ir, opcodes|dispatcher-implementations +desc: Handles x86/64 AVX instructions to 128-bit IR +$end_info$ +*/ + +#include "Interface/Core/X86Tables/X86Tables.h" + +#include +#include "Interface/Core/OpcodeDispatcher.h" + +#include +#include +#include +#include + +namespace FEXCore::IR { +class OrderedNode; + +#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op + +void OpDispatchBuilder::InstallAVX128Handlers() { +#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode)) + static constexpr std::tuple AVX128Table[] = { + {OPD(1, 0b00, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b01, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b10, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVSS}, + {OPD(1, 0b11, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVSD}, + {OPD(1, 0b00, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b01, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b10, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVSS}, + {OPD(1, 0b11, 0x11), 1, &OpDispatchBuilder::AVX128_VMOVSD}, + + // TODO: {OPD(1, 0b00, 0x12), 1, &OpDispatchBuilder::VMOVLPOp}, + // TODO: {OPD(1, 0b01, 0x12), 1, &OpDispatchBuilder::VMOVLPOp}, + // TODO: {OPD(1, 0b10, 0x12), 1, &OpDispatchBuilder::VMOVSLDUPOp}, + // TODO: {OPD(1, 0b11, 0x12), 1, &OpDispatchBuilder::VMOVDDUPOp}, + // TODO: {OPD(1, 0b00, 0x13), 1, &OpDispatchBuilder::VMOVLPOp}, + // TODO: {OPD(1, 0b01, 0x13), 1, &OpDispatchBuilder::VMOVLPOp}, + + // TODO: {OPD(1, 0b00, 0x14), 1, &OpDispatchBuilder::VPUNPCKLOp<4>}, + // TODO: {OPD(1, 0b01, 0x14), 1, &OpDispatchBuilder::VPUNPCKLOp<8>}, + + // TODO: {OPD(1, 0b00, 0x15), 1, &OpDispatchBuilder::VPUNPCKHOp<4>}, + // TODO: {OPD(1, 0b01, 0x15), 1, &OpDispatchBuilder::VPUNPCKHOp<8>}, + + // TODO: {OPD(1, 0b00, 0x16), 1, &OpDispatchBuilder::VMOVHPOp}, + // TODO: {OPD(1, 0b01, 0x16), 1, &OpDispatchBuilder::VMOVHPOp}, + // TODO: {OPD(1, 0b10, 0x16), 1, &OpDispatchBuilder::VMOVSHDUPOp}, + // TODO: {OPD(1, 0b00, 0x17), 1, &OpDispatchBuilder::VMOVHPOp}, + // TODO: {OPD(1, 0b01, 0x17), 1, &OpDispatchBuilder::VMOVHPOp}, + + {OPD(1, 0b00, 0x28), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b01, 0x28), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b00, 0x29), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b01, 0x29), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + + // TODO: {OPD(1, 0b10, 0x2A), 1, &OpDispatchBuilder::AVXInsertCVTGPR_To_FPR<4>}, + // TODO: {OPD(1, 0b11, 0x2A), 1, &OpDispatchBuilder::AVXInsertCVTGPR_To_FPR<8>}, + + // TODO: {OPD(1, 0b00, 0x2B), 1, &OpDispatchBuilder::MOVVectorNTOp}, + // TODO: {OPD(1, 0b01, 0x2B), 1, &OpDispatchBuilder::MOVVectorNTOp}, + + // TODO: {OPD(1, 0b10, 0x2C), 1, &OpDispatchBuilder::CVTFPR_To_GPR<4, false>}, + // TODO: {OPD(1, 0b11, 0x2C), 1, &OpDispatchBuilder::CVTFPR_To_GPR<8, false>}, + + // TODO: {OPD(1, 0b10, 0x2D), 1, &OpDispatchBuilder::CVTFPR_To_GPR<4, true>}, + // TODO: {OPD(1, 0b11, 0x2D), 1, &OpDispatchBuilder::CVTFPR_To_GPR<8, true>}, + + // TODO: {OPD(1, 0b00, 0x2E), 1, &OpDispatchBuilder::UCOMISxOp<4>}, + // TODO: {OPD(1, 0b01, 0x2E), 1, &OpDispatchBuilder::UCOMISxOp<8>}, + // TODO: {OPD(1, 0b00, 0x2F), 1, &OpDispatchBuilder::UCOMISxOp<4>}, + // TODO: {OPD(1, 0b01, 0x2F), 1, &OpDispatchBuilder::UCOMISxOp<8>}, + + // TODO: {OPD(1, 0b00, 0x50), 1, &OpDispatchBuilder::MOVMSKOp<4>}, + // TODO: {OPD(1, 0b01, 0x50), 1, &OpDispatchBuilder::MOVMSKOp<8>}, + + {OPD(1, 0b00, 0x51), 1, &OpDispatchBuilder::AVX128_VectorUnary}, + {OPD(1, 0b01, 0x51), 1, &OpDispatchBuilder::AVX128_VectorUnary}, + // TODO: {OPD(1, 0b10, 0x51), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp}, + // TODO: {OPD(1, 0b11, 0x51), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp}, + + {OPD(1, 0b00, 0x52), 1, &OpDispatchBuilder::AVX128_VectorUnary}, + // TODO: {OPD(1, 0b10, 0x52), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp}, + + {OPD(1, 0b00, 0x53), 1, &OpDispatchBuilder::AVX128_VectorUnary}, + // TODO: {OPD(1, 0b10, 0x53), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp}, + + {OPD(1, 0b00, 0x54), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x54), 1, &OpDispatchBuilder::AVX128_VectorALU}, + + // TODO: {OPD(1, 0b00, 0x55), 1, &OpDispatchBuilder::VANDNOp}, + // TODO: {OPD(1, 0b01, 0x55), 1, &OpDispatchBuilder::VANDNOp}, + + {OPD(1, 0b00, 0x56), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x56), 1, &OpDispatchBuilder::AVX128_VectorALU}, + + {OPD(1, 0b00, 0x57), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x57), 1, &OpDispatchBuilder::AVX128_VectorALU}, + + {OPD(1, 0b00, 0x58), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x58), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b10, 0x58), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + // TODO: {OPD(1, 0b11, 0x58), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + + {OPD(1, 0b00, 0x59), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x59), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b10, 0x59), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + // TODO: {OPD(1, 0b11, 0x59), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + + // TODO: {OPD(1, 0b00, 0x5A), 1, &OpDispatchBuilder::AVXVector_CVT_Float_To_Float<8, 4>}, + // TODO: {OPD(1, 0b01, 0x5A), 1, &OpDispatchBuilder::AVXVector_CVT_Float_To_Float<4, 8>}, + // TODO: {OPD(1, 0b10, 0x5A), 1, &OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float<8, 4>}, + // TODO: {OPD(1, 0b11, 0x5A), 1, &OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float<4, 8>}, + + // TODO: {OPD(1, 0b00, 0x5B), 1, &OpDispatchBuilder::AVXVector_CVT_Int_To_Float<4, false>}, + // TODO: {OPD(1, 0b01, 0x5B), 1, &OpDispatchBuilder::AVXVector_CVT_Float_To_Int<4, false, true>}, + // TODO: {OPD(1, 0b10, 0x5B), 1, &OpDispatchBuilder::AVXVector_CVT_Float_To_Int<4, false, false>}, + + {OPD(1, 0b00, 0x5C), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x5C), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b10, 0x5C), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + // TODO: {OPD(1, 0b11, 0x5C), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + + {OPD(1, 0b00, 0x5D), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x5D), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b10, 0x5D), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + // TODO: {OPD(1, 0b11, 0x5D), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + + {OPD(1, 0b00, 0x5E), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x5E), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b10, 0x5E), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + // TODO: {OPD(1, 0b11, 0x5E), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + + {OPD(1, 0b00, 0x5F), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x5F), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b10, 0x5F), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + // TODO: {OPD(1, 0b11, 0x5F), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp}, + + // TODO: {OPD(1, 0b01, 0x60), 1, &OpDispatchBuilder::VPUNPCKLOp<1>}, + // TODO: {OPD(1, 0b01, 0x61), 1, &OpDispatchBuilder::VPUNPCKLOp<2>}, + // TODO: {OPD(1, 0b01, 0x62), 1, &OpDispatchBuilder::VPUNPCKLOp<4>}, + // TODO: {OPD(1, 0b01, 0x63), 1, &OpDispatchBuilder::VPACKSSOp<2>}, + {OPD(1, 0b01, 0x64), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x65), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x66), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b01, 0x67), 1, &OpDispatchBuilder::VPACKUSOp<2>}, + // TODO: {OPD(1, 0b01, 0x68), 1, &OpDispatchBuilder::VPUNPCKHOp<1>}, + // TODO: {OPD(1, 0b01, 0x69), 1, &OpDispatchBuilder::VPUNPCKHOp<2>}, + // TODO: {OPD(1, 0b01, 0x6A), 1, &OpDispatchBuilder::VPUNPCKHOp<4>}, + // TODO: {OPD(1, 0b01, 0x6B), 1, &OpDispatchBuilder::VPACKSSOp<4>}, + // TODO: {OPD(1, 0b01, 0x6C), 1, &OpDispatchBuilder::VPUNPCKLOp<8>}, + // TODO: {OPD(1, 0b01, 0x6D), 1, &OpDispatchBuilder::VPUNPCKHOp<8>}, + // TODO: {OPD(1, 0b01, 0x6E), 1, &OpDispatchBuilder::MOVBetweenGPR_FPR}, + + {OPD(1, 0b01, 0x6F), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b10, 0x6F), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + + // TODO: {OPD(1, 0b01, 0x70), 1, &OpDispatchBuilder::VPSHUFWOp<4, true>}, + // TODO: {OPD(1, 0b10, 0x70), 1, &OpDispatchBuilder::VPSHUFWOp<2, false>}, + // TODO: {OPD(1, 0b11, 0x70), 1, &OpDispatchBuilder::VPSHUFWOp<2, true>}, + + {OPD(1, 0b01, 0x74), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x75), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0x76), 1, &OpDispatchBuilder::AVX128_VectorALU}, + + {OPD(1, 0b00, 0x77), 1, &OpDispatchBuilder::AVX128_VZERO}, + + // TODO: {OPD(1, 0b01, 0x7C), 1, &OpDispatchBuilder::VHADDPOp}, + // TODO: {OPD(1, 0b11, 0x7C), 1, &OpDispatchBuilder::VHADDPOp}, + // TODO: {OPD(1, 0b01, 0x7D), 1, &OpDispatchBuilder::VHSUBPOp<8>}, + // TODO: {OPD(1, 0b11, 0x7D), 1, &OpDispatchBuilder::VHSUBPOp<4>}, + + // TODO: {OPD(1, 0b01, 0x7E), 1, &OpDispatchBuilder::MOVBetweenGPR_FPR}, + // TODO: {OPD(1, 0b10, 0x7E), 1, &OpDispatchBuilder::MOVQOp}, + + {OPD(1, 0b01, 0x7F), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + {OPD(1, 0b10, 0x7F), 1, &OpDispatchBuilder::AVX128_VMOVAPS}, + + // TODO: {OPD(1, 0b00, 0xC2), 1, &OpDispatchBuilder::AVXVFCMPOp<4>}, + // TODO: {OPD(1, 0b01, 0xC2), 1, &OpDispatchBuilder::AVXVFCMPOp<8>}, + // TODO: {OPD(1, 0b10, 0xC2), 1, &OpDispatchBuilder::AVXInsertScalarFCMPOp<4>}, + // TODO: {OPD(1, 0b11, 0xC2), 1, &OpDispatchBuilder::AVXInsertScalarFCMPOp<8>}, + + // TODO: {OPD(1, 0b01, 0xC4), 1, &OpDispatchBuilder::VPINSRWOp}, + // TODO: {OPD(1, 0b01, 0xC5), 1, &OpDispatchBuilder::PExtrOp<2>}, + + // TODO: {OPD(1, 0b00, 0xC6), 1, &OpDispatchBuilder::VSHUFOp<4>}, + // TODO: {OPD(1, 0b01, 0xC6), 1, &OpDispatchBuilder::VSHUFOp<8>}, + + // TODO: {OPD(1, 0b01, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp<8>}, + // TODO: {OPD(1, 0b11, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp<4>}, + + // TODO: {OPD(1, 0b01, 0xD1), 1, &OpDispatchBuilder::VPSRLDOp<2>}, + // TODO: {OPD(1, 0b01, 0xD2), 1, &OpDispatchBuilder::VPSRLDOp<4>}, + // TODO: {OPD(1, 0b01, 0xD3), 1, &OpDispatchBuilder::VPSRLDOp<8>}, + {OPD(1, 0b01, 0xD4), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xD5), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b01, 0xD6), 1, &OpDispatchBuilder::MOVQOp}, + // TODO: {OPD(1, 0b01, 0xD7), 1, &OpDispatchBuilder::MOVMSKOpOne}, + + {OPD(1, 0b01, 0xD8), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xD9), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xDA), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xDB), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xDC), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xDD), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xDE), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b01, 0xDF), 1, &OpDispatchBuilder::VANDNOp}, + + {OPD(1, 0b01, 0xE0), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b01, 0xE1), 1, &OpDispatchBuilder::VPSRAOp<2>}, + // TODO: {OPD(1, 0b01, 0xE2), 1, &OpDispatchBuilder::VPSRAOp<4>}, + {OPD(1, 0b01, 0xE3), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(1, 0b01, 0xE4), 1, &OpDispatchBuilder::VPMULHWOp}, + // TODO: {OPD(1, 0b01, 0xE5), 1, &OpDispatchBuilder::VPMULHWOp}, + + // TODO: {OPD(1, 0b01, 0xE6), 1, &OpDispatchBuilder::AVXVector_CVT_Float_To_Int<8, true, false>}, + // TODO: {OPD(1, 0b10, 0xE6), 1, &OpDispatchBuilder::AVXVector_CVT_Int_To_Float<4, true>}, + // TODO: {OPD(1, 0b11, 0xE6), 1, &OpDispatchBuilder::AVXVector_CVT_Float_To_Int<8, true, true>}, + + // TODO: {OPD(1, 0b01, 0xE7), 1, &OpDispatchBuilder::MOVVectorNTOp}, + + {OPD(1, 0b01, 0xE8), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xE9), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xEA), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xEB), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xEC), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xED), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xEE), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xEF), 1, &OpDispatchBuilder::AVX128_VectorALU}, + + // TODO: {OPD(1, 0b11, 0xF0), 1, &OpDispatchBuilder::MOVVectorUnalignedOp}, + // TODO: {OPD(1, 0b01, 0xF1), 1, &OpDispatchBuilder::VPSLLOp<2>}, + // TODO: {OPD(1, 0b01, 0xF2), 1, &OpDispatchBuilder::VPSLLOp<4>}, + // TODO: {OPD(1, 0b01, 0xF3), 1, &OpDispatchBuilder::VPSLLOp<8>}, + // TODO: {OPD(1, 0b01, 0xF4), 1, &OpDispatchBuilder::VPMULLOp<4, false>}, + // TODO: {OPD(1, 0b01, 0xF5), 1, &OpDispatchBuilder::VPMADDWDOp}, + // TODO: {OPD(1, 0b01, 0xF6), 1, &OpDispatchBuilder::VPSADBWOp}, + // TODO: {OPD(1, 0b01, 0xF7), 1, &OpDispatchBuilder::MASKMOVOp}, + + {OPD(1, 0b01, 0xF8), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xF9), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xFA), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xFB), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xFC), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xFD), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(1, 0b01, 0xFE), 1, &OpDispatchBuilder::AVX128_VectorALU}, + + // TODO: {OPD(2, 0b01, 0x00), 1, &OpDispatchBuilder::VPSHUFBOp}, + // TODO: {OPD(2, 0b01, 0x01), 1, &OpDispatchBuilder::VHADDPOp}, + // TODO: {OPD(2, 0b01, 0x02), 1, &OpDispatchBuilder::VHADDPOp}, + // TODO: {OPD(2, 0b01, 0x03), 1, &OpDispatchBuilder::VPHADDSWOp}, + // TODO: {OPD(2, 0b01, 0x04), 1, &OpDispatchBuilder::VPMADDUBSWOp}, + + // TODO: {OPD(2, 0b01, 0x05), 1, &OpDispatchBuilder::VPHSUBOp<2>}, + // TODO: {OPD(2, 0b01, 0x06), 1, &OpDispatchBuilder::VPHSUBOp<4>}, + // TODO: {OPD(2, 0b01, 0x07), 1, &OpDispatchBuilder::VPHSUBSWOp}, + + // TODO: {OPD(2, 0b01, 0x08), 1, &OpDispatchBuilder::VPSIGN<1>}, + // TODO: {OPD(2, 0b01, 0x09), 1, &OpDispatchBuilder::VPSIGN<2>}, + // TODO: {OPD(2, 0b01, 0x0A), 1, &OpDispatchBuilder::VPSIGN<4>}, + // TODO: {OPD(2, 0b01, 0x0B), 1, &OpDispatchBuilder::VPMULHRSWOp}, + // TODO: {OPD(2, 0b01, 0x0C), 1, &OpDispatchBuilder::VPERMILRegOp<4>}, + // TODO: {OPD(2, 0b01, 0x0D), 1, &OpDispatchBuilder::VPERMILRegOp<8>}, + // TODO: {OPD(2, 0b01, 0x0E), 1, &OpDispatchBuilder::VTESTPOp<4>}, + // TODO: {OPD(2, 0b01, 0x0F), 1, &OpDispatchBuilder::VTESTPOp<8>}, + + // TODO: {OPD(2, 0b01, 0x16), 1, &OpDispatchBuilder::VPERMDOp}, + // TODO: {OPD(2, 0b01, 0x17), 1, &OpDispatchBuilder::PTestOp}, + // TODO: {OPD(2, 0b01, 0x18), 1, &OpDispatchBuilder::VBROADCASTOp<4>}, + // TODO: {OPD(2, 0b01, 0x19), 1, &OpDispatchBuilder::VBROADCASTOp<8>}, + // TODO: {OPD(2, 0b01, 0x1A), 1, &OpDispatchBuilder::VBROADCASTOp<16>}, + {OPD(2, 0b01, 0x1C), 1, &OpDispatchBuilder::AVX128_VectorUnary}, + {OPD(2, 0b01, 0x1D), 1, &OpDispatchBuilder::AVX128_VectorUnary}, + {OPD(2, 0b01, 0x1E), 1, &OpDispatchBuilder::AVX128_VectorUnary}, + + // TODO: {OPD(2, 0b01, 0x20), 1, &OpDispatchBuilder::ExtendVectorElements<1, 2, true>}, + // TODO: {OPD(2, 0b01, 0x21), 1, &OpDispatchBuilder::ExtendVectorElements<1, 4, true>}, + // TODO: {OPD(2, 0b01, 0x22), 1, &OpDispatchBuilder::ExtendVectorElements<1, 8, true>}, + // TODO: {OPD(2, 0b01, 0x23), 1, &OpDispatchBuilder::ExtendVectorElements<2, 4, true>}, + // TODO: {OPD(2, 0b01, 0x24), 1, &OpDispatchBuilder::ExtendVectorElements<2, 8, true>}, + // TODO: {OPD(2, 0b01, 0x25), 1, &OpDispatchBuilder::ExtendVectorElements<4, 8, true>}, + + // TODO: {OPD(2, 0b01, 0x28), 1, &OpDispatchBuilder::VPMULLOp<4, true>}, + {OPD(2, 0b01, 0x29), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(2, 0b01, 0x2A), 1, &OpDispatchBuilder::MOVVectorNTOp}, + // TODO: {OPD(2, 0b01, 0x2B), 1, &OpDispatchBuilder::VPACKUSOp<4>}, + // TODO: {OPD(2, 0b01, 0x2C), 1, &OpDispatchBuilder::VMASKMOVOp<4, false>}, + // TODO: {OPD(2, 0b01, 0x2D), 1, &OpDispatchBuilder::VMASKMOVOp<8, false>}, + // TODO: {OPD(2, 0b01, 0x2E), 1, &OpDispatchBuilder::VMASKMOVOp<4, true>}, + // TODO: {OPD(2, 0b01, 0x2F), 1, &OpDispatchBuilder::VMASKMOVOp<8, true>}, + + // TODO: {OPD(2, 0b01, 0x30), 1, &OpDispatchBuilder::ExtendVectorElements<1, 2, false>}, + // TODO: {OPD(2, 0b01, 0x31), 1, &OpDispatchBuilder::ExtendVectorElements<1, 4, false>}, + // TODO: {OPD(2, 0b01, 0x32), 1, &OpDispatchBuilder::ExtendVectorElements<1, 8, false>}, + // TODO: {OPD(2, 0b01, 0x33), 1, &OpDispatchBuilder::ExtendVectorElements<2, 4, false>}, + // TODO: {OPD(2, 0b01, 0x34), 1, &OpDispatchBuilder::ExtendVectorElements<2, 8, false>}, + // TODO: {OPD(2, 0b01, 0x35), 1, &OpDispatchBuilder::ExtendVectorElements<4, 8, false>}, + // TODO: {OPD(2, 0b01, 0x36), 1, &OpDispatchBuilder::VPERMDOp}, + + {OPD(2, 0b01, 0x37), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x38), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x39), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x3A), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x3B), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x3C), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x3D), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x3E), 1, &OpDispatchBuilder::AVX128_VectorALU}, + {OPD(2, 0b01, 0x3F), 1, &OpDispatchBuilder::AVX128_VectorALU}, + + {OPD(2, 0b01, 0x40), 1, &OpDispatchBuilder::AVX128_VectorALU}, + // TODO: {OPD(2, 0b01, 0x41), 1, &OpDispatchBuilder::PHMINPOSUWOp}, + // TODO: {OPD(2, 0b01, 0x45), 1, &OpDispatchBuilder::VPSRLVOp}, + // TODO: {OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::VPSRAVDOp}, + // TODO: {OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::VPSLLVOp}, + + // TODO: {OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::VBROADCASTOp<4>}, + // TODO: {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::VBROADCASTOp<8>}, + // TODO: {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::VBROADCASTOp<16>}, + + // TODO: {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::VBROADCASTOp<1>}, + // TODO: {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::VBROADCASTOp<2>}, + + // TODO: {OPD(2, 0b01, 0x8C), 1, &OpDispatchBuilder::VPMASKMOVOp}, + // TODO: {OPD(2, 0b01, 0x8E), 1, &OpDispatchBuilder::VPMASKMOVOp}, + + // TODO: {OPD(2, 0b01, 0xDB), 1, &OpDispatchBuilder::AESImcOp}, + // TODO: {OPD(2, 0b01, 0xDC), 1, &OpDispatchBuilder::VAESEncOp}, + // TODO: {OPD(2, 0b01, 0xDD), 1, &OpDispatchBuilder::VAESEncLastOp}, + // TODO: {OPD(2, 0b01, 0xDE), 1, &OpDispatchBuilder::VAESDecOp}, + // TODO: {OPD(2, 0b01, 0xDF), 1, &OpDispatchBuilder::VAESDecLastOp}, + + // TODO: {OPD(3, 0b01, 0x00), 1, &OpDispatchBuilder::VPERMQOp}, + // TODO: {OPD(3, 0b01, 0x01), 1, &OpDispatchBuilder::VPERMQOp}, + // TODO: {OPD(3, 0b01, 0x02), 1, &OpDispatchBuilder::VPBLENDDOp}, + // TODO: {OPD(3, 0b01, 0x04), 1, &OpDispatchBuilder::VPERMILImmOp<4>}, + // TODO: {OPD(3, 0b01, 0x05), 1, &OpDispatchBuilder::VPERMILImmOp<8>}, + // TODO: {OPD(3, 0b01, 0x06), 1, &OpDispatchBuilder::VPERM2Op}, + // TODO: {OPD(3, 0b01, 0x08), 1, &OpDispatchBuilder::AVXVectorRound<4>}, + // TODO: {OPD(3, 0b01, 0x09), 1, &OpDispatchBuilder::AVXVectorRound<8>}, + // TODO: {OPD(3, 0b01, 0x0A), 1, &OpDispatchBuilder::AVXInsertScalarRound<4>}, + // TODO: {OPD(3, 0b01, 0x0B), 1, &OpDispatchBuilder::AVXInsertScalarRound<8>}, + // TODO: {OPD(3, 0b01, 0x0C), 1, &OpDispatchBuilder::VPBLENDDOp}, + // TODO: {OPD(3, 0b01, 0x0D), 1, &OpDispatchBuilder::VBLENDPDOp}, + // TODO: {OPD(3, 0b01, 0x0E), 1, &OpDispatchBuilder::VPBLENDWOp}, + // TODO: {OPD(3, 0b01, 0x0F), 1, &OpDispatchBuilder::VPALIGNROp}, + + // TODO: {OPD(3, 0b01, 0x14), 1, &OpDispatchBuilder::PExtrOp<1>}, + // TODO: {OPD(3, 0b01, 0x15), 1, &OpDispatchBuilder::PExtrOp<2>}, + // TODO: {OPD(3, 0b01, 0x16), 1, &OpDispatchBuilder::PExtrOp<4>}, + // TODO: {OPD(3, 0b01, 0x17), 1, &OpDispatchBuilder::PExtrOp<4>}, + + // TODO: {OPD(3, 0b01, 0x18), 1, &OpDispatchBuilder::VINSERTOp}, + // TODO: {OPD(3, 0b01, 0x19), 1, &OpDispatchBuilder::VEXTRACT128Op}, + // TODO: {OPD(3, 0b01, 0x20), 1, &OpDispatchBuilder::VPINSRBOp}, + // TODO: {OPD(3, 0b01, 0x21), 1, &OpDispatchBuilder::VINSERTPSOp}, + // TODO: {OPD(3, 0b01, 0x22), 1, &OpDispatchBuilder::VPINSRDQOp}, + + // TODO: {OPD(3, 0b01, 0x38), 1, &OpDispatchBuilder::VINSERTOp}, + // TODO: {OPD(3, 0b01, 0x39), 1, &OpDispatchBuilder::VEXTRACT128Op}, + + // TODO: {OPD(3, 0b01, 0x40), 1, &OpDispatchBuilder::VDPPOp<4>}, + // TODO: {OPD(3, 0b01, 0x41), 1, &OpDispatchBuilder::VDPPOp<8>}, + // TODO: {OPD(3, 0b01, 0x42), 1, &OpDispatchBuilder::VMPSADBWOp}, + + // TODO: {OPD(3, 0b01, 0x46), 1, &OpDispatchBuilder::VPERM2Op}, + + // TODO: {OPD(3, 0b01, 0x4A), 1, &OpDispatchBuilder::AVXVectorVariableBlend<4>}, + // TODO: {OPD(3, 0b01, 0x4B), 1, &OpDispatchBuilder::AVXVectorVariableBlend<8>}, + // TODO: {OPD(3, 0b01, 0x4C), 1, &OpDispatchBuilder::AVXVectorVariableBlend<1>}, + + // TODO: {OPD(3, 0b01, 0x60), 1, &OpDispatchBuilder::VPCMPESTRMOp}, + // TODO: {OPD(3, 0b01, 0x61), 1, &OpDispatchBuilder::VPCMPESTRIOp}, + // TODO: {OPD(3, 0b01, 0x62), 1, &OpDispatchBuilder::VPCMPISTRMOp}, + // TODO: {OPD(3, 0b01, 0x63), 1, &OpDispatchBuilder::VPCMPISTRIOp}, + + // TODO: {OPD(3, 0b01, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist}, + }; +#undef OPD + +#define OPD(group, pp, opcode) (((group - X86Tables::TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode)) + static constexpr std::tuple VEX128TableGroupOps[] { + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b010), 1, &OpDispatchBuilder::VPSRLIOp<2>}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b110), 1, &OpDispatchBuilder::VPSLLIOp<2>}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b100), 1, &OpDispatchBuilder::VPSRAIOp<2>}, + + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b010), 1, &OpDispatchBuilder::VPSRLIOp<4>}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b110), 1, &OpDispatchBuilder::VPSLLIOp<4>}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b100), 1, &OpDispatchBuilder::VPSRAIOp<4>}, + + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b010), 1, &OpDispatchBuilder::VPSRLIOp<8>}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b011), 1, &OpDispatchBuilder::VPSRLDQOp}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b110), 1, &OpDispatchBuilder::VPSLLIOp<8>}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b111), 1, &OpDispatchBuilder::VPSLLDQOp}, + + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b010), 1, &OpDispatchBuilder::LDMXCSR}, + // TODO: {OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b011), 1, &OpDispatchBuilder::STMXCSR}, + }; +#undef OPD + + auto InstallToTable = [](auto& FinalTable, auto& LocalTable) { + for (auto Op : LocalTable) { + auto OpNum = std::get<0>(Op); + auto Dispatcher = std::get<2>(Op); + for (uint8_t i = 0; i < std::get<1>(Op); ++i) { + LOGMAN_THROW_A_FMT(FinalTable[OpNum + i].OpcodeDispatcher == nullptr, "Duplicate Entry"); + FinalTable[OpNum + i].OpcodeDispatcher = Dispatcher; + } + } + }; + + InstallToTable(FEXCore::X86Tables::VEXTableOps, AVX128Table); + InstallToTable(FEXCore::X86Tables::VEXTableGroupOps, VEX128TableGroupOps); +} + +OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_LoadSource_WithOpSize( + const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags, bool NeedsHigh, MemoryAccessType AccessType) { + + if (Operand.IsGPR()) { + const auto gpr = Operand.Data.GPR.GPR; + LOGMAN_THROW_AA_FMT(gpr >= FEXCore::X86State::REG_XMM_0 && gpr <= FEXCore::X86State::REG_XMM_15, "must be AVX reg"); + const auto gprIndex = gpr - X86State::REG_XMM_0; + return { + .Low = AVX128_LoadXMMRegister(gprIndex, false), + .High = NeedsHigh ? AVX128_LoadXMMRegister(gprIndex, true) : nullptr, + }; + } else { + LOGMAN_THROW_A_FMT(IsOperandMem(Operand, true), "only memory sources"); + + AddressMode A = DecodeAddress(Op, Operand, AccessType, true /* IsLoad */); + A = AddSegmentToAddress(A, Flags); + + AddressMode HighA = A; + HighA.Offset += 16; + + ///< TODO: Implement VSIB once we get there. + if (Operand.IsSIB()) { + const bool IsVSIB = (Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0; + LOGMAN_THROW_AA_FMT(!IsVSIB, "VSIB currently unsupported"); + } + + return { + .Low = _LoadMemAutoTSO(FPRClass, 16, A, 1), + .High = NeedsHigh ? _LoadMemAutoTSO(FPRClass, 16, HighA, 1) : nullptr, + }; + } +} + +void OpDispatchBuilder::AVX128_StoreResult_WithOpSize(FEXCore::X86Tables::DecodedOp Op, const FEXCore::X86Tables::DecodedOperand& Operand, + const RefPair Src, MemoryAccessType AccessType) { + if (Operand.IsGPR()) { + const auto gpr = Operand.Data.GPR.GPR; + LOGMAN_THROW_AA_FMT(gpr >= FEXCore::X86State::REG_XMM_0 && gpr <= FEXCore::X86State::REG_XMM_15, "expected AVX register"); + const auto gprIndex = gpr - X86State::REG_XMM_0; + + if (Src.Low) { + AVX128_StoreXMMRegister(gprIndex, Src.Low, false); + } + + if (Src.High) { + AVX128_StoreXMMRegister(gprIndex, Src.High, true); + } + } else { + AddressMode A = DecodeAddress(Op, Operand, AccessType, false /* IsLoad */); + A = AddSegmentToAddress(A, Op->Flags); + + _StoreMemAutoTSO(FPRClass, 16, A, Src.Low, 1); + + if (Src.High) { + AddressMode HighA = A; + HighA.Offset += 16; + + _StoreMemAutoTSO(FPRClass, 16, HighA, Src.High, 1); + } + } +} + +Ref OpDispatchBuilder::AVX128_LoadXMMRegister(uint32_t XMM, bool High) { + if (High) { + return _LoadContext(16, FPRClass, offsetof(FEXCore::Core::CPUState, avx_high[XMM][0])); + } else { + return _LoadRegister(XMM, FPRClass, 16); + } +} + +void OpDispatchBuilder::AVX128_StoreXMMRegister(uint32_t XMM, const Ref Src, bool High) { + if (High) { + _StoreContext(16, FPRClass, Src, offsetof(FEXCore::Core::CPUState, avx_high[XMM][0])); + } else { + _StoreRegister(Src, XMM, FPRClass, 16); + } +} + +void OpDispatchBuilder::AVX128_VMOVAPS(OpcodeArgs) { + // Reg <- Mem or Reg <- Reg + const auto SrcSize = GetSrcSize(Op); + const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE; + + if (Is128Bit) { + // Zero upper 128-bits + auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false); + + ///< Zero upper bits when destination is GPR. + if (Op->Dest.IsGPR()) { + Src.High = LoadZeroVector(OpSize::i128Bit); + } + AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src); + } else { + // Copy or memory load + auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, true); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, Src); + } +} + +void OpDispatchBuilder::AVX128_VMOVScalarImpl(OpcodeArgs, size_t ElementSize) { + if (Op->Dest.IsGPR() && Op->Src[0].IsGPR() && Op->Src[1].IsGPR()) { + // VMOVSS/SD xmm1, xmm2, xmm3 + // Lower 128-bits are merged + // Upper 128-bits are zero'd + auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false); + auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false); + Ref Result = _VInsElement(16, ElementSize, 0, 0, Src1.Low, Src2.Low); + auto High = LoadZeroVector(OpSize::i128Bit); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result, .High = High}); + } else if (Op->Dest.IsGPR()) { + // VMOVSS/SD xmm1, mem32/mem64 + Ref Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[1], ElementSize, Op->Flags); + auto High = LoadZeroVector(OpSize::i128Bit); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Src, .High = High}); + } else { + // VMOVSS/SD mem32/mem64, xmm1 + auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false); + StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Src.Low, ElementSize, -1); + } +} + +void OpDispatchBuilder::AVX128_VMOVSD(OpcodeArgs) { + AVX128_VMOVScalarImpl(Op, OpSize::i64Bit); +} + +void OpDispatchBuilder::AVX128_VMOVSS(OpcodeArgs) { + AVX128_VMOVScalarImpl(Op, OpSize::i32Bit); +} + +void OpDispatchBuilder::AVX128_VectorALUImpl(OpcodeArgs, IROps IROp, size_t ElementSize) { + const auto SrcSize = GetSrcSize(Op); + const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE; + + auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit); + auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit); + DeriveOp(Result_Low, IROp, _VAdd(16, ElementSize, Src1.Low, Src2.Low)); + + if (Is128Bit) { + auto High = LoadZeroVector(OpSize::i128Bit); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = High}); + } else { + DeriveOp(Result_High, IROp, _VAdd(16, ElementSize, Src1.High, Src2.High)); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = Result_High}); + } +} + +void OpDispatchBuilder::AVX128_VectorUnaryImpl(OpcodeArgs, IROps IROp, size_t ElementSize) { + const auto SrcSize = GetSrcSize(Op); + const auto Is128Bit = SrcSize == Core::CPUState::XMM_SSE_REG_SIZE; + + auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit); + DeriveOp(Result_Low, IROp, _VFSqrt(16, ElementSize, Src.Low)); + + if (Is128Bit) { + auto High = LoadZeroVector(OpSize::i128Bit); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = High}); + } else { + DeriveOp(Result_High, IROp, _VFSqrt(16, ElementSize, Src.High)); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, RefPair {.Low = Result_Low, .High = Result_High}); + } +} + +template +void OpDispatchBuilder::AVX128_VectorALU(OpcodeArgs) { + AVX128_VectorALUImpl(Op, IROp, ElementSize); +} + +template +void OpDispatchBuilder::AVX128_VectorUnary(OpcodeArgs) { + AVX128_VectorUnaryImpl(Op, IROp, ElementSize); +} + +void OpDispatchBuilder::AVX128_VZERO(OpcodeArgs) { + const auto DstSize = GetDstSize(Op); + const auto IsVZEROALL = DstSize == Core::CPUState::XMM_AVX_REG_SIZE; + const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U; + + if (IsVZEROALL) { + // NOTE: Despite the name being VZEROALL, this will still only ever + // zero out up to the first 16 registers (even on AVX-512, where we have 32 registers) + + for (uint32_t i = 0; i < NumRegs; i++) { + // Explicitly not caching named vector zero. This ensures that every register gets movi #0.0 directly. + Ref ZeroVector = LoadUncachedZeroVector(OpSize::i128Bit); + AVX128_StoreXMMRegister(i, ZeroVector, false); + } + + // More efficient for non-SRA upper-halves to cache the constant and store directly. + const auto ZeroVector = LoadZeroVector(OpSize::i128Bit); + for (uint32_t i = 0; i < NumRegs; i++) { + AVX128_StoreXMMRegister(i, ZeroVector, true); + } + } else { + // Likewise, VZEROUPPER will only ever zero only up to the first 16 registers + const auto ZeroVector = LoadZeroVector(OpSize::i128Bit); + for (uint32_t i = 0; i < NumRegs; i++) { + AVX128_StoreXMMRegister(i, ZeroVector, true); + } + } +} + +} // namespace FEXCore::IR