From ab4fb7b3adcd1a3ef80cb2ff34a5d1457efc1d50 Mon Sep 17 00:00:00 2001 From: LC Date: Fri, 3 Jul 2026 17:19:54 -0400 Subject: [PATCH] [SVE256] Ensure insertion behavior for AES operations on SSE These slipped through, so now we can add tests for them to prevent that from happening again. --- .../Source/Interface/Core/OpcodeDispatcher.h | 4 +-- .../Core/OpcodeDispatcher/Crypto.cpp | 26 +++++++++++++------ .../Core/OpcodeDispatcher/H0F38Tables.h | 2 +- .../Core/OpcodeDispatcher/H0F3ATables.h | 2 +- .../Interface/Core/X86Tables/VEXTables.cpp | 4 +-- unittests/ASM/SSELanePreservation/aesdec.asm | 25 ++++++++++++++++++ .../ASM/SSELanePreservation/aesdeclast.asm | 25 ++++++++++++++++++ unittests/ASM/SSELanePreservation/aesenc.asm | 25 ++++++++++++++++++ .../ASM/SSELanePreservation/aesenclast.asm | 25 ++++++++++++++++++ unittests/ASM/SSELanePreservation/aesimc.asm | 25 ++++++++++++++++++ .../SSELanePreservation/aeskeygenassist.asm | 23 ++++++++++++++++ 11 files changed, 172 insertions(+), 14 deletions(-) create mode 100644 unittests/ASM/SSELanePreservation/aesdec.asm create mode 100644 unittests/ASM/SSELanePreservation/aesdeclast.asm create mode 100644 unittests/ASM/SSELanePreservation/aesenc.asm create mode 100644 unittests/ASM/SSELanePreservation/aesenclast.asm create mode 100644 unittests/ASM/SSELanePreservation/aesimc.asm create mode 100644 unittests/ASM/SSELanePreservation/aeskeygenassist.asm diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index 482ccce8a..e2239ae0e 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -839,12 +839,12 @@ public: void SHA256MSG2Op(OpcodeArgs); void SHA256RNDS2Op(OpcodeArgs); - void AESImcOp(OpcodeArgs); + void AESImcOp(OpcodeArgs, bool IsAVX); void AESEncOp(OpcodeArgs); void AESEncLastOp(OpcodeArgs); void AESDecOp(OpcodeArgs); void AESDecLastOp(OpcodeArgs); - void AESKeyGenAssist(OpcodeArgs); + void AESKeyGenAssist(OpcodeArgs, bool IsAVX); void VFMAImpl(OpcodeArgs, IROps IROp, bool Scalar, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx); void VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx); diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp index 1cd2948d7..bce65da54 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp @@ -180,14 +180,19 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) { StoreResultFPR(Op, Result); } -void OpDispatchBuilder::AESImcOp(OpcodeArgs) { +void OpDispatchBuilder::AESImcOp(OpcodeArgs, bool IsAVX) { if (!CTX->HostFeatures.SupportsAES) { UnimplementedOp(Op); return; } Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags); Ref Result = _VAESImc(Src); - StoreResultFPR(Op, Result); + + if (IsAVX) { + StoreResultFPR(Op, Result); + } else { + StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result); + } } void OpDispatchBuilder::AESEncOp(OpcodeArgs) { @@ -198,7 +203,7 @@ void OpDispatchBuilder::AESEncOp(OpcodeArgs) { Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags); Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags); Ref Result = _VAESEnc(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit)); - StoreResultFPR(Op, Result); + StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result); } void OpDispatchBuilder::VAESEncOp(OpcodeArgs) { @@ -223,7 +228,7 @@ void OpDispatchBuilder::AESEncLastOp(OpcodeArgs) { Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags); Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags); Ref Result = _VAESEncLast(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit)); - StoreResultFPR(Op, Result); + StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result); } void OpDispatchBuilder::VAESEncLastOp(OpcodeArgs) { @@ -248,7 +253,7 @@ void OpDispatchBuilder::AESDecOp(OpcodeArgs) { Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags); Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags); Ref Result = _VAESDec(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit)); - StoreResultFPR(Op, Result); + StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result); } void OpDispatchBuilder::VAESDecOp(OpcodeArgs) { @@ -273,7 +278,7 @@ void OpDispatchBuilder::AESDecLastOp(OpcodeArgs) { Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags); Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags); Ref Result = _VAESDecLast(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit)); - StoreResultFPR(Op, Result); + StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result); } void OpDispatchBuilder::VAESDecLastOp(OpcodeArgs) { @@ -298,14 +303,19 @@ Ref OpDispatchBuilder::AESKeyGenAssistImpl(OpcodeArgs) { return _VAESKeyGenAssist(Src, KeyGenSwizzle, LoadZeroVector(OpSize::i128Bit), RCON); } -void OpDispatchBuilder::AESKeyGenAssist(OpcodeArgs) { +void OpDispatchBuilder::AESKeyGenAssist(OpcodeArgs, bool IsAVX) { if (!CTX->HostFeatures.SupportsAES) { UnimplementedOp(Op); return; } Ref Result = AESKeyGenAssistImpl(Op); - StoreResultFPR(Op, Result); + + if (IsAVX) { + StoreResultFPR(Op, Result); + } else { + StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result); + } } void OpDispatchBuilder::PCLMULQDQOp(OpcodeArgs) { diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F38Tables.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F38Tables.h index 7a6487fe3..e001432ad 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F38Tables.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F38Tables.h @@ -79,7 +79,7 @@ constexpr DispatchTableEntry OpDispatch_H0F38Table[] = { {OPD(PF_38_NONE, 0xCC), 1, &OpDispatchBuilder::SHA256MSG1Op}, {OPD(PF_38_NONE, 0xCD), 1, &OpDispatchBuilder::SHA256MSG2Op}, - {OPD(PF_38_66, 0xDB), 1, &OpDispatchBuilder::AESImcOp}, + {OPD(PF_38_66, 0xDB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AESImcOp, false>}, {OPD(PF_38_66, 0xDC), 1, &OpDispatchBuilder::AESEncOp}, {OPD(PF_38_66, 0xDD), 1, &OpDispatchBuilder::AESEncLastOp}, {OPD(PF_38_66, 0xDE), 1, &OpDispatchBuilder::AESDecOp}, diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F3ATables.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F3ATables.h index fb200ad3f..d3152c9b1 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F3ATables.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F3ATables.h @@ -37,7 +37,7 @@ constexpr auto OpDispatchTableGenH0F3A = []() consteval { {OPD(REX, PF_3A_66, 0x63), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPCMPISTRIOp, false>}, {OPD(REX, PF_3A_NONE, 0xCC), 1, &OpDispatchBuilder::SHA1RNDS4Op}, - {OPD(REX, PF_3A_66, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist}, + {OPD(REX, PF_3A_66, 0xDF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AESKeyGenAssist, false>}, }; return std::to_array(Table); diff --git a/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp b/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp index da099f286..4ecfe5658 100644 --- a/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp +++ b/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp @@ -808,7 +808,7 @@ namespace AVX256 { {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD - {OPD(2, 0b01, 0xDB), 1, &OpDispatchBuilder::AESImcOp}, + {OPD(2, 0b01, 0xDB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AESImcOp, true>}, {OPD(2, 0b01, 0xDC), 1, &OpDispatchBuilder::VAESEncOp}, {OPD(2, 0b01, 0xDD), 1, &OpDispatchBuilder::VAESEncLastOp}, {OPD(2, 0b01, 0xDE), 1, &OpDispatchBuilder::VAESDecOp}, @@ -860,7 +860,7 @@ namespace AVX256 { {OPD(3, 0b01, 0x62), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPCMPISTRMOp, true>}, {OPD(3, 0b01, 0x63), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPCMPISTRIOp, true>}, - {OPD(3, 0b01, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist}, + {OPD(3, 0b01, 0xDF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AESKeyGenAssist, true>}, }; #undef OPD diff --git a/unittests/ASM/SSELanePreservation/aesdec.asm b/unittests/ASM/SSELanePreservation/aesdec.asm new file mode 100644 index 000000000..77368c5c5 --- /dev/null +++ b/unittests/ASM/SSELanePreservation/aesdec.asm @@ -0,0 +1,25 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AES", "AVX"], + "RegData": { + "XMM0": ["0x0871fe8ffd3f45ff", "0xf85579bd6f9b3f24", "0xccdbcfc31f3ff0f3", "0x108390defebac4be"], + "XMM2": ["0x0871fe8ffd3f45ff", "0xf85579bd6f9b3f24", "0", "0"] + } +} +%endif + +vmovapd ymm0, [rel .data] +vmovapd ymm1, [rel .data + 32] +vmovapd ymm2, [rel .data] +vmovapd ymm3, [rel .data + 32] + +aesdec xmm0, xmm1 +vaesdec xmm2, xmm2, xmm3 + +hlt + +align 32 +.data: +dq 0xfdecd28fab3fa4a5, 0x7d7ccd8836d09fc2, 0xccdbcfc31f3ff0f3, 0x108390defebac4be +dq 0x43cc1ad6970b4549, 0xbd7eb46a1278f793, 0xef673dac6e4cbb7b, 0x5b3d85d342718be9 + diff --git a/unittests/ASM/SSELanePreservation/aesdeclast.asm b/unittests/ASM/SSELanePreservation/aesdeclast.asm new file mode 100644 index 000000000..6e05f59d4 --- /dev/null +++ b/unittests/ASM/SSELanePreservation/aesdeclast.asm @@ -0,0 +1,25 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AES", "AVX"], + "RegData": { + "XMM0": ["0x67cd07a5b66bc560", "0xb3fddafd015d883b", "0xccdbcfc31f3ff0f3", "0x108390defebac4be"], + "XMM2": ["0x67cd07a5b66bc560", "0xb3fddafd015d883b", "0", "0"] + } +} +%endif + +vmovapd ymm0, [rel .data] +vmovapd ymm1, [rel .data + 32] +vmovapd ymm2, [rel .data] +vmovapd ymm3, [rel .data + 32] + +aesdeclast xmm0, xmm1 +vaesdeclast xmm2, xmm2, xmm3 + +hlt + +align 32 +.data: +dq 0xfdecd28fab3fa4a5, 0x7d7ccd8836d09fc2, 0xccdbcfc31f3ff0f3, 0x108390defebac4be +dq 0x43cc1ad6970b4549, 0xbd7eb46a1278f793, 0xef673dac6e4cbb7b, 0x5b3d85d342718be9 + diff --git a/unittests/ASM/SSELanePreservation/aesenc.asm b/unittests/ASM/SSELanePreservation/aesenc.asm new file mode 100644 index 000000000..f1128e86d --- /dev/null +++ b/unittests/ASM/SSELanePreservation/aesenc.asm @@ -0,0 +1,25 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AES", "AVX"], + "RegData": { + "XMM0": ["0xd9e29634bd425d0e", "0x677baee91df67824", "0xccdbcfc31f3ff0f3", "0x108390defebac4be"], + "XMM2": ["0xd9e29634bd425d0e", "0x677baee91df67824", "0", "0"] + } +} +%endif + +vmovapd ymm0, [rel .data] +vmovapd ymm1, [rel .data + 32] +vmovapd ymm2, [rel .data] +vmovapd ymm3, [rel .data + 32] + +aesenc xmm0, xmm1 +vaesenc xmm2, xmm2, xmm3 + +hlt + +align 32 +.data: +dq 0xfdecd28fab3fa4a5, 0x7d7ccd8836d09fc2, 0xccdbcfc31f3ff0f3, 0x108390defebac4be +dq 0x43cc1ad6970b4549, 0xbd7eb46a1278f793, 0xef673dac6e4cbb7b, 0x5b3d85d342718be9 + diff --git a/unittests/ASM/SSELanePreservation/aesenclast.asm b/unittests/ASM/SSELanePreservation/aesenclast.asm new file mode 100644 index 000000000..5570371c9 --- /dev/null +++ b/unittests/ASM/SSELanePreservation/aesenclast.asm @@ -0,0 +1,25 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AES", "AVX"], + "RegData": { + "XMM0": ["0x21dcc1a5687bf04f", "0xb8b0fdae460d4ab6", "0xccdbcfc31f3ff0f3", "0x108390defebac4be"], + "XMM2": ["0x21dcc1a5687bf04f", "0xb8b0fdae460d4ab6", "0", "0"] + } +} +%endif + +vmovapd ymm0, [rel .data] +vmovapd ymm1, [rel .data + 32] +vmovapd ymm2, [rel .data] +vmovapd ymm3, [rel .data + 32] + +aesenclast xmm0, xmm1 +vaesenclast xmm2, xmm2, xmm3 + +hlt + +align 32 +.data: +dq 0xfdecd28fab3fa4a5, 0x7d7ccd8836d09fc2, 0xccdbcfc31f3ff0f3, 0x108390defebac4be +dq 0x43cc1ad6970b4549, 0xbd7eb46a1278f793, 0xef673dac6e4cbb7b, 0x5b3d85d342718be9 + diff --git a/unittests/ASM/SSELanePreservation/aesimc.asm b/unittests/ASM/SSELanePreservation/aesimc.asm new file mode 100644 index 000000000..c58a76882 --- /dev/null +++ b/unittests/ASM/SSELanePreservation/aesimc.asm @@ -0,0 +1,25 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AES", "AVX"], + "RegData": { + "XMM0": ["0x60ccbf502947c03e", "0xd946ea68cc5335a4", "0xccdbcfc31f3ff0f3", "0x108390defebac4be"], + "XMM2": ["0x60ccbf502947c03e", "0xd946ea68cc5335a4", "0", "0"] + } +} +%endif + +vmovapd ymm0, [rel .data] +vmovapd ymm1, [rel .data + 32] +vmovapd ymm2, [rel .data] +vmovapd ymm3, [rel .data + 32] + +aesimc xmm0, xmm1 +vaesimc xmm2, xmm3 + +hlt + +align 32 +.data: +dq 0xfdecd28fab3fa4a5, 0x7d7ccd8836d09fc2, 0xccdbcfc31f3ff0f3, 0x108390defebac4be +dq 0x43cc1ad6970b4549, 0xbd7eb46a1278f793, 0xef673dac6e4cbb7b, 0x5b3d85d342718be9 + diff --git a/unittests/ASM/SSELanePreservation/aeskeygenassist.asm b/unittests/ASM/SSELanePreservation/aeskeygenassist.asm new file mode 100644 index 000000000..690cf449c --- /dev/null +++ b/unittests/ASM/SSELanePreservation/aeskeygenassist.asm @@ -0,0 +1,23 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AES", "AVX"], + "RegData": { + "XMM0": ["0xf61a4ba21a4ba2f6", "0x027af38d7af38d02", "0xccdbcfc31f3ff0f3", "0x108390defebac4be"], + "XMM2": ["0xf61a4ba21a4ba2f6", "0x027af38d7af38d02", "0", "0"] + } +} +%endif + +vmovapd ymm0, [rel .data] +vmovapd ymm2, [rel .data] + +aeskeygenassist xmm0, [rel .data + 32], 0 +vaeskeygenassist xmm2, [rel .data + 32], 0 + +hlt + +align 32 +.data: +dq 0xfdecd28fab3fa4a5, 0x7d7ccd8836d09fc2, 0xccdbcfc31f3ff0f3, 0x108390defebac4be +dq 0x43cc1ad6970b4549, 0xbd7eb46a1278f793, 0xef673dac6e4cbb7b, 0x5b3d85d342718be9 +