diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index bb1fe4251..f106bbc7a 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -1357,16 +1357,18 @@ DEF_OP(VURAvg) { if (HostSupportsSVE256 && Is256Bit) { const auto Mask = PRED_TMP_32B.Merging(); + // Trivial cases where we already have source data to be averaged in + // the destination register. We can just do the operation in place. if (Dst == Vector1) { - // Trivial case where we already have source data to be averaged in - // the destination register. We can just do the operation in place. urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z()); + } else if (Dst == Vector2) { + urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector1.Z()); } else { - // SVE URHADD is a destructive operation, so we need - // a temporary for performing operations. - movprfx(VTMP1.Z(), Vector1.Z()); - urhadd(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), Vector2.Z()); - mov(Dst.Z(), VTMP1.Z()); + // SVE URHADD is a destructive operation, but we know that + // we don't have any source/destination aliasing happening here + // so we can safely move one of the source operands into the destination. + movprfx(Dst.Z(), Vector1.Z()); + urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z()); } } else { urhadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q()); diff --git a/unittests/ASM/VEX/vpavgb_aliasing.asm b/unittests/ASM/VEX/vpavgb_aliasing.asm new file mode 100644 index 000000000..f7ab624a8 --- /dev/null +++ b/unittests/ASM/VEX/vpavgb_aliasing.asm @@ -0,0 +1,42 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AVX"], + "RegData": { + "XMM10": ["0x2179B0697D5378C4", "0x3B8E6EAE8C165248", "0x2179B0697D5378C4", "0x3B8E6EAE8C165248"], + "XMM11": ["0x1ED68638699D35CA", "0x5E2E7560AB7B5262", "0x1ED68638699D35CA", "0x5E2E7560AB7B5262"] + } +} +%endif + +; Small test that ensures aliasing source/dest is handled properly. + +lea rdx, [rel .data] + +vmovapd ymm6, [rdx + 32] +vmovapd ymm7, [rdx] + +; 256-bit register only +vmovapd ymm10, ymm7 +vpavgb ymm10, ymm10, ymm6 + +vmovapd ymm11, [rdx + 64] +vpavgb ymm11, ymm7, ymm11 + +hlt + +align 32 +.data: +dq 0x2BB883523D4F3197 +dq 0x1246C77764260189 +dq 0x2BB883523D4F3197 +dq 0x1246C77764260189 + +dq 0x163ADD80BC57BEF1 +dq 0x64D615E5B405A306 +dq 0x163ADD80BC57BEF1 +dq 0x64D615E5B405A306 + +dq 0x11F4881D94EB39FC +dq 0xA9162248F2D0A23A +dq 0x11F4881D94EB39FC +dq 0xA9162248F2D0A23A diff --git a/unittests/ASM/VEX/vpavgw_aliasing.asm b/unittests/ASM/VEX/vpavgw_aliasing.asm new file mode 100644 index 000000000..6d3bcfa76 --- /dev/null +++ b/unittests/ASM/VEX/vpavgw_aliasing.asm @@ -0,0 +1,42 @@ +%ifdef CONFIG +{ + "HostFeatures": ["AVX"], + "RegData": { + "XMM10": ["0x20F9B0697CD37844", "0x3B8E6EAE8C165248", "0x20F9B0697CD37844", "0x3B8E6EAE8C165248"], + "XMM11": ["0x1ED685B8691D35CA", "0x5DAE74E0AB7B51E2", "0x1ED685B8691D35CA", "0x5DAE74E0AB7B51E2"] + } +} +%endif + +; Small test that ensures aliasing source/dest is handled properly. + +lea rdx, [rel .data] + +vmovapd ymm6, [rdx + 32] +vmovapd ymm7, [rdx] + +; 256-bit register only +vmovapd ymm10, ymm7 +vpavgw ymm10, ymm10, ymm6 + +vmovapd ymm11, [rdx + 64] +vpavgw ymm11, ymm7, ymm11 + +hlt + +align 32 +.data: +dq 0x2BB883523D4F3197 +dq 0x1246C77764260189 +dq 0x2BB883523D4F3197 +dq 0x1246C77764260189 + +dq 0x163ADD80BC57BEF1 +dq 0x64D615E5B405A306 +dq 0x163ADD80BC57BEF1 +dq 0x64D615E5B405A306 + +dq 0x11F4881D94EB39FC +dq 0xA9162248F2D0A23A +dq 0x11F4881D94EB39FC +dq 0xA9162248F2D0A23A diff --git a/unittests/InstructionCountCI/VEX_map1.json b/unittests/InstructionCountCI/VEX_map1.json index efd80f37b..81b4c3f1e 100644 --- a/unittests/InstructionCountCI/VEX_map1.json +++ b/unittests/InstructionCountCI/VEX_map1.json @@ -4799,16 +4799,37 @@ "urhadd v16.16b, v17.16b, v18.16b" ] }, + "vpavgb ymm0, ymm1, ymm0": { + "ExpectedInstructionCount": 1, + "Optimal": "Yes", + "Comment": [ + "Aliasing source and destination", + "Map 1 0b01 0xe0 256-bit" + ], + "ExpectedArm64ASM": [ + "urhadd z16.b, p7/m, z16.b, z17.b" + ] + }, + "vpavgb ymm0, ymm0, ymm2": { + "ExpectedInstructionCount": 1, + "Optimal": "Yes", + "Comment": [ + "Aliasing source and destination", + "Map 1 0b01 0xe0 256-bit" + ], + "ExpectedArm64ASM": [ + "urhadd z16.b, p7/m, z16.b, z18.b" + ] + }, "vpavgb ymm0, ymm1, ymm2": { - "ExpectedInstructionCount": 3, - "Optimal": "No", + "ExpectedInstructionCount": 2, + "Optimal": "Yes", "Comment": [ "Map 1 0b01 0xe0 256-bit" ], "ExpectedArm64ASM": [ - "movprfx z0, z17", - "urhadd z0.b, p7/m, z0.b, z18.b", - "mov z16.d, z0.d" + "movprfx z16, z17", + "urhadd z16.b, p7/m, z16.b, z18.b" ] }, "vpsraw xmm0, xmm1, xmm2": { @@ -4871,16 +4892,37 @@ "urhadd v16.8h, v17.8h, v18.8h" ] }, + "vpavgw ymm0, ymm1, ymm0": { + "ExpectedInstructionCount": 1, + "Optimal": "Yes", + "Comment": [ + "Aliasing source and destination", + "Map 1 0b01 0xe3 256-bit" + ], + "ExpectedArm64ASM": [ + "urhadd z16.h, p7/m, z16.h, z17.h" + ] + }, + "vpavgw ymm0, ymm0, ymm2": { + "ExpectedInstructionCount": 1, + "Optimal": "Yes", + "Comment": [ + "Aliasing source and destination", + "Map 1 0b01 0xe3 256-bit" + ], + "ExpectedArm64ASM": [ + "urhadd z16.h, p7/m, z16.h, z18.h" + ] + }, "vpavgw ymm0, ymm1, ymm2": { - "ExpectedInstructionCount": 3, - "Optimal": "No", + "ExpectedInstructionCount": 2, + "Optimal": "Yes", "Comment": [ "Map 1 0b01 0xe3 256-bit" ], "ExpectedArm64ASM": [ - "movprfx z0, z17", - "urhadd z0.h, p7/m, z0.h, z18.h", - "mov z16.d, z0.d" + "movprfx z16, z17", + "urhadd z16.h, p7/m, z16.h, z18.h" ] }, "vpmulhuw xmm0, xmm1, xmm2": {