Merge pull request #3208 from lioncash/avg

VectorOps: Handle SVE VURAvg a little better

FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp

@@ -1357,16 +1357,18 @@ DEF_OP(VURAvg) {
   if (HostSupportsSVE256 && Is256Bit) {
     const auto Mask = PRED_TMP_32B.Merging();
 
+    // Trivial cases where we already have source data to be averaged in
+    // the destination register. We can just do the operation in place.
     if (Dst == Vector1) {
-      // Trivial case where we already have source data to be averaged in
-      // the destination register. We can just do the operation in place.
       urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
+    } else if (Dst == Vector2) {
+      urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector1.Z());
     } else {
-      // SVE URHADD is a destructive operation, so we need
-      // a temporary for performing operations.
-      movprfx(VTMP1.Z(), Vector1.Z());
-      urhadd(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), Vector2.Z());
-      mov(Dst.Z(), VTMP1.Z());
+      // SVE URHADD is a destructive operation, but we know that
+      // we don't have any source/destination aliasing happening here
+      // so we can safely move one of the source operands into the destination.
+      movprfx(Dst.Z(), Vector1.Z());
+      urhadd(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
     }
   } else {
     urhadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());

unittests/ASM/VEX/vpavgb_aliasing.asm

@@ -0,0 +1,42 @@
+%ifdef CONFIG
+{
+  "HostFeatures": ["AVX"],
+  "RegData": {
+    "XMM10": ["0x2179B0697D5378C4", "0x3B8E6EAE8C165248", "0x2179B0697D5378C4", "0x3B8E6EAE8C165248"],
+    "XMM11": ["0x1ED68638699D35CA", "0x5E2E7560AB7B5262", "0x1ED68638699D35CA", "0x5E2E7560AB7B5262"]
+  }
+}
+%endif
+
+; Small test that ensures aliasing source/dest is handled properly.
+
+lea rdx, [rel .data]
+
+vmovapd ymm6, [rdx + 32]
+vmovapd ymm7, [rdx]
+
+; 256-bit register only
+vmovapd ymm10, ymm7
+vpavgb ymm10, ymm10, ymm6
+
+vmovapd ymm11, [rdx + 64]
+vpavgb ymm11, ymm7, ymm11
+
+hlt
+
+align 32
+.data:
+dq 0x2BB883523D4F3197
+dq 0x1246C77764260189
+dq 0x2BB883523D4F3197
+dq 0x1246C77764260189
+
+dq 0x163ADD80BC57BEF1
+dq 0x64D615E5B405A306
+dq 0x163ADD80BC57BEF1
+dq 0x64D615E5B405A306
+
+dq 0x11F4881D94EB39FC
+dq 0xA9162248F2D0A23A
+dq 0x11F4881D94EB39FC
+dq 0xA9162248F2D0A23A

unittests/ASM/VEX/vpavgw_aliasing.asm

@@ -0,0 +1,42 @@
+%ifdef CONFIG
+{
+  "HostFeatures": ["AVX"],
+  "RegData": {
+    "XMM10": ["0x20F9B0697CD37844", "0x3B8E6EAE8C165248", "0x20F9B0697CD37844", "0x3B8E6EAE8C165248"],
+    "XMM11": ["0x1ED685B8691D35CA", "0x5DAE74E0AB7B51E2", "0x1ED685B8691D35CA", "0x5DAE74E0AB7B51E2"]
+  }
+}
+%endif
+
+; Small test that ensures aliasing source/dest is handled properly.
+
+lea rdx, [rel .data]
+
+vmovapd ymm6, [rdx + 32]
+vmovapd ymm7, [rdx]
+
+; 256-bit register only
+vmovapd ymm10, ymm7
+vpavgw ymm10, ymm10, ymm6
+
+vmovapd ymm11, [rdx + 64]
+vpavgw ymm11, ymm7, ymm11
+
+hlt
+
+align 32
+.data:
+dq 0x2BB883523D4F3197
+dq 0x1246C77764260189
+dq 0x2BB883523D4F3197
+dq 0x1246C77764260189
+
+dq 0x163ADD80BC57BEF1
+dq 0x64D615E5B405A306
+dq 0x163ADD80BC57BEF1
+dq 0x64D615E5B405A306
+
+dq 0x11F4881D94EB39FC
+dq 0xA9162248F2D0A23A
+dq 0x11F4881D94EB39FC
+dq 0xA9162248F2D0A23A

unittests/InstructionCountCI/VEX_map1.json

@@ -4799,16 +4799,37 @@
         "urhadd v16.16b, v17.16b, v18.16b"
       ]
     },
+    "vpavgb ymm0, ymm1, ymm0": {
+      "ExpectedInstructionCount": 1,
+      "Optimal": "Yes",
+      "Comment": [
+        "Aliasing source and destination",
+        "Map 1 0b01 0xe0 256-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "urhadd z16.b, p7/m, z16.b, z17.b"
+      ]
+    },
+    "vpavgb ymm0, ymm0, ymm2": {
+      "ExpectedInstructionCount": 1,
+      "Optimal": "Yes",
+      "Comment": [
+        "Aliasing source and destination",
+        "Map 1 0b01 0xe0 256-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "urhadd z16.b, p7/m, z16.b, z18.b"
+      ]
+    },
     "vpavgb ymm0, ymm1, ymm2": {
-      "ExpectedInstructionCount": 3,
-      "Optimal": "No",
+      "ExpectedInstructionCount": 2,
+      "Optimal": "Yes",
       "Comment": [
         "Map 1 0b01 0xe0 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "movprfx z0, z17",
-        "urhadd z0.b, p7/m, z0.b, z18.b",
-        "mov z16.d, z0.d"
+        "movprfx z16, z17",
+        "urhadd z16.b, p7/m, z16.b, z18.b"
       ]
     },
     "vpsraw xmm0, xmm1, xmm2": {
@@ -4871,16 +4892,37 @@
         "urhadd v16.8h, v17.8h, v18.8h"
       ]
     },
+    "vpavgw ymm0, ymm1, ymm0": {
+      "ExpectedInstructionCount": 1,
+      "Optimal": "Yes",
+      "Comment": [
+        "Aliasing source and destination",
+        "Map 1 0b01 0xe3 256-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "urhadd z16.h, p7/m, z16.h, z17.h"
+      ]
+    },
+    "vpavgw ymm0, ymm0, ymm2": {
+      "ExpectedInstructionCount": 1,
+      "Optimal": "Yes",
+      "Comment": [
+        "Aliasing source and destination",
+        "Map 1 0b01 0xe3 256-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "urhadd z16.h, p7/m, z16.h, z18.h"
+      ]
+    },
     "vpavgw ymm0, ymm1, ymm2": {
-      "ExpectedInstructionCount": 3,
-      "Optimal": "No",
+      "ExpectedInstructionCount": 2,
+      "Optimal": "Yes",
       "Comment": [
         "Map 1 0b01 0xe3 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "movprfx z0, z17",
-        "urhadd z0.h, p7/m, z0.h, z18.h",
-        "mov z16.d, z0.d"
+        "movprfx z16, z17",
+        "urhadd z16.h, p7/m, z16.h, z18.h"
       ]
     },
     "vpmulhuw xmm0, xmm1, xmm2": {