Arm64: Fixes SVE V{S,U}MulH

When the destination overlaps one of the sources we must be careful to
follow a movprfx rule.
```
The destination register must not refer to architectural register state
referenced by any other source operand register of this instruction.
```

We ended up in a situation in the vpmulh{u,}w AVX tests where zm was
overlapping the destination which violated that rule. This also
generated invalid code for this instruction.
```
[INFO] movprfx z6, z4
[INFO] umulh z6.h, p6/m, z6.h, z6.h
```

As seen, we were overwriting one of the sources because the destination
overlapped it. Now instead check if each individual overlap so invalid
code isn't generated.

InstCountCI results aren't affected since this only happens in
situations with multiple instructions.
This commit is contained in:
Ryan Houdek committed 2023-09-07 16:47:08 -07:00
1 parent 7b80427de0
commit 22ca46a227
1 file changed
+19 -5
@@ -3664,11 +3664,18 @@ DEF_OP(VUMulH) {
// Do predicated to ensure upper-bits get zero as expected
const auto Mask = PRED_TMP_16B.Merging();
if (Dst != Vector1) {
if (Dst == Vector1) {
umulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
}
else if (Dst == Vector2) {
umulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector1.Z());
}
else {
// Destination register doesn't overlap either source.
// NOTE: SVE umulh (predicated) is a destructive operation.
movprfx(Dst.Z(), Vector1.Z());
umulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
}
umulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
}
else {
umulh(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
@@ -3717,11 +3724,18 @@ DEF_OP(VSMulH) {
// Do predicated to ensure upper-bits get zero as expected
const auto Mask = PRED_TMP_16B.Merging();
if (Dst != Vector1) {
// NOTE: SVE smulh (predicated) is a destructive operation.
if (Dst == Vector1) {
smulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
}
else if (Dst == Vector2) {
smulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector1.Z());
}
else {
// Destination register doesn't overlap either source.
// NOTE: SVE umulh (predicated) is a destructive operation.
movprfx(Dst.Z(), Vector1.Z());
smulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
}
smulh(SubRegSize, Dst.Z(), Mask, Dst.Z(), Vector2.Z());
}
else {
smulh(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());