unittests/ASM: Adds REX encoding bug tests

To ensure we decode REX at the correct locations even with padding bytes
being encoded as REX.

unittests/ASM/FEX_bugs/REX/0F_38.asm

@@ -0,0 +1,35 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x424446484a4c4e50",
+    "RCX": "0x00000000d2af0486",
+    "R8": "0"
+  }
+}
+%endif
+
+mov rax, 0x4142434445464748
+mov rcx, 0x4142434445464748
+
+mov r8, 0
+
+lea rbx, [rel .data]
+jmp .test
+.test:
+
+; adcx rax, [rbx]
+; Real encoding: 0x66, 0x48, 0x0f, 0x38, 0xf6, 0x03
+; Add a dummy REX prefix that enables everything. Really mess up FEX's cumulative usage.
+db 0x4f, 0x66, 0x48, 0x0f, 0x38, 0xf6, 0x03
+
+; crc32 ecx, dword [rbx]
+; Real encoding: 0xf2, 0x0f, 0x38, 0xf1, 0x0b
+; Add a dummy rex encoding with the widening bit set.
+; If FEX parsed this incorrectly, then it converts the crc in to a 64-bit version.
+db 0x48, 0xf2, 0x0f, 0x38, 0xf1, 0x0b
+
+hlt
+
+align 16
+.data:
+dq 0x0102030405060708

unittests/ASM/FEX_bugs/REX/0F_3A.asm

@@ -0,0 +1,30 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x0000000055565758",
+    "RCX": "0x5152535455565758",
+    "R8": "0"
+  }
+}
+%endif
+
+mov rax, 0x4142434445464748
+mov rcx, 0x4142434445464748
+mov r8, 0
+movups xmm0, [rel .data]
+jmp .test
+.test:
+
+; pextrd eax, xmm0, 0
+; Real encoding: 0x66, 0x0f, 0x3a, 0x16, 0xc0, 0x00
+; Add a NOP REX encoding. Would convert `eax` to `rax` if decoded incorrectly.
+db 0x4f, 0x66, 0x0f, 0x3a, 0x16, 0xc0, 0x00
+; pextrq rcx, xmm0, 0
+; Real encoding: 0x66, 0x48, 0x0f, 0x3a, 0x16, 0xc1, 0x00
+; Add a NOP REX encoding, should do nothing. Might convert rcx to ecx if only first REX decoded.
+db 0x47, 0x66, 0x48, 0x0f, 0x3a, 0x16, 0xc1, 0x00
+hlt
+
+align 16
+.data:
+dq 0x5152535455565758, 0x6162636465666768

unittests/ASM/FEX_bugs/REX/DDDNow.asm

@@ -0,0 +1,31 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x0506070801020304",
+    "MM0": "0x0506070801020304"
+  },
+  "HostFeatures": ["3DNOW"]
+}
+%endif
+
+femms
+mov rax, 0x4142434445464748
+
+mov r8, 0
+
+lea rbx, [rel .data]
+jmp .test
+.test:
+
+; pswapd mm0, [rbx]
+; Real encoding: 0x0f, 0x0f, 0x03, 0xbb
+; Add a NOP REX encoding between a volatile REX and the 3DNow! instruction.
+; FEX accidentally being cumulative will cause rbx to convert to r8.
+db 0x41, 0x40, 0x0f, 0x0f, 0x03, 0xbb
+
+movd rax, mm0
+hlt
+
+align 16
+.data:
+dq 0x0102030405060708

unittests/ASM/FEX_bugs/REX/Primary.asm

@@ -0,0 +1,37 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x4142434445464e50",
+    "RCX": "0x000000004a4c4e50",
+    "R8": "0x4142434445464748",
+    "R9": "0x4142434445464748"
+  }
+}
+%endif
+
+; FEX-Emu had a bug where REX was not correctly ignored if it was placed at the wrong location.
+; "Wrong" means it wasn't encoded just before the opcode byte.
+; This can be done for multiple reasons, either padding or anti-emulation.
+mov rax, 0x4142434445464748
+mov rcx, 0x4142434445464748
+mov r8, 0x4142434445464748
+mov r9, 0x4142434445464748
+
+lea rbx, [rel .data]
+jmp .test
+.test:
+
+; add r8w, [rbx]
+; Real encoding: 0x66, 0x44, 0x03, 0x03
+; Swap operand-size override and REX. Converts r8 to rax, and stays a 16-bit operation.
+db 0x44, 0x66, 0x03, 0x03
+
+; add r9, [rbx]
+; Real encoding: 0x4c, 0x03, 0x0b
+; Add extraneous segment-overide between REX prefix and op, changes r9 to rcx, and 64-bit to 32-bit.
+db 0x4c, 0x2e, 0x03, 0x0b
+hlt
+
+align 16
+.data:
+dq 0x0102030405060708

unittests/ASM/FEX_bugs/REX/Primary_2.asm

@@ -0,0 +1,31 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x424446484a4c4e50",
+    "RCX": "0x4142434445464748",
+    "R8": "0x4142434445464748",
+    "R9": "0x4142434445464748"
+  }
+}
+%endif
+
+mov rax, 0x4142434445464748
+mov rcx, 0x4142434445464748
+mov r8, 0x4142434445464748
+mov r9, 0x4142434445464748
+
+lea rbx, [rel .data]
+jmp .test
+.test:
+
+; add rax, [rbx]
+; Real encoding: 0x44, 0x03, 0x03
+; Add additional false REX as padding that would convert `rax` to `r8`.
+; FEX treated REX prefixes as cumulative at one point.
+db 0x44, 0x48, 0x03, 0x03
+
+hlt
+
+align 16
+.data:
+dq 0x0102030405060708

unittests/ASM/FEX_bugs/REX/TwoByte.asm

@@ -0,0 +1,33 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x828486888a8c8e90",
+    "RBX": "0x000000008a8c8e90",
+    "RCX": "0x4142434445464748",
+    "RDX": "0x0000000045464748",
+    "R8": "1",
+    "R9": "1"
+  }
+}
+%endif
+
+mov rax, 0x4142434445464748
+mov rbx, 0x4142434445464748
+mov rcx, 0x4142434445464748
+mov rdx, 0x4142434445464748
+mov r8, 1
+mov r9, 1
+jmp .test
+.test:
+
+; xadd rax, rcx
+; Real encoding: 0x48, 0x0f, 0xc1, 0xc8
+; For cumulative decode errors, add a REX with all bits set. Will convert rax to r8, and rcx to r9.
+db 0x4f, 0x48, 0x0f, 0xc1, 0xc8
+
+; xadd ebx, edx
+; Real encoding: 0x0f, 0xc1, 0xd3
+; Add a nop-prefix pad between the opcode and full REX.
+db 0x4f, 0x2e, 0x0f, 0xc1, 0xd3
+
+hlt