diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp index 9869195b2..88be70900 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp @@ -1213,6 +1213,11 @@ DEF_OP(FindTrailingZeroes) { rbit(EmitSize, Dst, Src); if (OpSize == 2) { + // This orr does two things. First, if the (masked) source is zero, it + // reverses to zero in the top so it forces clz to return 16. Second, it + // ensures garbage in the upper bits of the source don't affect clz, because + // they'll rbit to garbage in the bottom below the 0x8000 and be ignored by + // the clz. So we handle Src upper garbage without explicitly masking. orr(EmitSize, Dst, Dst, 0x8000); } @@ -1230,6 +1235,8 @@ DEF_OP(CountLeadingZeroes) { const auto Src = GetReg(Op->Src.ID()); if (OpSize == 2) { + // Expressing as lsl+orr+clz clears away any garbage in the upper bits + // (alternatively could do uxth+clz+sub.. equal cost in total). lsl(EmitSize, Dst, Src, 16); orr(EmitSize, Dst, Dst, 0x8000); clz(EmitSize, Dst, Dst); diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index 9ecf41acd..0658150d0 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -2113,8 +2113,9 @@ void OpDispatchBuilder::BEXTRBMIOp(OpcodeArgs) { // Essentially (Src1 >> Start) & ((1 << Length) - 1) // along with some edge-case handling and flag setting. - auto* Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); - auto* Src2 = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags); + LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed"); + auto* Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true}); + auto* Src2 = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}); const auto Size = GetSrcSize(Op); const auto SrcSize = Size * 8; @@ -2152,10 +2153,12 @@ void OpDispatchBuilder::BEXTRBMIOp(OpcodeArgs) { void OpDispatchBuilder::BLSIBMIOp(OpcodeArgs) { // Equivalent to performing: SRC & -SRC + LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed"); + auto Size = OpSizeFromSrc(Op); - auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); - auto NegatedSrc = _Neg(OpSizeFromSrc(Op), Src); - auto Result = _And(OpSizeFromSrc(Op), Src, NegatedSrc); + auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true}); + auto NegatedSrc = _Neg(Size, Src); + auto Result = _And(Size, Src, NegatedSrc); // ...and we're done. Painless! StoreResult(GPRClass, Op, Result, -1); @@ -2165,10 +2168,11 @@ void OpDispatchBuilder::BLSIBMIOp(OpcodeArgs) { void OpDispatchBuilder::BLSMSKBMIOp(OpcodeArgs) { // Equivalent to: (Src - 1) ^ Src - auto One = _Constant(1); + LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed"); + auto Size = OpSizeFromSrc(Op); - auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); - auto Result = _Xor(OpSize::i64Bit, _Sub(OpSize::i64Bit, Src, One), Src); + auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true}); + auto Result = _Xor(Size, _Sub(Size, Src, _Constant(1)), Src); StoreResult(GPRClass, Op, Result, -1); GenerateFlags_BLSMSK(Op, Src); @@ -2176,11 +2180,11 @@ void OpDispatchBuilder::BLSMSKBMIOp(OpcodeArgs) { void OpDispatchBuilder::BLSRBMIOp(OpcodeArgs) { // Equivalent to: (Src - 1) & Src - auto One = _Constant(1); - - auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); - auto Result = _And(OpSize::i64Bit, _Sub(OpSize::i64Bit, Src, One), Src); + LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed"); + auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true}); + auto Size = OpSizeFromSrc(Op); + auto Result = _And(Size, _Sub(Size, Src, _Constant(1)), Src); StoreResult(GPRClass, Op, Result, -1); GenerateFlags_BLSR(Op, Result, Src); @@ -2316,8 +2320,9 @@ void OpDispatchBuilder::PDEP(OpcodeArgs) { } void OpDispatchBuilder::PEXT(OpcodeArgs) { - auto* Input = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); - auto* Mask = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags); + LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed"); + auto* Input = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true}); + auto* Mask = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}); auto Result = _PExt(OpSizeFromSrc(Op), Input, Mask); StoreResult(GPRClass, Op, Op->Dest, Result, -1); @@ -3180,7 +3185,7 @@ void OpDispatchBuilder::XADDOp(OpcodeArgs) { } void OpDispatchBuilder::PopcountOp(OpcodeArgs) { - OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); + OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = GetSrcSize(Op) >= 4}); Src = _Popcount(OpSizeFromSrc(Op), Src); StoreResult(GPRClass, Op, Src, -1); @@ -5367,7 +5372,8 @@ void OpDispatchBuilder::INTOp(OpcodeArgs) { } void OpDispatchBuilder::TZCNT(OpcodeArgs) { - OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); + // _FindTrailingZeroes ignores upper garbage so we don't need to mask + OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true}); Src = _FindTrailingZeroes(OpSizeFromSrc(Op), Src); StoreResult(GPRClass, Op, Src, -1); @@ -5376,7 +5382,8 @@ void OpDispatchBuilder::TZCNT(OpcodeArgs) { } void OpDispatchBuilder::LZCNT(OpcodeArgs) { - OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags); + // _CountLeadingZeroes clears upper garbage so we don't need to mask + OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true}); auto Res = _CountLeadingZeroes(OpSizeFromSrc(Op), Src); StoreResult(GPRClass, Op, Res, -1); diff --git a/unittests/InstructionCountCI/FlagM/Secondary_REP.json b/unittests/InstructionCountCI/FlagM/Secondary_REP.json index 5e8037620..12a75b610 100644 --- a/unittests/InstructionCountCI/FlagM/Secondary_REP.json +++ b/unittests/InstructionCountCI/FlagM/Secondary_REP.json @@ -32,11 +32,10 @@ ] }, "popcnt eax, ebx": { - "ExpectedInstructionCount": 11, + "ExpectedInstructionCount": 10, "Comment": "0xf3 0x0f 0xb8", "ExpectedArm64ASM": [ - "mov w20, w7", - "fmov s0, w20", + "fmov s0, w7", "cnt v0.8b, v0.8b", "addv b0, v0.8b", "umov w4, v0.b[0]", @@ -65,11 +64,10 @@ ] }, "tzcnt ax, bx": { - "ExpectedInstructionCount": 10, + "ExpectedInstructionCount": 9, "Comment": "0xf3 0x0f 0xbc", "ExpectedArm64ASM": [ - "uxth w20, w7", - "rbit w20, w20", + "rbit w20, w7", "orr w20, w20, #0x8000", "clz w20, w20", "bfxil x4, x20, #0, #16", @@ -81,11 +79,10 @@ ] }, "tzcnt eax, ebx": { - "ExpectedInstructionCount": 8, + "ExpectedInstructionCount": 7, "Comment": "0xf3 0x0f 0xbc", "ExpectedArm64ASM": [ - "mov w20, w7", - "rbit w4, w20", + "rbit w4, w7", "clz w4, w4", "cmp x4, #0x0 (0)", "cset x20, eq", @@ -108,32 +105,30 @@ ] }, "lzcnt ax, bx": { - "ExpectedInstructionCount": 10, + "ExpectedInstructionCount": 9, "Comment": "0xf3 0x0f 0xbd", "ExpectedArm64ASM": [ - "uxth w20, w7", - "lsl w21, w20, #16", - "orr w21, w21, #0x8000", - "clz w21, w21", - "bfxil x4, x21, #0, #16", - "cmp x20, #0x0 (0)", - "cset x21, eq", - "lsl x21, x21, #29", - "msr nzcv, x21", - "rmif x20, #13, #nZcv" + "lsl w20, w7, #16", + "orr w20, w20, #0x8000", + "clz w20, w20", + "bfxil x4, x20, #0, #16", + "cmp x7, #0x0 (0)", + "cset x20, eq", + "lsl x20, x20, #29", + "msr nzcv, x20", + "rmif x7, #13, #nZcv" ] }, "lzcnt eax, ebx": { - "ExpectedInstructionCount": 7, + "ExpectedInstructionCount": 6, "Comment": "0xf3 0x0f 0xbd", "ExpectedArm64ASM": [ - "mov w20, w7", - "clz w4, w20", - "cmp x20, #0x0 (0)", - "cset x21, eq", - "lsl x21, x21, #29", - "msr nzcv, x21", - "rmif x20, #29, #nZcv" + "clz w4, w7", + "cmp x7, #0x0 (0)", + "cset x20, eq", + "lsl x20, x20, #29", + "msr nzcv, x20", + "rmif x7, #29, #nZcv" ] }, "lzcnt rax, rbx": { diff --git a/unittests/InstructionCountCI/FlagM/VEX_map2.json b/unittests/InstructionCountCI/FlagM/VEX_map2.json index a32efac3d..28b791206 100644 --- a/unittests/InstructionCountCI/FlagM/VEX_map2.json +++ b/unittests/InstructionCountCI/FlagM/VEX_map2.json @@ -442,26 +442,24 @@ ] }, "bextr eax, ebx, ecx": { - "ExpectedInstructionCount": 19, + "ExpectedInstructionCount": 17, "Comment": [ "Map 2 0b00 0xf7 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "mov w21, w5", - "mov w22, #0x1f", - "uxtb w23, w21", - "lsr w20, w20, w23", - "mov w24, #0x0", - "cmp w23, #0x1f (31)", - "csel w20, w20, w24, ls", - "ubfx w21, w21, #8, #8", + "mov w20, #0x1f", + "uxtb w21, w5", + "lsr w22, w7, w21", + "mov w23, #0x0", "cmp w21, #0x1f (31)", - "csel w21, w21, w22, ls", + "csel w21, w22, w23, ls", + "ubfx w22, w5, #8, #8", + "cmp w22, #0x1f (31)", + "csel w20, w22, w20, ls", "mov w22, #0x1", - "lsl w21, w22, w21", - "sub w21, w21, #0x1 (1)", - "and w4, w20, w21", + "lsl w20, w22, w20", + "sub w20, w20, #0x1 (1)", + "and w4, w21, w20", "cmp x4, #0x0 (0)", "cset x20, eq", "lsl x20, x20, #30", diff --git a/unittests/InstructionCountCI/FlagM/VEX_map_group.json b/unittests/InstructionCountCI/FlagM/VEX_map_group.json index 004f18b06..cb482da17 100644 --- a/unittests/InstructionCountCI/FlagM/VEX_map_group.json +++ b/unittests/InstructionCountCI/FlagM/VEX_map_group.json @@ -10,20 +10,18 @@ }, "Instructions": { "blsr eax, ebx": { - "ExpectedInstructionCount": 10, + "ExpectedInstructionCount": 8, "Comment": [ "Map group 17 0b001 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "sub x21, x20, #0x1 (1)", - "and x21, x21, x20", - "mov w4, w21", - "tst w21, w21", - "mrs x21, nzcv", - "cmp x20, #0x0 (0)", - "cset x20, ne", - "orr w20, w21, w20, lsl #29", + "sub w20, w7, #0x1 (1)", + "and w4, w20, w7", + "tst w4, w4", + "mrs x20, nzcv", + "cmp x7, #0x0 (0)", + "cset x21, ne", + "orr w20, w20, w21, lsl #29", "msr nzcv, x20" ] }, @@ -44,22 +42,20 @@ ] }, "blsmsk eax, ebx": { - "ExpectedInstructionCount": 11, + "ExpectedInstructionCount": 9, "Comment": [ "Map group 17 0b010 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "sub x21, x20, #0x1 (1)", - "eor x21, x21, x20", - "mov w4, w21", - "mov w21, #0x50000000", - "mrs x22, nzcv", - "bic x21, x22, x21", - "cmp x20, #0x0 (0)", - "cset x20, ne", - "msr nzcv, x21", - "rmif x20, #63, #nzCv" + "sub w20, w7, #0x1 (1)", + "eor w4, w20, w7", + "mov w20, #0x50000000", + "mrs x21, nzcv", + "bic x20, x21, x20", + "cmp x7, #0x0 (0)", + "cset x21, ne", + "msr nzcv, x20", + "rmif x21, #63, #nzCv" ] }, "blsmsk rax, rbx": { @@ -80,14 +76,13 @@ ] }, "blsi eax, ebx": { - "ExpectedInstructionCount": 9, + "ExpectedInstructionCount": 8, "Comment": [ "Map group 17 0b011 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "neg w21, w20", - "and w4, w20, w21", + "neg w20, w7", + "and w4, w7, w20", "tst w4, w4", "mrs x20, nzcv", "cmp x4, #0x0 (0)", diff --git a/unittests/InstructionCountCI/Secondary_REP.json b/unittests/InstructionCountCI/Secondary_REP.json index 71004c1fb..4dacfc523 100644 --- a/unittests/InstructionCountCI/Secondary_REP.json +++ b/unittests/InstructionCountCI/Secondary_REP.json @@ -417,11 +417,10 @@ ] }, "popcnt eax, ebx": { - "ExpectedInstructionCount": 11, + "ExpectedInstructionCount": 10, "Comment": "0xf3 0x0f 0xb8", "ExpectedArm64ASM": [ - "mov w20, w7", - "fmov s0, w20", + "fmov s0, w7", "cnt v0.8b, v0.8b", "addv b0, v0.8b", "umov w4, v0.b[0]", @@ -450,11 +449,10 @@ ] }, "tzcnt ax, bx": { - "ExpectedInstructionCount": 11, + "ExpectedInstructionCount": 10, "Comment": "0xf3 0x0f 0xbc", "ExpectedArm64ASM": [ - "uxth w20, w7", - "rbit w20, w20", + "rbit w20, w7", "orr w20, w20, #0x8000", "clz w20, w20", "bfxil x4, x20, #0, #16", @@ -467,11 +465,10 @@ ] }, "tzcnt eax, ebx": { - "ExpectedInstructionCount": 9, + "ExpectedInstructionCount": 8, "Comment": "0xf3 0x0f 0xbc", "ExpectedArm64ASM": [ - "mov w20, w7", - "rbit w4, w20", + "rbit w4, w7", "clz w4, w4", "cmp x4, #0x0 (0)", "cset x20, eq", @@ -496,33 +493,31 @@ ] }, "lzcnt ax, bx": { - "ExpectedInstructionCount": 11, + "ExpectedInstructionCount": 10, "Comment": "0xf3 0x0f 0xbd", "ExpectedArm64ASM": [ - "uxth w20, w7", - "lsl w21, w20, #16", - "orr w21, w21, #0x8000", - "clz w21, w21", - "bfxil x4, x21, #0, #16", - "cmp x20, #0x0 (0)", - "cset x21, eq", - "lsl x21, x21, #29", - "ubfx x20, x20, #15, #1", - "orr w20, w21, w20, lsl #30", + "lsl w20, w7, #16", + "orr w20, w20, #0x8000", + "clz w20, w20", + "bfxil x4, x20, #0, #16", + "cmp x7, #0x0 (0)", + "cset x20, eq", + "lsl x20, x20, #29", + "ubfx x21, x7, #15, #1", + "orr w20, w20, w21, lsl #30", "msr nzcv, x20" ] }, "lzcnt eax, ebx": { - "ExpectedInstructionCount": 8, + "ExpectedInstructionCount": 7, "Comment": "0xf3 0x0f 0xbd", "ExpectedArm64ASM": [ - "mov w20, w7", - "clz w4, w20", - "cmp x20, #0x0 (0)", - "cset x21, eq", - "lsl x21, x21, #29", - "ubfx x20, x20, #31, #1", - "orr w20, w21, w20, lsl #30", + "clz w4, w7", + "cmp x7, #0x0 (0)", + "cset x20, eq", + "lsl x20, x20, #29", + "ubfx x21, x7, #31, #1", + "orr w20, w20, w21, lsl #30", "msr nzcv, x20" ] }, diff --git a/unittests/InstructionCountCI/VEX_map2.json b/unittests/InstructionCountCI/VEX_map2.json index 9735787a4..bdde71dfd 100644 --- a/unittests/InstructionCountCI/VEX_map2.json +++ b/unittests/InstructionCountCI/VEX_map2.json @@ -3225,16 +3225,14 @@ ] }, "pext eax, ebx, ecx": { - "ExpectedInstructionCount": 14, + "ExpectedInstructionCount": 12, "Comment": [ "Map 2 0b10 0xf5 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "mov w21, w5", - "cbz w21, #+0x2c", - "mov w0, w21", - "mov w2, w20", + "cbz w5, #+0x2c", + "mov w0, w5", + "mov w2, w7", "mov w4, wzr", "cbz w0, #+0x20", "clz w1, w0", @@ -3383,26 +3381,24 @@ ] }, "bextr eax, ebx, ecx": { - "ExpectedInstructionCount": 19, + "ExpectedInstructionCount": 17, "Comment": [ "Map 2 0b00 0xf7 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "mov w21, w5", - "mov w22, #0x1f", - "uxtb w23, w21", - "lsr w20, w20, w23", - "mov w24, #0x0", - "cmp w23, #0x1f (31)", - "csel w20, w20, w24, ls", - "ubfx w21, w21, #8, #8", + "mov w20, #0x1f", + "uxtb w21, w5", + "lsr w22, w7, w21", + "mov w23, #0x0", "cmp w21, #0x1f (31)", - "csel w21, w21, w22, ls", + "csel w21, w22, w23, ls", + "ubfx w22, w5, #8, #8", + "cmp w22, #0x1f (31)", + "csel w20, w22, w20, ls", "mov w22, #0x1", - "lsl w21, w22, w21", - "sub w21, w21, #0x1 (1)", - "and w4, w20, w21", + "lsl w20, w22, w20", + "sub w20, w20, #0x1 (1)", + "and w4, w21, w20", "cmp x4, #0x0 (0)", "cset x20, eq", "lsl x20, x20, #30", diff --git a/unittests/InstructionCountCI/VEX_map_group.json b/unittests/InstructionCountCI/VEX_map_group.json index 2f4f8fa8f..19187a9fe 100644 --- a/unittests/InstructionCountCI/VEX_map_group.json +++ b/unittests/InstructionCountCI/VEX_map_group.json @@ -609,20 +609,18 @@ ] }, "blsr eax, ebx": { - "ExpectedInstructionCount": 10, + "ExpectedInstructionCount": 8, "Comment": [ "Map group 17 0b001 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "sub x21, x20, #0x1 (1)", - "and x21, x21, x20", - "mov w4, w21", - "tst w21, w21", - "mrs x21, nzcv", - "cmp x20, #0x0 (0)", - "cset x20, ne", - "orr w20, w21, w20, lsl #29", + "sub w20, w7, #0x1 (1)", + "and w4, w20, w7", + "tst w4, w4", + "mrs x20, nzcv", + "cmp x7, #0x0 (0)", + "cset x21, ne", + "orr w20, w20, w21, lsl #29", "msr nzcv, x20" ] }, @@ -643,23 +641,19 @@ ] }, "blsmsk eax, ebx": { - "ExpectedInstructionCount": 13, + "ExpectedInstructionCount": 9, "Comment": [ "Map group 17 0b010 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "sub x21, x20, #0x1 (1)", - "eor x21, x21, x20", - "mov w4, w21", - "mov w21, #0x50000000", - "mrs x22, nzcv", - "bic x21, x22, x21", - "cmp x20, #0x0 (0)", - "cset x20, ne", - "mov w0, w21", - "bfi w0, w20, #29, #1", - "mov w20, w0", + "sub w20, w7, #0x1 (1)", + "eor w4, w20, w7", + "mov w20, #0x50000000", + "mrs x21, nzcv", + "bic x20, x21, x20", + "cmp x7, #0x0 (0)", + "cset x21, ne", + "bfi w20, w21, #29, #1", "msr nzcv, x20" ] }, @@ -681,14 +675,13 @@ ] }, "blsi eax, ebx": { - "ExpectedInstructionCount": 9, + "ExpectedInstructionCount": 8, "Comment": [ "Map group 17 0b011 32-bit" ], "ExpectedArm64ASM": [ - "mov w20, w7", - "neg w21, w20", - "and w4, w20, w21", + "neg w20, w7", + "and w4, w7, w20", "tst w4, w4", "mrs x20, nzcv", "cmp x4, #0x0 (0)",