diff --git a/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp b/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp index 685a23786..b509bd3fb 100644 --- a/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp @@ -1849,13 +1849,6 @@ DEF_OP(StoreMemTSO) { } DEF_OP(MemSet) { - // TODO: A future looking task would be to support this with ARM's MOPS instructions. - // The 8-bit non-atomic forward path directly matches ARM's SETP/SETM/SETE instruction, - // while the backward version needs some fixup to convert it to a forward direction. - // - // Assuming non-atomicity and non-faulting behaviour, this can accelerate this implementation. - // Additionally: This is commonly used as a memset to zero. If we know up-front with an inline constant - // that the value is zero, we can optimize any operation larger than 8-bit down to 8-bit to use the MOPS implementation. const auto Op = IROp->C(); const bool IsAtomic = CTX->IsMemcpyAtomicTSOEnabled(); @@ -1937,6 +1930,28 @@ DEF_OP(MemSet) { const int32_t SizeDirection = Size * Direction; const bool IsBackwards = Direction == -1; + // Sets the result to the final address written depending on + // whether or not the memset is forwards or backwards. + const auto MakeFinalAddress = [&] { + if (IsBackwards) { + switch (OpSize) { + case 1: sub(Dst.X(), MemReg.X(), Length.X()); break; + case 2: sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 1); break; + case 4: sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 2); break; + case 8: sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 3); break; + default: LOGMAN_MSG_A_FMT("Unhandled MemSet size: {}", OpSize); break; + } + } else { + switch (OpSize) { + case 1: add(Dst.X(), MemReg.X(), Length.X()); break; + case 2: add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 1); break; + case 4: add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 2); break; + case 8: add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 3); break; + default: LOGMAN_MSG_A_FMT("Unhandled MemSet size: {}", OpSize); break; + } + } + }; + ARMEmitter::BiDirectionalLabel AgainInternal {}; ARMEmitter::ForwardLabel DoneInternal {}; @@ -1945,24 +1960,44 @@ DEF_OP(MemSet) { if (!IsAtomic) { if (CTX->HostFeatures.SupportsMOPS) { - if (SubRegSize == ARMEmitter::SubRegSize::i8Bit) { + const bool Is8Bit = SubRegSize == ARMEmitter::SubRegSize::i8Bit; + + // We can handle 8-bit memsets and any other size that happens + // to be using an inlined zero value (resulting in the use of ZR). + // + // NOTE: + // Strictly speaking, this can also be trivially expanded to handle other sizes + // that happen to use any value that could fit inside a byte if the need + // arises. This does increase branching and code generation, however, since + // we'd still need to emit the fallback in the event a value for a larger size + // falls outside the range of a byte instead of only generating the MOPS code. + if (Is8Bit || Value == ARMEmitter::Reg::zr) { + // If we're performing a non-byte-sized zeroing operation then we need to + // scale the counter accordingly. (e.g. a 64-bit memset of size 2 needs to + // be turned into an 8-bit memset of size 16) + if (!Is8Bit) { + lsl(ARMEmitter::Size::i64Bit, TMP1, TMP1, FEXCore::ToUnderlying(SubRegSize)); + } + // If backwards, then we need to adjust the starting address because // set{p, m, e} memset forwards, so we need to slide this bad boy - // back like: address - (count + 1). + // back like: (address - count) + 1. + // + // This lets us offset the address such that we can treat a backwards + // memset as if it were a forwards one. if (IsBackwards) { sub(TMP2, TMP2, TMP1); add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 1); } + // Unfortunately set operations fiddle with NZCV, so we need to preserve it. + mrs(TMP3, ARMEmitter::SystemRegister::NZCV); setp(TMP2, TMP1, Value.X()); setm(TMP2, TMP1, Value.X()); sete(TMP2, TMP1, Value.X()); + msr(ARMEmitter::SystemRegister::NZCV, TMP3); - if (IsBackwards) { - sub(Dst.X(), MemReg.X(), Length.X()); - } else { - add(Dst.X(), MemReg.X(), Length.X()); - } + MakeFinalAddress(); (void)Bind(&DoneInternal); return; @@ -2028,23 +2063,7 @@ DEF_OP(MemSet) { (void)Bind(&DoneInternal); - if (SizeDirection >= 0) { - switch (OpSize) { - case 1: add(Dst.X(), MemReg.X(), Length.X()); break; - case 2: add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 1); break; - case 4: add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 2); break; - case 8: add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 3); break; - default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); break; - } - } else { - switch (OpSize) { - case 1: sub(Dst.X(), MemReg.X(), Length.X()); break; - case 2: sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 1); break; - case 4: sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 2); break; - case 8: sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 3); break; - default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); break; - } - } + MakeFinalAddress(); }; if (DirectionIsInline) { diff --git a/unittests/ASM/STOS.asm b/unittests/ASM/STOS.asm index 7410fe697..1c24ca6ab 100644 --- a/unittests/ASM/STOS.asm +++ b/unittests/ASM/STOS.asm @@ -1,6 +1,10 @@ %ifdef CONFIG { "Match": "All", + "RegData": { + "RDI": "0xE8000020", + "R11": "0xDAD10" + }, "MemoryRegions": { "0x100000000": "4096" } @@ -14,7 +18,7 @@ mov rax, 0xDEADBEEFBAD0DAD1 mov rdi, 0xe8000000 ; How many elements we want to store -mov rcx, 0x0 +mov rcx, 0x10 ; Direction to increment (Increment when cleared) cld @@ -27,35 +31,35 @@ mov r10, 0xe8000000 movzx r12, word [r10 + 0] add r11, r12 -movzx r12, word [r10 + 1] -add r11, r12 movzx r12, word [r10 + 2] add r11, r12 -movzx r12, word [r10 + 3] -add r11, r12 movzx r12, word [r10 + 4] add r11, r12 -movzx r12, word [r10 + 5] -add r11, r12 movzx r12, word [r10 + 6] add r11, r12 -movzx r12, word [r10 + 7] -add r11, r12 movzx r12, word [r10 + 8] add r11, r12 -movzx r12, word [r10 + 9] -add r11, r12 movzx r12, word [r10 + 10] add r11, r12 -movzx r12, word [r10 + 11] -add r11, r12 movzx r12, word [r10 + 12] add r11, r12 -movzx r12, word [r10 + 13] -add r11, r12 movzx r12, word [r10 + 14] add r11, r12 -movzx r12, word [r10 + 15] +movzx r12, word [r10 + 16] +add r11, r12 +movzx r12, word [r10 + 18] +add r11, r12 +movzx r12, word [r10 + 20] +add r11, r12 +movzx r12, word [r10 + 22] +add r11, r12 +movzx r12, word [r10 + 24] +add r11, r12 +movzx r12, word [r10 + 26] +add r11, r12 +movzx r12, word [r10 + 28] +add r11, r12 +movzx r12, word [r10 + 30] add r11, r12 hlt diff --git a/unittests/ASM/STOSQ.asm b/unittests/ASM/STOSQ.asm index b27e64f49..623cc5bdd 100644 --- a/unittests/ASM/STOSQ.asm +++ b/unittests/ASM/STOSQ.asm @@ -4,7 +4,8 @@ "RegData": { "RAX": "0", "RCX": "0", - "RDI": "0xE8000100" + "RDI": "0xE8000100", + "R11": "0" }, "MemoryRegions": { diff --git a/unittests/ASM/STOSQ_REPNE.asm b/unittests/ASM/STOSQ_REPNE.asm index 712a6c063..11f50b179 100644 --- a/unittests/ASM/STOSQ_REPNE.asm +++ b/unittests/ASM/STOSQ_REPNE.asm @@ -4,7 +4,8 @@ "RegData": { "RAX": "0", "RCX": "0", - "RDI": "0xE8000100" + "RDI": "0xE8000100", + "R11": "0" }, "MemoryRegions": { diff --git a/unittests/ASM/STOS_REPNE.asm b/unittests/ASM/STOS_REPNE.asm index 88d2ccec6..0fcdb8551 100644 --- a/unittests/ASM/STOS_REPNE.asm +++ b/unittests/ASM/STOS_REPNE.asm @@ -1,6 +1,10 @@ %ifdef CONFIG { "Match": "All", + "RegData": { + "RDI": "0xE8000020", + "R11": "0xDAD10" + }, "MemoryRegions": { "0x100000000": "4096" } @@ -14,7 +18,7 @@ mov rax, 0xDEADBEEFBAD0DAD1 mov rdi, 0xe8000000 ; How many elements we want to store -mov rcx, 0x0 +mov rcx, 0x10 ; Direction to increment (Increment when cleared) cld @@ -27,35 +31,35 @@ mov r10, 0xe8000000 movzx r12, word [r10 + 0] add r11, r12 -movzx r12, word [r10 + 1] -add r11, r12 movzx r12, word [r10 + 2] add r11, r12 -movzx r12, word [r10 + 3] -add r11, r12 movzx r12, word [r10 + 4] add r11, r12 -movzx r12, word [r10 + 5] -add r11, r12 movzx r12, word [r10 + 6] add r11, r12 -movzx r12, word [r10 + 7] -add r11, r12 movzx r12, word [r10 + 8] add r11, r12 -movzx r12, word [r10 + 9] -add r11, r12 movzx r12, word [r10 + 10] add r11, r12 -movzx r12, word [r10 + 11] -add r11, r12 movzx r12, word [r10 + 12] add r11, r12 -movzx r12, word [r10 + 13] -add r11, r12 movzx r12, word [r10 + 14] add r11, r12 -movzx r12, word [r10 + 15] +movzx r12, word [r10 + 16] +add r11, r12 +movzx r12, word [r10 + 18] +add r11, r12 +movzx r12, word [r10 + 20] +add r11, r12 +movzx r12, word [r10 + 22] +add r11, r12 +movzx r12, word [r10 + 24] +add r11, r12 +movzx r12, word [r10 + 26] +add r11, r12 +movzx r12, word [r10 + 28] +add r11, r12 +movzx r12, word [r10 + 30] add r11, r12 hlt