From 2bcf435e0a72eb8e7ec411e5e2b6f0813b9da196 Mon Sep 17 00:00:00 2001 From: Lioncache Date: Thu, 19 Mar 2026 12:15:45 -0400 Subject: [PATCH] MemoryOps: Handle overlapping memcpy --- .../Source/Interface/Core/JIT/MemoryOps.cpp | 29 +- .../InstructionCountCI/FEXOpt/MultiInst.json | 342 +++++++++++++++-- .../InstructionCountCI/MOPS/Primary.json | 350 ++++++++++++++++-- 3 files changed, 661 insertions(+), 60 deletions(-) diff --git a/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp b/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp index 513f690b4..2643bf78a 100644 --- a/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp @@ -2258,6 +2258,20 @@ DEF_OP(MemCpy) { if (!IsAtomic) { if (CTX->HostFeatures.SupportsMOPS) { + // In the event we have an overlap (gross), we need to fall back + // to the non-mops copy handler. Since the overlap check needs to + // make use of NZCV, we need to save it. This can be avoided with + // ARMv9.6+'s FEAT_CMPBR, but alas, we don't have access to that right now. + // + // NOTE: That we need to temporarily trash TMP1 and restore it after the + // comparison. + ARMEmitter::ForwardLabel OverlapCase; + mrs(TMP4, ARMEmitter::SystemRegister::NZCV); + sub(ARMEmitter::Size::i64Bit, TMP1, TMP2, TMP3); + cmp(ARMEmitter::Size::i64Bit, TMP1, Length.X()); + mov(TMP1, Length.X()); + (void)bc(ARMEmitter::Condition::CC_LT, &OverlapCase); + // If doing something larger than a byte copy, then we need to scale // the counter value accordingly to convert it to bytes. if (Size > 1) { @@ -2273,21 +2287,16 @@ DEF_OP(MemCpy) { } // Unfortunately copy operations fiddle with NZCV, so we need to preserve it. - mrs(TMP4, ARMEmitter::SystemRegister::NZCV); cpyfp(TMP2, TMP3, TMP1); cpyfm(TMP2, TMP3, TMP1); cpyfe(TMP2, TMP3, TMP1); msr(ARMEmitter::SystemRegister::NZCV, TMP4); - // Needs to use temporaries just in case of overwrite - mov(TMP1, MemRegDest.X()); - mov(TMP2, MemRegSrc.X()); - mov(TMP3, Length.X()); + (void)b(&DoneInternal); - FinalizeAddresses(); - - (void)Bind(&DoneInternal); - return; + // Turns out we overlap and need to fall back. Make sure to restore NZCV. + (void)Bind(&OverlapCase); + msr(ARMEmitter::SystemRegister::NZCV, TMP4); } ARMEmitter::ForwardLabel AbsPos {}; @@ -2367,7 +2376,7 @@ DEF_OP(MemCpy) { LOGMAN_THROW_A_FMT(DirectionConstant == 1 || DirectionConstant == -1, "unexpected direction"); EmitMemcpy(DirectionConstant); } else { - // Emit forward direction memset then backward direction memset. + // Emit forward direction memcpy then backward direction memcpy. for (int32_t Direction : {1, -1}) { EmitMemcpy(Direction); if (Direction == 1) { diff --git a/unittests/InstructionCountCI/FEXOpt/MultiInst.json b/unittests/InstructionCountCI/FEXOpt/MultiInst.json index 542978fac..8c6ee3c1e 100644 --- a/unittests/InstructionCountCI/FEXOpt/MultiInst.json +++ b/unittests/InstructionCountCI/FEXOpt/MultiInst.json @@ -363,7 +363,7 @@ }, "positive rep movsb": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 19, + "ExpectedInstructionCount": 54, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -377,12 +377,47 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x2c", + "cbz x0, #+0xa4", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x18", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x40 (64)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x40 (64)", + "cbz x0, #+0x34", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x14", + "ldrb w3, [x2], #1", + "strb w3, [x1], #1", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -396,7 +431,7 @@ }, "positive rep movsw": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 20, + "ExpectedInstructionCount": 55, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -410,13 +445,48 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x30", - "lsl x0, x0, #1", + "cbz x0, #+0xa8", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x1c", + "lsl x0, x0, #1", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x34", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x14", + "ldrh w3, [x2], #2", + "strh w3, [x1], #2", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -430,7 +500,7 @@ }, "positive rep movsd": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 20, + "ExpectedInstructionCount": 55, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -444,13 +514,48 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x30", - "lsl x0, x0, #2", + "cbz x0, #+0xa8", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x1c", + "lsl x0, x0, #2", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x34", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x14", + "ldr w3, [x2], #4", + "str w3, [x1], #4", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -464,7 +569,7 @@ }, "positive rep movsq": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 20, + "ExpectedInstructionCount": 55, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -478,13 +583,48 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x30", - "lsl x0, x0, #3", + "cbz x0, #+0xa8", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x1c", + "lsl x0, x0, #3", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x34", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x4 (4)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x4 (4)", + "cbz x0, #+0x14", + "ldr x3, [x2], #8", + "str x3, [x1], #8", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -498,7 +638,7 @@ }, "negative rep movsb": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 23, + "ExpectedInstructionCount": 62, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -512,16 +652,55 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x3c", + "cbz x0, #+0xc4", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x28", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x1 (1)", "add x2, x2, #0x1 (1)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x1f (31)", + "sub x2, x2, #0x1f (31)", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x40 (64)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x40 (64)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x1c", + "add x1, x1, #0x1f (31)", + "add x2, x2, #0x1f (31)", + "ldrb w3, [x2], #-1", + "strb w3, [x1], #-1", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -535,7 +714,7 @@ }, "negative rep movsw": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 24, + "ExpectedInstructionCount": 63, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -549,17 +728,56 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x40", + "cbz x0, #+0xc8", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x2c", "lsl x0, x0, #1", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x2 (2)", "add x2, x2, #0x2 (2)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x1e (30)", + "sub x2, x2, #0x1e (30)", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x1c", + "add x1, x1, #0x1e (30)", + "add x2, x2, #0x1e (30)", + "ldrh w3, [x2], #-2", + "strh w3, [x1], #-2", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -573,7 +791,7 @@ }, "negative rep movsd": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 24, + "ExpectedInstructionCount": 63, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -587,17 +805,56 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x40", + "cbz x0, #+0xc8", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x2c", "lsl x0, x0, #2", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x4 (4)", "add x2, x2, #0x4 (4)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x1c (28)", + "sub x2, x2, #0x1c (28)", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x1c", + "add x1, x1, #0x1c (28)", + "add x2, x2, #0x1c (28)", + "ldr w3, [x2], #-4", + "str w3, [x1], #-4", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -611,7 +868,7 @@ }, "negative rep movsq": { "x86InstructionCount": 2, - "ExpectedInstructionCount": 24, + "ExpectedInstructionCount": 63, "Comment": [ "When direction flag is a compile time constant we can optimize", "loads and stores can turn in to post-increment when known" @@ -625,17 +882,56 @@ "mov x0, x7", "mov x1, x11", "mov x2, x10", - "cbz x0, #+0x40", + "cbz x0, #+0xc8", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x2c", "lsl x0, x0, #3", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x8 (8)", "add x2, x2, #0x8 (8)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x18 (24)", + "sub x2, x2, #0x18 (24)", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x4 (4)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x4 (4)", + "cbz x0, #+0x1c", + "add x1, x1, #0x18 (24)", + "add x2, x2, #0x18 (24)", + "ldr x3, [x2], #-8", + "str x3, [x1], #-8", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", diff --git a/unittests/InstructionCountCI/MOPS/Primary.json b/unittests/InstructionCountCI/MOPS/Primary.json index bd1d79dd4..a89a622a0 100644 --- a/unittests/InstructionCountCI/MOPS/Primary.json +++ b/unittests/InstructionCountCI/MOPS/Primary.json @@ -14,36 +14,110 @@ }, "Instructions": { "rep movsb": { - "ExpectedInstructionCount": 35, + "ExpectedInstructionCount": 109, "Comment": "0xa4", "ExpectedArm64ASM": [ "ldrsb x22, [x28, #1018]", "mov x0, x7", "mov x1, x11", "mov x2, x10", - "tbnz w22, #1, #+0x34", - "cbz x0, #+0x2c", + "tbnz w22, #1, #+0xc0", + "cbz x0, #+0xa4", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x18", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x40 (64)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x40 (64)", + "cbz x0, #+0x34", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x14", + "ldrb w3, [x2], #1", + "strb w3, [x1], #1", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", "add x21, x0, x2", "add x20, x1, x2", - "b #+0x40", - "cbz x0, #+0x3c", + "b #+0xdc", + "cbz x0, #+0xc4", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x28", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x1 (1)", "add x2, x2, #0x1 (1)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x1f (31)", + "sub x2, x2, #0x1f (31)", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x40 (64)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x40 (64)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x20 (32)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x1c", + "add x1, x1, #0x1f (31)", + "add x2, x2, #0x1f (31)", + "ldrb w3, [x2], #-1", + "strb w3, [x1], #-1", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -55,38 +129,112 @@ ] }, "rep movsw": { - "ExpectedInstructionCount": 37, + "ExpectedInstructionCount": 111, "Comment": "0xa5", "ExpectedArm64ASM": [ "ldrsb x22, [x28, #1018]", "mov x0, x7", "mov x1, x11", "mov x2, x10", - "tbnz w22, #1, #+0x38", - "cbz x0, #+0x30", - "lsl x0, x0, #1", + "tbnz w22, #1, #+0xc4", + "cbz x0, #+0xa8", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x1c", + "lsl x0, x0, #1", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x34", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x14", + "ldrh w3, [x2], #2", + "strh w3, [x1], #2", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", "add x21, x0, x2, lsl #1", "add x20, x1, x2, lsl #1", - "b #+0x44", - "cbz x0, #+0x40", + "b #+0xe0", + "cbz x0, #+0xc8", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x2c", "lsl x0, x0, #1", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x2 (2)", "add x2, x2, #0x2 (2)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x1e (30)", + "sub x2, x2, #0x1e (30)", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x20 (32)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x20 (32)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x10 (16)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x1c", + "add x1, x1, #0x1e (30)", + "add x2, x2, #0x1e (30)", + "ldrh w3, [x2], #-2", + "strh w3, [x1], #-2", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -98,38 +246,112 @@ ] }, "rep movsd": { - "ExpectedInstructionCount": 37, + "ExpectedInstructionCount": 111, "Comment": "0xa5", "ExpectedArm64ASM": [ "ldrsb x22, [x28, #1018]", "mov x0, x7", "mov x1, x11", "mov x2, x10", - "tbnz w22, #1, #+0x38", - "cbz x0, #+0x30", - "lsl x0, x0, #2", + "tbnz w22, #1, #+0xc4", + "cbz x0, #+0xa8", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x1c", + "lsl x0, x0, #2", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x34", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x14", + "ldr w3, [x2], #4", + "str w3, [x1], #4", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", "add x21, x0, x2, lsl #2", "add x20, x1, x2, lsl #2", - "b #+0x44", - "cbz x0, #+0x40", + "b #+0xe0", + "cbz x0, #+0xc8", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x2c", "lsl x0, x0, #2", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x4 (4)", "add x2, x2, #0x4 (4)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x1c (28)", + "sub x2, x2, #0x1c (28)", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x10 (16)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x10 (16)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x8 (8)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x1c", + "add x1, x1, #0x1c (28)", + "add x2, x2, #0x1c (28)", + "ldr w3, [x2], #-4", + "str w3, [x1], #-4", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", @@ -141,38 +363,112 @@ ] }, "rep movsq": { - "ExpectedInstructionCount": 37, + "ExpectedInstructionCount": 111, "Comment": "0xa5", "ExpectedArm64ASM": [ "ldrsb x22, [x28, #1018]", "mov x0, x7", "mov x1, x11", "mov x2, x10", - "tbnz w22, #1, #+0x38", - "cbz x0, #+0x30", - "lsl x0, x0, #3", + "tbnz w22, #1, #+0xc4", + "cbz x0, #+0xa8", "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x1c", + "lsl x0, x0, #3", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x7c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x54", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x34", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #32", + "stp q0, q1, [x1], #32", + "sub x0, x0, #0x4 (4)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x4 (4)", + "cbz x0, #+0x14", + "ldr x3, [x2], #8", + "str x3, [x1], #8", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7", "add x21, x0, x2, lsl #3", "add x20, x1, x2, lsl #3", - "b #+0x44", - "cbz x0, #+0x40", + "b #+0xe0", + "cbz x0, #+0xc8", + "mrs x3, nzcv", + "sub x0, x1, x2", + "cmp x0, x7", + "mov x0, x7", + "bc.lt #+0x2c", "lsl x0, x0, #3", "sub x1, x1, x0", "sub x2, x2, x0", "add x1, x1, #0x8 (8)", "add x2, x2, #0x8 (8)", - "mrs x3, nzcv", "cpyfp [x1]!, [x2]!, x0!", "cpyfm [x1]!, [x2]!, x0!", "cpyfe [x1]!, [x2]!, x0!", "msr nzcv, x3", + "b #+0x8c", + "msr nzcv, x3", + "sub x3, x1, x2", + "tbz x3, #63, #+0x8", + "neg x3, x3", + "sub x3, x3, #0x20 (32)", + "tbnz x3, #63, #+0x64", + "sub x1, x1, #0x18 (24)", + "sub x2, x2, #0x18 (24)", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x44", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x1c", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x8 (8)", + "tbz x0, #63, #-0x14", + "add x0, x0, #0x8 (8)", + "cbz x0, #+0x3c", + "sub x0, x0, #0x4 (4)", + "tbnz x0, #63, #+0x14", + "ldp q0, q1, [x2], #-32", + "stp q0, q1, [x1], #-32", + "sub x0, x0, #0x4 (4)", + "tbz x0, #63, #-0xc", + "add x0, x0, #0x4 (4)", + "cbz x0, #+0x1c", + "add x1, x1, #0x18 (24)", + "add x2, x2, #0x18 (24)", + "ldr x3, [x2], #-8", + "str x3, [x1], #-8", + "sub x0, x0, #0x1 (1)", + "cbnz x0, #-0xc", "mov x0, x11", "mov x1, x10", "mov x2, x7",