diff --git a/FEXCore/Source/Interface/Core/CPUBackend.cpp b/FEXCore/Source/Interface/Core/CPUBackend.cpp index 5f7416430..dec22b54f 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.cpp +++ b/FEXCore/Source/Interface/Core/CPUBackend.cpp @@ -186,6 +186,9 @@ CPUBackend::CPUBackend(FEXCore::Core::InternalThreadState *ThreadState, size_t I Common.NamedVectorConstantPointers[i] = reinterpret_cast(NamedVectorConstants[i]); } + // Copy named vector constants. + memcpy(Common.NamedVectorConstants, NamedVectorConstants, sizeof(NamedVectorConstants)); + // Initialize Indexed named vector constants. Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW] = reinterpret_cast(PSHUFLW_LUT.data()); Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW] = reinterpret_cast(PSHUFHW_LUT.data()); diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index a7b68a428..a5423982a 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -101,28 +101,42 @@ DEF_OP(LoadNamedVectorConstant) { } } // Load the pointer. - ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.NamedVectorConstantPointers[Op->Constant])); + auto GenerateMemOperand = [this](uint8_t OpSize, uint32_t NamedConstant, FEXCore::ARMEmitter::Register Base) { + const auto ConstantOffset = offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.NamedVectorConstants[NamedConstant]); + if (ConstantOffset <= 255 || // Unscaled 9-bit signed + ((ConstantOffset & (OpSize - 1)) == 0 && FEXCore::DividePow2(ConstantOffset, OpSize) <= 4095)) /* 12-bit unsigned scaled */ { + return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, ConstantOffset); + } + + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.NamedVectorConstantPointers[NamedConstant])); + return ARMEmitter::ExtendedMemOperand(TMP1, ARMEmitter::IndexType::OFFSET, 0); + }; + + if (OpSize == 32) { + // Handle SVE 32-byte variant upfront. + ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.NamedVectorConstantPointers[Op->Constant])); + ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), TMP1, 0); + return; + } + + auto MemOperand = GenerateMemOperand(OpSize, Op->Constant, STATE); switch (OpSize) { case 1: - ldrb(Dst, TMP1, 0); + ldrb(Dst, MemOperand); break; case 2: - ldrh(Dst, TMP1, 0); + ldrh(Dst, MemOperand); break; case 4: - ldr(Dst.S(), TMP1, 0); + ldr(Dst.S(), MemOperand); break; case 8: - ldr(Dst.D(), TMP1, 0); + ldr(Dst.D(), MemOperand); break; case 16: - ldr(Dst.Q(), TMP1, 0); + ldr(Dst.Q(), MemOperand); break; - case 32: { - ld1b(Dst.Z(), PRED_TMP_32B.Zeroing(), TMP1, 0); - break; - } default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); break; diff --git a/FEXCore/include/FEXCore/Core/CoreState.h b/FEXCore/include/FEXCore/Core/CoreState.h index 42d8f1ebd..1eb250245 100644 --- a/FEXCore/include/FEXCore/Core/CoreState.h +++ b/FEXCore/include/FEXCore/Core/CoreState.h @@ -259,6 +259,9 @@ namespace FEXCore::Core { uint64_t L1Pointer{}; uint64_t L2Pointer{}; /** @} */ + + // Copy of process-wide named vector constants data. + alignas(16) uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2]; } Common; union { diff --git a/FEXCore/include/FEXCore/Utils/MathUtils.h b/FEXCore/include/FEXCore/Utils/MathUtils.h index 76740ea5d..d86aa7fa6 100644 --- a/FEXCore/include/FEXCore/Utils/MathUtils.h +++ b/FEXCore/include/FEXCore/Utils/MathUtils.h @@ -25,4 +25,12 @@ requires(std::is_unsigned_v) return std::countr_zero(Value); } +// Divide a number by a power-of-2 by avoiding integer division. +// Can be a faster implementation than regular integer divide. +// Divisor requires to be power-of-2, is enforced in ilog2 helper. +template +requires(std::is_unsigned_v && std::is_unsigned_v) +[[nodiscard]] constexpr T DividePow2(T Dividend, TT Divisor) { + return Dividend >> ilog2(Divisor); +} } // namespace FEXCore diff --git a/FEXCore/unittests/APITests/ILog2.cpp b/FEXCore/unittests/APITests/ILog2.cpp index e4fca68de..5555fd71e 100644 --- a/FEXCore/unittests/APITests/ILog2.cpp +++ b/FEXCore/unittests/APITests/ILog2.cpp @@ -5,3 +5,9 @@ TEST_CASE("ILog2") { auto i = GENERATE(range(0, 64)); REQUIRE(FEXCore::ilog2(1ull << i) == i); } + +TEST_CASE("DividePow2") { + auto j = GENERATE(range(0, 64)); + auto i = GENERATE(range(0, 64)); + REQUIRE(FEXCore::DividePow2(1ull << j, 1ull << i) == ((1ull << j) / (1ull << i))); +} diff --git a/unittests/InstructionCountCI/H0F38.json b/unittests/InstructionCountCI/H0F38.json index ab50cb0d4..ebf8529d9 100644 --- a/unittests/InstructionCountCI/H0F38.json +++ b/unittests/InstructionCountCI/H0F38.json @@ -674,14 +674,13 @@ ] }, "phminposuw xmm0, xmm1": { - "ExpectedInstructionCount": 7, + "ExpectedInstructionCount": 6, "Optimal": "Yes", "Comment": [ "0x66 0x0f 0x38 0x41" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1544]", - "ldr q2, [x0]", + "ldr q2, [x28, #1888]", "zip1 v3.8h, v2.8h, v17.8h", "zip2 v2.8h, v2.8h, v17.8h", "umin v2.4s, v3.4s, v2.4s", diff --git a/unittests/InstructionCountCI/H0F3A.json b/unittests/InstructionCountCI/H0F3A.json index 9a784780b..9ec45c218 100644 --- a/unittests/InstructionCountCI/H0F3A.json +++ b/unittests/InstructionCountCI/H0F3A.json @@ -1550,14 +1550,13 @@ ] }, "aeskeygenassist xmm0, xmm1, 0": { - "ExpectedInstructionCount": 6, + "ExpectedInstructionCount": 5, "Optimal": "Yes", "Comment": [ "0x66 0x0f 0x3a 0xdf" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1600]", - "ldr q2, [x0]", + "ldr q2, [x28, #2000]", "movi v3.2d, #0x0", "mov v16.16b, v17.16b", "unimplemented (Unimplemented)", @@ -1565,14 +1564,13 @@ ] }, "aeskeygenassist xmm0, xmm1, 0xFF": { - "ExpectedInstructionCount": 9, + "ExpectedInstructionCount": 8, "Optimal": "No", "Comment": [ "0x66 0x0f 0x3a 0xdf" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1600]", - "ldr q2, [x0]", + "ldr q2, [x28, #2000]", "movi v3.2d, #0x0", "mov v16.16b, v17.16b", "unimplemented (Unimplemented)", diff --git a/unittests/InstructionCountCI/PrimaryGroup.json b/unittests/InstructionCountCI/PrimaryGroup.json index 0bb417746..3753de59b 100644 --- a/unittests/InstructionCountCI/PrimaryGroup.json +++ b/unittests/InstructionCountCI/PrimaryGroup.json @@ -3795,7 +3795,7 @@ "mov x0, x6", "mov x1, x20", "mov x2, x7", - "ldr x3, [x28, #1912]", + "ldr x3, [x28, #2048]", "str x30, [sp, #-16]!", "blr x3", "ldr x30, [sp], #16", @@ -3806,7 +3806,7 @@ "mov x0, x6", "mov x1, x20", "mov x2, x7", - "ldr x3, [x28, #1928]", + "ldr x3, [x28, #2064]", "str x30, [sp, #-16]!", "blr x3", "ldr x30, [sp], #16", @@ -3870,7 +3870,7 @@ "mov x0, x6", "mov x1, x20", "mov x2, x7", - "ldr x3, [x28, #1920]", + "ldr x3, [x28, #2056]", "str x30, [sp, #-16]!", "blr x3", "ldr x30, [sp], #16", @@ -3883,7 +3883,7 @@ "mov x0, x6", "mov x1, x20", "mov x2, x7", - "ldr x3, [x28, #1936]", + "ldr x3, [x28, #2072]", "str x30, [sp, #-16]!", "blr x3", "ldr x30, [sp], #16", diff --git a/unittests/InstructionCountCI/Secondary.json b/unittests/InstructionCountCI/Secondary.json index c839791d8..f201f7737 100644 --- a/unittests/InstructionCountCI/Secondary.json +++ b/unittests/InstructionCountCI/Secondary.json @@ -833,26 +833,24 @@ ] }, "movmskps eax, xmm0": { - "ExpectedInstructionCount": 6, + "ExpectedInstructionCount": 5, "Optimal": "Yes", "Comment": "0x0f 0x50", "ExpectedArm64ASM": [ "ushr v2.4s, v16.4s, #31", - "ldr x0, [x28, #1592]", - "ldr q3, [x0]", + "ldr q3, [x28, #1984]", "ushl v2.4s, v2.4s, v3.4s", "addv s2, v2.4s", "mov w4, v2.s[0]" ] }, "movmskps rax, xmm0": { - "ExpectedInstructionCount": 6, + "ExpectedInstructionCount": 5, "Optimal": "Yes", "Comment": "0x0f 0x50", "ExpectedArm64ASM": [ "ushr v2.4s, v16.4s, #31", - "ldr x0, [x28, #1592]", - "ldr q3, [x0]", + "ldr q3, [x28, #1984]", "ushl v2.4s, v2.4s, v3.4s", "addv s2, v2.4s", "mov w4, v2.s[0]" diff --git a/unittests/InstructionCountCI/Secondary_OpSize.json b/unittests/InstructionCountCI/Secondary_OpSize.json index fb43aaf8b..bf53ba026 100644 --- a/unittests/InstructionCountCI/Secondary_OpSize.json +++ b/unittests/InstructionCountCI/Secondary_OpSize.json @@ -1130,12 +1130,11 @@ ] }, "addsubpd xmm0, xmm1": { - "ExpectedInstructionCount": 4, + "ExpectedInstructionCount": 3, "Optimal": "Yes", "Comment": "0x66 0x0f 0xd0", "ExpectedArm64ASM": [ - "ldr x0, [x28, #1576]", - "ldr q2, [x0]", + "ldr q2, [x28, #1952]", "eor v2.16b, v17.16b, v2.16b", "fadd v16.2d, v16.2d, v2.2d" ] diff --git a/unittests/InstructionCountCI/Secondary_REPNE.json b/unittests/InstructionCountCI/Secondary_REPNE.json index 1230a4622..16c435991 100644 --- a/unittests/InstructionCountCI/Secondary_REPNE.json +++ b/unittests/InstructionCountCI/Secondary_REPNE.json @@ -511,12 +511,11 @@ ] }, "addsubps xmm0, xmm1": { - "ExpectedInstructionCount": 4, + "ExpectedInstructionCount": 3, "Optimal": "Yes", "Comment": "0xf2 0x0f 0xd0", "ExpectedArm64ASM": [ - "ldr x0, [x28, #1560]", - "ldr q2, [x0]", + "ldr q2, [x28, #1920]", "eor v2.16b, v17.16b, v2.16b", "fadd v16.4s, v16.4s, v2.4s" ] diff --git a/unittests/InstructionCountCI/VEX_map1.json b/unittests/InstructionCountCI/VEX_map1.json index b6df17a2e..51f129f0b 100644 --- a/unittests/InstructionCountCI/VEX_map1.json +++ b/unittests/InstructionCountCI/VEX_map1.json @@ -4449,14 +4449,13 @@ ] }, "vaddsubpd xmm0, xmm1, xmm2": { - "ExpectedInstructionCount": 5, + "ExpectedInstructionCount": 4, "Optimal": "No", "Comment": [ "Map 1 0b01 0xd0 128-bit" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1576]", - "ldr q2, [x0]", + "ldr q2, [x28, #1952]", "eor v2.16b, v18.16b, v2.16b", "fadd v2.2d, v17.2d, v2.2d", "mov v16.16b, v2.16b" @@ -4476,14 +4475,13 @@ ] }, "vaddsubps xmm0, xmm1, xmm2": { - "ExpectedInstructionCount": 5, + "ExpectedInstructionCount": 4, "Optimal": "No", "Comment": [ "Map 1 0b11 0xd0 128-bit" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1560]", - "ldr q2, [x0]", + "ldr q2, [x28, #1920]", "eor v2.16b, v18.16b, v2.16b", "fadd v2.4s, v17.4s, v2.4s", "mov v16.16b, v2.16b" diff --git a/unittests/InstructionCountCI/VEX_map2.json b/unittests/InstructionCountCI/VEX_map2.json index 202210415..cd5081c81 100644 --- a/unittests/InstructionCountCI/VEX_map2.json +++ b/unittests/InstructionCountCI/VEX_map2.json @@ -1579,14 +1579,13 @@ ] }, "vphminposuw xmm0, xmm1": { - "ExpectedInstructionCount": 8, + "ExpectedInstructionCount": 7, "Optimal": "No", "Comment": [ "Map 2 0b01 0x41 256-bit" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1544]", - "ldr q2, [x0]", + "ldr q2, [x28, #1888]", "zip1 v3.8h, v2.8h, v17.8h", "zip2 v2.8h, v2.8h, v17.8h", "umin v2.4s, v3.4s, v2.4s", diff --git a/unittests/InstructionCountCI/VEX_map3.json b/unittests/InstructionCountCI/VEX_map3.json index 65fa07bb3..d8a39ee2e 100644 --- a/unittests/InstructionCountCI/VEX_map3.json +++ b/unittests/InstructionCountCI/VEX_map3.json @@ -4713,14 +4713,13 @@ ] }, "vaeskeygenassist xmm0, xmm1, 0": { - "ExpectedInstructionCount": 7, + "ExpectedInstructionCount": 6, "Optimal": "No", "Comment": [ "Map 3 0b01 0xdf 128-bit" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1600]", - "ldr q2, [x0]", + "ldr q2, [x28, #2000]", "movi v3.2d, #0x0", "mov v2.16b, v17.16b", "unimplemented (Unimplemented)", @@ -4729,14 +4728,13 @@ ] }, "vaeskeygenassist xmm0, xmm1, 0xFF": { - "ExpectedInstructionCount": 10, + "ExpectedInstructionCount": 9, "Optimal": "No", "Comment": [ "Map 3 0b01 0xdf 128-bit" ], "ExpectedArm64ASM": [ - "ldr x0, [x28, #1600]", - "ldr q2, [x0]", + "ldr q2, [x28, #2000]", "movi v3.2d, #0x0", "mov v2.16b, v17.16b", "unimplemented (Unimplemented)",