From e1d21f0bff3f0dfd0485ffc0d4160dfae2fd2f83 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 3 Dec 2021 21:14:02 -0800 Subject: [PATCH] IR: Implements new CacheLineZero op This zeroes out an emulated 64byte cacheline. Writing zeros to memory. This very specifically is only 64bytes to match x86 behaviour. Also specifically non-temporal and weakly ordered. Which matches x86 CLZero behaviour. --- .../Core/Interpreter/InterpreterOps.cpp | 1 + .../Core/Interpreter/InterpreterOps.h | 1 + .../Interface/Core/Interpreter/MemoryOps.cpp | 17 ++++++++++++++ .../Interface/Core/JIT/Arm64/JITClass.h | 1 + .../Interface/Core/JIT/Arm64/MemoryOps.cpp | 23 +++++++++++++++++++ .../Interface/Core/JIT/x86_64/JITClass.h | 1 + .../Interface/Core/JIT/x86_64/MemoryOps.cpp | 18 +++++++++++++++ External/FEXCore/Source/Interface/IR/IR.json | 17 +++++++++++++- 8 files changed, 78 insertions(+), 1 deletion(-) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp index 249fc0c59..04d865132 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp @@ -157,6 +157,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] { REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement); REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement); REGISTER_OP(CACHELINECLEAR, CacheLineClear); + REGISTER_OP(CACHELINEZERO, CacheLineZero); // Misc ops REGISTER_OP(DUMMY, NoOp); diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index 940114083..662df2b51 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -180,6 +180,7 @@ namespace FEXCore::CPU { DEF_OP(VLoadMemElement); DEF_OP(VStoreMemElement); DEF_OP(CacheLineClear); + DEF_OP(CacheLineZero); ///< Misc ops DEF_OP(EndBlock); diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp index 3f4ea7df8..ed6c0cb03 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp @@ -264,5 +264,22 @@ DEF_OP(CacheLineClear) { CacheLineFlush(MemData); } +DEF_OP(CacheLineZero) { + auto Op = IROp->C(); + + uintptr_t MemData = *GetSrc(Data->SSAData, Op->Addr); + + // Force cacheline alignment + MemData = MemData & ~(CPUIDEmu::CACHELINE_SIZE - 1); + + using DataType = uint64_t; + DataType *MemData64 = reinterpret_cast(MemData); + + // 64-byte cache line zero + for (size_t i = 0; i < (CPUIDEmu::CACHELINE_SIZE / sizeof(DataType)); ++i) { + MemData64[i] = 0; + } +} + #undef DEF_OP } // namespace FEXCore::CPU diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index b269c74a2..253903380 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -321,6 +321,7 @@ private: DEF_OP(VLoadMemElement); DEF_OP(VStoreMemElement); DEF_OP(CacheLineClear); + DEF_OP(CacheLineZero); ///< Misc ops DEF_OP(EndBlock); diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp index 125f09f45..35a9031ec 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp @@ -946,6 +946,28 @@ DEF_OP(CacheLineClear) { dsb(InnerShareable, BarrierAll); } +DEF_OP(CacheLineZero) { + auto Op = IROp->C(); + + auto MemReg = GetReg(Op->Header.Args[0].ID()); + + if (CTX->HostFeatures.SupportsCLZERO) { + // We can use this instruction directly + dc(DataCacheOp::ZVA, MemReg); + } + else { + // We must walk the cacheline ourselves + // Force cacheline alignment + and_(TMP1, MemReg, ~(CPUIDEmu::CACHELINE_SIZE - 1)); + // This will end up being four STPs + // Depending on uarch it could be slightly more efficient in instructions emitted + // and uops to use vector pair STP, but we want the non-temporal bit specifically here + for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += 16) { + stnp(xzr, xzr, MemOperand(TMP1, i, Offset)); + } + } +} + #undef DEF_OP void Arm64JITCore::RegisterMemoryHandlers() { #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x @@ -972,6 +994,7 @@ void Arm64JITCore::RegisterMemoryHandlers() { REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement); REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement); REGISTER_OP(CACHELINECLEAR, CacheLineClear); + REGISTER_OP(CACHELINEZERO, CacheLineZero); #undef REGISTER_OP } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h index 55c9f4c46..d26b469d6 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h @@ -315,6 +315,7 @@ private: DEF_OP(VLoadMemElement); DEF_OP(VStoreMemElement); DEF_OP(CacheLineClear); + DEF_OP(CacheLineZero); ///< Misc ops DEF_OP(EndBlock); diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp index 52e9bc8f5..f8be6a794 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp @@ -595,6 +595,23 @@ DEF_OP(CacheLineClear) { clflush(ptr [MemReg]); } +DEF_OP(CacheLineZero) { + auto Op = IROp->C(); + + Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); + + // Align by cacheline + mov (TMP1, CPUIDEmu::CACHELINE_SIZE - 1); + andn(TMP1, TMP1, MemReg.cvt64()); + xor_(TMP2, TMP2); + + using DataType = uint64_t; + // 64-byte cache line zero + for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += sizeof(DataType)) { + mov (qword [TMP1 + i], TMP2); + } +} + #undef DEF_OP void X86JITCore::RegisterMemoryHandlers() { #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x @@ -615,6 +632,7 @@ void X86JITCore::RegisterMemoryHandlers() { REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement); REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement); REGISTER_OP(CACHELINECLEAR, CacheLineClear); + REGISTER_OP(CACHELINEZERO, CacheLineZero); #undef REGISTER_OP } } diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index 8c3dd834b..4cfeb4aff 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -858,7 +858,22 @@ }, "CacheLineClear": { - "Desc": ["Does a 64 byte cacheline clear at the address specified" + "Desc": ["Does a 64 byte cacheline clear at the address specified", + "Only clears the data cachelines. Doesn't do any zeroing" + ], + "HasSideEffects": true, + "OpClass": "Memory", + "SSAArgs": "1", + "SSANames": [ + "Addr" + ] + }, + + "CacheLineZero": { + "Desc": ["Does a 64 byte zero at the address specified", + "Writing zeroes to memory", + "It is specifically non-temporal and weakly ordered", + "This matches CLZero behaviour" ], "HasSideEffects": true, "OpClass": "Memory",