steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 21:00:17 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity
Files
80687c8d2d79c2e7ab89abf5184530478622c525
FEX-Emu--FEX/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
T
Ryan Houdek 80687c8d2d ConstProp: Limits which addressing modes can be used for vector loadstores 
This was causing us to generate invalid code in Darwinia, resulting in a
crash. With assertions enabled this would be picked up in the emitter.

Only implement AddShift optimizations for now because I don't want to do
the remaining optimizations in a bug fix PR.

Fixes Darwinia.
2024-05-29 04:42:11 -07:00

1048 lines
38 KiB
C++
Raw Blame History

// SPDX-License-Identifier: MIT
/*
$info$
tags: ir|opts
desc: ConstProp, ZExt elim, addressgen coalesce, const pooling, fcmp reduction, const inlining
$end_info$
*/
// aarch64 heuristics
#include "aarch64/assembler-aarch64.h"
#include "aarch64/cpu-aarch64.h"
#include "aarch64/disasm-aarch64.h"
#include "aarch64/assembler-aarch64.h"
#include "Interface/IR/IREmitter.h"
#include "Interface/IR/PassManager.h"
#include <FEXCore/IR/IR.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Profiler.h>
#include <FEXCore/fextl/map.h>
#include <FEXCore/fextl/robin_map.h>
#include <FEXCore/fextl/unordered_map.h>
#include <bit>
#include <cstdint>
#include <memory>
#include <optional>
#include <string.h>
#include <tuple>
#include <utility>
namespace FEXCore::IR {
uint64_t getMask(IROp_Header* Op) {
uint64_t NumBits = Op->Size * 8;
return (~0ULL) >> (64 - NumBits);
}
// Returns true if the number bits from [0:width) contain the same bit.
// Ensuring that the consecutive bits in the range are entirely 0 or 1.
static bool HasConsecutiveBits(uint64_t imm, unsigned width) {
if (width == 0) {
return true;
}
// Credit to https://github.com/dougallj for this implementation.
return ((imm ^ (imm >> 1)) & ((1ULL << (width - 1)) - 1)) == 0;
}
// aarch64 heuristics
static bool IsImmLogical(uint64_t imm, unsigned width) {
if (width < 32) {
width = 32;
}
return vixl::aarch64::Assembler::IsImmLogical(imm, width);
}
static bool IsImmAddSub(uint64_t imm) {
return vixl::aarch64::Assembler::IsImmAddSub(imm);
}
static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) {
return Scale == AccessSize;
}
static bool IsSIMM9Range(uint64_t imm) {
// AArch64 signed immediate unscaled 9-bit range.
// Used for both regular unscaled loadstore instructions
// and LRPCPC2 unscaled loadstore instructions.
return ((int64_t)imm >= -256) && ((int64_t)imm <= 255);
}
static bool IsImmMemory(uint64_t imm, uint8_t AccessSize) {
if (IsSIMM9Range(imm)) {
return true;
} else if ((imm & (AccessSize - 1)) == 0 && imm / AccessSize <= 4095) {
return true;
} else {
return false;
}
}
static bool IsTSOImm9(uint64_t imm) {
// RCPC2 only has a 9-bit signed offset
if (IsSIMM9Range(imm)) {
return true;
} else {
return false;
}
}
struct MemExtendedAddrResult {
MemOffsetType OffsetType;
uint8_t OffsetScale;
OrderedNode* Base;
OrderedNode* OffsetReg;
};
static inline std::optional<MemExtendedAddrResult> TryAddShiftScale(IREmitter* IREmit, uint8_t AccessSize, IROp_Header* AddressHeader) {
auto AddShift = AddressHeader->C<IROp_AddShift>();
if (AddShift->Shift == IR::ShiftType::LSL) {
auto Scale = 1U << AddShift->ShiftAmount;
if (IsMemoryScale(Scale, AccessSize)) {
// remove shift as it can be folded to the mem op
return MemExtendedAddrResult {MEM_OFFSET_SXTX, (uint8_t)Scale, IREmit->UnwrapNode(AddShift->Src1), IREmit->UnwrapNode(AddShift->Src2)};
} else if (Scale == 1) {
return MemExtendedAddrResult {MEM_OFFSET_SXTX, 1, IREmit->UnwrapNode(AddShift->Src1), IREmit->UnwrapNode(AddShift->Src2)};
}
}
return std::nullopt;
}
// If this optimization doesn't succeed, it will return the nullopt
static std::optional<MemExtendedAddrResult> MemExtendedAddressing(IREmitter* IREmit, uint8_t AccessSize, IROp_Header* AddressHeader) {
// Try to optimize: AddShift Base, LSHL(Offset, Scale)
if (AddressHeader->Op == OP_ADDSHIFT) {
return TryAddShiftScale(IREmit, AccessSize, AddressHeader);
}
LOGMAN_THROW_A_FMT(AddressHeader->Op == OP_ADD, "Invalid address Op");
auto Src0Header = IREmit->GetOpHeader(AddressHeader->Args[0]);
if (Src0Header->Size == 8) {
// Try to optimize: Base + MUL(Offset, Scale)
if (Src0Header->Op == OP_MUL) {
uint64_t Scale;
if (IREmit->IsValueConstant(Src0Header->Args[1], &Scale)) {
if (IsMemoryScale(Scale, AccessSize)) {
// remove mul as it can be folded to the mem op
return MemExtendedAddrResult {MEM_OFFSET_SXTX, (uint8_t)Scale, IREmit->UnwrapNode(AddressHeader->Args[1]),
IREmit->UnwrapNode(Src0Header->Args[0])};
} else if (Scale == 1) {
// remove nop mul
return MemExtendedAddrResult {MEM_OFFSET_SXTX, 1, IREmit->UnwrapNode(AddressHeader->Args[1]), IREmit->UnwrapNode(Src0Header->Args[0])};
}
}
}
// Try to optimize: Base + LSHL(Offset, Scale)
else if (Src0Header->Op == OP_LSHL) {
uint64_t Constant2;
if (IREmit->IsValueConstant(Src0Header->Args[1], &Constant2)) {
uint8_t Scale = 1 << Constant2;
if (IsMemoryScale(Scale, AccessSize)) {
// remove shift as it can be folded to the mem op
return MemExtendedAddrResult {MEM_OFFSET_SXTX, Scale, IREmit->UnwrapNode(AddressHeader->Args[1]),
IREmit->UnwrapNode(Src0Header->Args[0])};
} else if (Scale == 1) {
// remove nop shift
return MemExtendedAddrResult {MEM_OFFSET_SXTX, 1, IREmit->UnwrapNode(AddressHeader->Args[1]), IREmit->UnwrapNode(Src0Header->Args[0])};
}
}
}
// Try to optimize: Base + (u32)Offset
else if (Src0Header->Op == OP_BFE) {
auto Bfe = Src0Header->C<IROp_Bfe>();
if (Bfe->lsb == 0 && Bfe->Width == 32) {
// todo: arm can also scale here
return MemExtendedAddrResult {MEM_OFFSET_UXTW, 1, IREmit->UnwrapNode(AddressHeader->Args[1]), IREmit->UnwrapNode(Src0Header->Args[0])};
}
}
// Try to optimize: Base + (s32)Offset
else if (Src0Header->Op == OP_SBFE) {
auto Sbfe = Src0Header->C<IROp_Sbfe>();
if (Sbfe->lsb == 0 && Sbfe->Width == 32) {
// todo: arm can also scale here
return MemExtendedAddrResult {MEM_OFFSET_SXTW, 1, IREmit->UnwrapNode(AddressHeader->Args[1]), IREmit->UnwrapNode(Src0Header->Args[0])};
}
}
}
// no match anywhere, just add
// However, if we have one 32bit negative constant, we need to sign extend it
auto Arg0_ = AddressHeader->Args[0];
auto Arg1_ = AddressHeader->Args[1];
auto Arg1H = IREmit->GetOpHeader(Arg1_);
auto Arg0 = IREmit->UnwrapNode(Arg0_);
auto Arg1 = IREmit->UnwrapNode(Arg1_);
uint64_t ConstVal = 0;
// Only optimize in 32bits reg+const where const < 16Kb.
if (Arg1H->Size == 4 && IREmit->IsValueConstant(Arg1_, &ConstVal)) {
// Base is Arg0, Constant (Displacement in Arg1)
OrderedNode* Base = Arg0;
OrderedNode* Cnt = Arg1;
int32_t Val32 = (int32_t)ConstVal;
if (Val32 > -16384 && Val32 < 0) {
return MemExtendedAddrResult {MEM_OFFSET_SXTW, 1, Base, Cnt};
} else if (Val32 >= 0 && Val32 < 16384) {
return MemExtendedAddrResult {MEM_OFFSET_SXTX, 1, Base, Cnt};
}
} else if (AddressHeader->Size == 4) {
// Do not optimize 32bit reg+reg.
// Something like :
// add w20, w7, w5
// ldr w7, [x20]
//
// cannot be simplified to (or any other single load instruction)
// ldr w7, [x5, w7, sxtx]
return std::nullopt;
} else {
return MemExtendedAddrResult {MEM_OFFSET_SXTX, 1, Arg0, Arg1};
}
return std::nullopt;
}
static std::optional<MemExtendedAddrResult> MemVectorAtomicExtendedAddressing(IREmitter* IREmit, uint8_t AccessSize, IROp_Header* AddressHeader) {
// Atomic TSO emulation of vectors use half-barriers. So it gets the full addressing support of vector loadstores
// Addressing capabilities
// - LDR, [Reg, Reg, LSL <Size>]
// - LDR, [Reg], imm12 Scaled <Size> ///< TODO: Implement this
// - LDUR, [Reg], imm9 (Signed [-256,256)) ///< TODO: Implement this
// TODO: Implement support for FEAT_LRCPC3.
// - LDAPUR [reg], imm9 (Signed [-256,256))
// Try to optimize: AddShift Base, LSHL(Offset, Scale)
if (AddressHeader->Op == OP_ADDSHIFT) {
return TryAddShiftScale(IREmit, AccessSize, AddressHeader);
}
return std::nullopt;
}
static bool IsBfeAlreadyDone(IREmitter* IREmit, OrderedNodeWrapper src, uint64_t Width) {
auto IROp = IREmit->GetOpHeader(src);
if (IROp->Op == OP_BFE) {
auto Op = IROp->C<IR::IROp_Bfe>();
if (Width >= Op->Width) {
return true;
}
}
return false;
}
class ConstProp final : public FEXCore::IR::Pass {
public:
explicit ConstProp(bool DoInlineConstants, bool SupportsTSOImm9, bool Is64BitMode)
: InlineConstants(DoInlineConstants)
, SupportsTSOImm9 {SupportsTSOImm9}
, Is64BitMode(Is64BitMode) {}
void Run(IREmitter* IREmit) override;
bool InlineConstants;
private:
void HandleConstantPools(IREmitter* IREmit, const IRListView& CurrentIR);
void ConstantPropagation(IREmitter* IREmit, const IRListView& CurrentIR, OrderedNode* CodeNode, IROp_Header* IROp);
void ConstantInlining(IREmitter* IREmit, const IRListView& CurrentIR);
struct ConstPoolData {
OrderedNode* Node;
IR::NodeID NodeID;
};
fextl::unordered_map<uint64_t, ConstPoolData> ConstPool;
fextl::map<OrderedNode*, uint64_t> AddressgenConsts;
// Pool inline constant generation. These are typically very small and pool efficiently.
fextl::robin_map<uint64_t, OrderedNode*> InlineConstantGen;
OrderedNode* CreateInlineConstant(IREmitter* IREmit, uint64_t Constant) {
const auto it = InlineConstantGen.find(Constant);
if (it != InlineConstantGen.end()) {
return it->second;
}
auto Result = InlineConstantGen.insert_or_assign(Constant, IREmit->_InlineConstant(Constant));
return Result.first->second;
}
bool SupportsTSOImm9 {};
bool Is64BitMode;
// This is a heuristic to limit constant pool live ranges to reduce RA interference pressure.
// If the range is unbounded then RA interference pressure seems to increase to the point
// that long blocks of constant usage can slow to a crawl.
// See https://github.com/FEX-Emu/FEX/issues/2688 for more information.
constexpr static uint32_t CONSTANT_POOL_RANGE_LIMIT = 500;
};
// Constants are pooled per block. Similarly for LoadMem / StoreMem, if imms are
// close by, use address gen to generate the values instead of using a new imm.
void ConstProp::HandleConstantPools(IREmitter* IREmit, const IRListView& CurrentIR) {
for (auto [BlockNode, BlockIROp] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Op == OP_LOADMEM || IROp->Op == OP_STOREMEM) {
size_t AddrIndex = 0;
size_t OffsetIndex = 0;
if (IROp->Op == OP_LOADMEM) {
AddrIndex = IR::IROp_LoadMem::Addr_Index;
OffsetIndex = IR::IROp_LoadMem::Offset_Index;
} else {
AddrIndex = IR::IROp_StoreMem::Addr_Index;
OffsetIndex = IR::IROp_StoreMem::Offset_Index;
}
uint64_t Addr;
if (IREmit->IsValueConstant(IROp->Args[AddrIndex], &Addr) && IROp->Args[OffsetIndex].IsInvalid()) {
for (auto& Const : AddressgenConsts) {
if ((Addr - Const.second) < 65536) {
IREmit->ReplaceNodeArgument(CodeNode, AddrIndex, Const.first);
IREmit->ReplaceNodeArgument(CodeNode, OffsetIndex, IREmit->_Constant(Addr - Const.second));
goto doneOp;
}
}
AddressgenConsts[IREmit->UnwrapNode(IROp->Args[AddrIndex])] = Addr;
}
doneOp:;
} else if (IROp->Op == OP_CONSTANT) {
auto Op = IROp->C<IR::IROp_Constant>();
const auto NewNodeID = CurrentIR.GetID(CodeNode);
auto it = ConstPool.find(Op->Constant);
if (it != ConstPool.end()) {
const auto OldNodeID = it->second.NodeID;
if ((NewNodeID.Value - OldNodeID.Value) > CONSTANT_POOL_RANGE_LIMIT) {
// Don't reuse if the live range is beyond the heurstic range.
// Update the tracked value to this new constant.
it->second.Node = CodeNode;
it->second.NodeID = NewNodeID;
continue;
}
auto CodeIter = CurrentIR.at(CodeNode);
IREmit->ReplaceUsesWithAfter(CodeNode, it->second.Node, CodeIter);
} else {
ConstPool[Op->Constant] = ConstPoolData {
.Node = CodeNode,
.NodeID = NewNodeID,
};
}
}
IREmit->SetWriteCursor(CodeNode);
}
AddressgenConsts.clear();
ConstPool.clear();
}
}
// constprop + some more per instruction logic
void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& CurrentIR, OrderedNode* CodeNode, IROp_Header* IROp) {
switch (IROp->Op) {
case OP_LOADMEMTSO: {
auto Op = IROp->CW<IR::IROp_LoadMemTSO>();
auto AddressHeader = IREmit->GetOpHeader(Op->Addr);
if (Op->Class == FEXCore::IR::FPRClass && AddressHeader->Size == 8) {
// TODO: LRCPC3 supports a vector unscaled offset like LRCPC2.
// Support once hardware is available to use this.
auto MaybeMemAddr = MemVectorAtomicExtendedAddressing(IREmit, IROp->Size, AddressHeader);
if (!MaybeMemAddr) {
break;
}
auto [OffsetType, OffsetScale, Base, OffsetReg] = *MaybeMemAddr;
Op->OffsetType = OffsetType;
Op->OffsetScale = OffsetScale;
IREmit->ReplaceNodeArgument(CodeNode, Op->Addr_Index, Base); // Addr
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, OffsetReg); // Offset
}
break;
}
case OP_STOREMEMTSO: {
auto Op = IROp->CW<IR::IROp_StoreMemTSO>();
auto AddressHeader = IREmit->GetOpHeader(Op->Addr);
if (Op->Class == FEXCore::IR::FPRClass && AddressHeader->Size == 8) {
// TODO: LRCPC3 supports a vector unscaled offset like LRCPC2.
// Support once hardware is available to use this.
auto MaybeMemAddr = MemVectorAtomicExtendedAddressing(IREmit, IROp->Size, AddressHeader);
if (!MaybeMemAddr) {
break;
}
auto [OffsetType, OffsetScale, Base, OffsetReg] = *MaybeMemAddr;
Op->OffsetType = OffsetType;
Op->OffsetScale = OffsetScale;
IREmit->ReplaceNodeArgument(CodeNode, Op->Addr_Index, Base); // Addr
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, OffsetReg); // Offset
}
break;
}
case OP_LOADMEM: {
auto Op = IROp->CW<IR::IROp_LoadMem>();
auto AddressHeader = IREmit->GetOpHeader(Op->Addr);
if (AddressHeader->Op == OP_ADD && ((Is64BitMode && AddressHeader->Size == 8) || (!Is64BitMode && AddressHeader->Size == 4))) {
auto MaybeMemAddr = MemExtendedAddressing(IREmit, IROp->Size, AddressHeader);
if (!MaybeMemAddr) {
break;
}
auto [OffsetType, OffsetScale, Base, OffsetReg] = *MaybeMemAddr;
Op->OffsetType = OffsetType;
Op->OffsetScale = OffsetScale;
IREmit->ReplaceNodeArgument(CodeNode, Op->Addr_Index, Base); // Addr
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, OffsetReg); // Offset
}
break;
}
case OP_STOREMEM: {
auto Op = IROp->CW<IR::IROp_StoreMem>();
auto AddressHeader = IREmit->GetOpHeader(Op->Addr);
if (AddressHeader->Op == OP_ADD && ((Is64BitMode && AddressHeader->Size == 8) || (!Is64BitMode && AddressHeader->Size == 4))) {
auto MaybeMemAddr = MemExtendedAddressing(IREmit, IROp->Size, AddressHeader);
if (!MaybeMemAddr) {
break;
}
auto [OffsetType, OffsetScale, Base, OffsetReg] = *MaybeMemAddr;
Op->OffsetType = OffsetType;
Op->OffsetScale = OffsetScale;
IREmit->ReplaceNodeArgument(CodeNode, Op->Addr_Index, Base); // Addr
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, OffsetReg); // Offset
}
break;
}
case OP_PREFETCH: {
auto Op = IROp->CW<IR::IROp_Prefetch>();
auto AddressHeader = IREmit->GetOpHeader(Op->Addr);
const bool SupportedOp = AddressHeader->Op == OP_ADD || AddressHeader->Op == OP_ADDSHIFT;
if (SupportedOp && ((Is64BitMode && AddressHeader->Size == 8) || (!Is64BitMode && AddressHeader->Size == 4))) {
auto MaybeMemAddr = MemExtendedAddressing(IREmit, IROp->Size, AddressHeader);
if (!MaybeMemAddr) {
break;
}
auto [OffsetType, OffsetScale, Base, OffsetReg] = *MaybeMemAddr;
Op->OffsetType = OffsetType;
Op->OffsetScale = OffsetScale;
IREmit->ReplaceNodeArgument(CodeNode, Op->Addr_Index, Base); // Addr
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, OffsetReg); // Offset
}
break;
}
case OP_ADD:
case OP_SUB:
case OP_ADDWITHFLAGS:
case OP_SUBWITHFLAGS: {
auto Op = IROp->C<IR::IROp_Add>();
uint64_t Constant1 {};
uint64_t Constant2 {};
bool IsConstant1 = IREmit->IsValueConstant(IROp->Args[0], &Constant1);
bool IsConstant2 = IREmit->IsValueConstant(IROp->Args[1], &Constant2);
if (IsConstant1 && IsConstant2 && IROp->Op == OP_ADD) {
uint64_t NewConstant = (Constant1 + Constant2) & getMask(IROp);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IsConstant1 && IsConstant2 && IROp->Op == OP_SUB) {
uint64_t NewConstant = (Constant1 - Constant2) & getMask(IROp);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IsConstant2 && !IsImmAddSub(Constant2) && IsImmAddSub(-Constant2)) {
// If the second argument is constant, the immediate is not ImmAddSub, but when negated is.
// So, negate the operation to negate (and inline) the constant.
if (IROp->Op == OP_ADD) {
IROp->Op = OP_SUB;
} else if (IROp->Op == OP_SUB) {
IROp->Op = OP_ADD;
} else if (IROp->Op == OP_ADDWITHFLAGS) {
IROp->Op = OP_SUBWITHFLAGS;
} else if (IROp->Op == OP_SUBWITHFLAGS) {
IROp->Op = OP_ADDWITHFLAGS;
}
IREmit->SetWriteCursorBefore(CodeNode);
// Negate the constant.
auto NegConstant = IREmit->_Constant(-Constant2);
// Replace the second source with the negated constant.
IREmit->ReplaceNodeArgument(CodeNode, Op->Src2_Index, NegConstant);
}
break;
}
case OP_SUBSHIFT: {
auto Op = IROp->C<IR::IROp_SubShift>();
uint64_t Constant1, Constant2;
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2) &&
Op->Shift == IR::ShiftType::LSL) {
// Optimize the LSL case when we know both sources are constant.
// This is a pattern that shows up with direction flag calculations if DF was set just before the operation.
uint64_t NewConstant = (Constant1 - (Constant2 << Op->ShiftAmount)) & getMask(IROp);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
}
break;
}
case OP_AND: {
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
uint64_t NewConstant = (Constant1 & Constant2) & getMask(IROp);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (Constant2 == 1) {
// happens from flag calcs
auto val = IREmit->GetOpHeader(IROp->Args[0]);
uint64_t Constant3;
if (val->Op == OP_SELECT && IREmit->IsValueConstant(val->Args[2], &Constant2) && IREmit->IsValueConstant(val->Args[3], &Constant3) &&
Constant2 == 1 && Constant3 == 0) {
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
}
} else if (IROp->Args[0].ID() == IROp->Args[1].ID()) {
// AND with same value results in original value
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
}
break;
}
case OP_OR: {
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
uint64_t NewConstant = Constant1 | Constant2;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IROp->Args[0].ID() == IROp->Args[1].ID()) {
// OR with same value results in original value
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
}
break;
}
case OP_ORLSHL: {
auto Op = IROp->CW<IR::IROp_Orlshl>();
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
uint64_t NewConstant = Constant1 | (Constant2 << Op->BitShift);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
}
break;
}
case OP_ORLSHR: {
auto Op = IROp->CW<IR::IROp_Orlshr>();
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
uint64_t NewConstant = Constant1 | (Constant2 >> Op->BitShift);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
}
break;
}
case OP_XOR: {
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
uint64_t NewConstant = Constant1 ^ Constant2;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IROp->Args[0].ID() == IROp->Args[1].ID()) {
// XOR with same value results to zero
IREmit->SetWriteCursor(CodeNode);
IREmit->ReplaceAllUsesWith(CodeNode, IREmit->_Constant(0));
} else {
// XOR with zero results in the nonzero source
for (unsigned i = 0; i < 2; ++i) {
if (!IREmit->IsValueConstant(IROp->Args[i], &Constant1)) {
continue;
}
if (Constant1 != 0) {
continue;
}
IREmit->SetWriteCursor(CodeNode);
OrderedNode* Arg = CurrentIR.GetNode(IROp->Args[1 - i]);
IREmit->ReplaceAllUsesWith(CodeNode, Arg);
break;
}
}
break;
}
case OP_NEG: {
uint64_t Constant {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant)) {
uint64_t NewConstant = -Constant;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
}
break;
}
case OP_LSHL: {
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
// Shifts mask the shift amount by 63 or 31 depending on operating size;
uint64_t ShiftMask = IROp->Size == 8 ? 63 : 31;
uint64_t NewConstant = (Constant1 << (Constant2 & ShiftMask)) & getMask(IROp);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IREmit->IsValueConstant(IROp->Args[1], &Constant2) && Constant2 == 0) {
IREmit->SetWriteCursor(CodeNode);
OrderedNode* Arg = CurrentIR.GetNode(IROp->Args[0]);
IREmit->ReplaceAllUsesWith(CodeNode, Arg);
}
break;
}
case OP_LSHR: {
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
// Shifts mask the shift amount by 63 or 31 depending on operating size;
uint64_t ShiftMask = IROp->Size == 8 ? 63 : 31;
uint64_t NewConstant = (Constant1 >> (Constant2 & ShiftMask)) & getMask(IROp);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IREmit->IsValueConstant(IROp->Args[1], &Constant2) && Constant2 == 0) {
IREmit->SetWriteCursor(CodeNode);
OrderedNode* Arg = CurrentIR.GetNode(IROp->Args[0]);
IREmit->ReplaceAllUsesWith(CodeNode, Arg);
}
break;
}
case OP_BFE: {
auto Op = IROp->C<IR::IROp_Bfe>();
uint64_t Constant;
// Is this value already BFE'd?
if (IsBfeAlreadyDone(IREmit, Op->Src, Op->Width)) {
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(Op->Src));
break;
}
// Is this value already ZEXT'd?
if (Op->lsb == 0) {
// LoadMem, LoadMemTSO & LoadContext ZExt
auto source = Op->Src;
auto sourceHeader = IREmit->GetOpHeader(source);
if (Op->Width >= (sourceHeader->Size * 8) &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)) {
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
break;
}
}
if (IROp->Size <= 8 && IREmit->IsValueConstant(Op->Src, &Constant)) {
uint64_t SourceMask = Op->Width == 64 ? ~0ULL : ((1ULL << Op->Width) - 1);
SourceMask <<= Op->lsb;
uint64_t NewConstant = (Constant & SourceMask) >> Op->lsb;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IROp->Size == CurrentIR.GetOp<IROp_Header>(IROp->Args[0])->Size && Op->Width == (IROp->Size * 8) && Op->lsb == 0) {
// A BFE that extracts all bits results in original value
// XXX - This is broken for now - see https://github.com/FEX-Emu/FEX/issues/351
// IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
} else if (Op->Width == 1 && Op->lsb == 0) {
// common from flag codegen
auto val = IREmit->GetOpHeader(IROp->Args[0]);
uint64_t Constant2 {};
uint64_t Constant3 {};
if (val->Op == OP_SELECT && IREmit->IsValueConstant(val->Args[2], &Constant2) && IREmit->IsValueConstant(val->Args[3], &Constant3) &&
Constant2 == 1 && Constant3 == 0) {
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
}
}
break;
}
case OP_SBFE: {
auto Op = IROp->C<IR::IROp_Bfe>();
uint64_t Constant;
if (IREmit->IsValueConstant(Op->Src, &Constant)) {
// SBFE of a constant can be converted to a constant.
uint64_t SourceMask = Op->Width == 64 ? ~0ULL : ((1ULL << Op->Width) - 1);
uint64_t DestSizeInBits = IROp->Size * 8;
uint64_t DestMask = DestSizeInBits == 64 ? ~0ULL : ((1ULL << DestSizeInBits) - 1);
SourceMask <<= Op->lsb;
int64_t NewConstant = (Constant & SourceMask) >> Op->lsb;
NewConstant <<= 64 - Op->Width;
NewConstant >>= 64 - Op->Width;
NewConstant &= DestMask;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
}
break;
}
case OP_BFI: {
auto Op = IROp->C<IR::IROp_Bfi>();
uint64_t ConstantDest {};
uint64_t ConstantSrc {};
bool DestIsConstant = IREmit->IsValueConstant(IROp->Args[0], &ConstantDest);
bool SrcIsConstant = IREmit->IsValueConstant(IROp->Args[1], &ConstantSrc);
if (DestIsConstant && SrcIsConstant) {
uint64_t SourceMask = Op->Width == 64 ? ~0ULL : ((1ULL << Op->Width) - 1);
uint64_t NewConstant = ConstantDest & ~(SourceMask << Op->lsb);
NewConstant |= (ConstantSrc & SourceMask) << Op->lsb;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (SrcIsConstant && HasConsecutiveBits(ConstantSrc, Op->Width)) {
// We are trying to insert constant, if it is a bitfield of only set bits then we can orr or and it.
IREmit->SetWriteCursor(CodeNode);
uint64_t SourceMask = Op->Width == 64 ? ~0ULL : ((1ULL << Op->Width) - 1);
uint64_t NewConstant = SourceMask << Op->lsb;
if (ConstantSrc & 1) {
auto orr = IREmit->_Or(IR::SizeToOpSize(IROp->Size), CurrentIR.GetNode(IROp->Args[0]), IREmit->_Constant(NewConstant));
IREmit->ReplaceAllUsesWith(CodeNode, orr);
} else {
// We are wanting to clear the bitfield.
auto andn = IREmit->_Andn(IR::SizeToOpSize(IROp->Size), CurrentIR.GetNode(IROp->Args[0]), IREmit->_Constant(NewConstant));
IREmit->ReplaceAllUsesWith(CodeNode, andn);
}
}
break;
}
case OP_MUL: {
uint64_t Constant1 {};
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) && IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
uint64_t NewConstant = (Constant1 * Constant2) & getMask(IROp);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
} else if (IREmit->IsValueConstant(IROp->Args[1], &Constant2) && std::popcount(Constant2) == 1) {
if (IROp->Size == 4 || IROp->Size == 8) {
uint64_t amt = std::countr_zero(Constant2);
IREmit->SetWriteCursor(CodeNode);
auto shift = IREmit->_Lshl(IR::SizeToOpSize(IROp->Size), CurrentIR.GetNode(IROp->Args[0]), IREmit->_Constant(amt));
IREmit->ReplaceAllUsesWith(CodeNode, shift);
}
}
break;
}
case OP_VMOV: {
// elim from load mem
auto source = IROp->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (IROp->Size >= sourceHeader->Size &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)) {
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
}
break;
}
default: break;
}
}
void ConstProp::ConstantInlining(IREmitter* IREmit, const IRListView& CurrentIR) {
InlineConstantGen.clear();
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
switch (IROp->Op) {
case OP_LSHR:
case OP_ASHR:
case OP_ROR:
case OP_LSHL: {
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
// this shouldn't be here, but rather on the emitter themselves or the constprop transformation?
if (IROp->Size <= 4) {
Constant2 &= 31;
} else {
Constant2 &= 63;
}
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
}
break;
}
case OP_ADD:
case OP_SUB:
case OP_ADDNZCV:
case OP_SUBNZCV:
case OP_ADDWITHFLAGS:
case OP_SUBWITHFLAGS: {
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
// We don't allow 8/16-bit operations to have constants, since no
// constant would be in bounds after the JIT's 24/16 shift.
if (IsImmAddSub(Constant2) && IROp->Size >= 4) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
}
} else if (IROp->Op == OP_SUBNZCV || IROp->Op == OP_SUBWITHFLAGS || IROp->Op == OP_SUB) {
// TODO: Generalize this
uint64_t Constant1 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
if (Constant1 == 0) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
IREmit->ReplaceNodeArgument(CodeNode, 0, CreateInlineConstant(IREmit, 0));
}
}
}
break;
}
case OP_ADC:
case OP_ADCWITHFLAGS: {
uint64_t Constant1 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
if (Constant1 == 0) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
IREmit->ReplaceNodeArgument(CodeNode, 0, CreateInlineConstant(IREmit, 0));
}
}
break;
}
case OP_RMIFNZCV: {
uint64_t Constant1 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
if (Constant1 == 0) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
IREmit->ReplaceNodeArgument(CodeNode, 0, CreateInlineConstant(IREmit, 0));
}
}
break;
}
case OP_CONDADDNZCV:
case OP_CONDSUBNZCV: {
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
}
}
uint64_t Constant1 {};
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
if (Constant1 == 0) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
IREmit->ReplaceNodeArgument(CodeNode, 0, CreateInlineConstant(IREmit, 0));
}
}
break;
}
case OP_TESTNZ: {
uint64_t Constant1 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant1)) {
if (IsImmLogical(Constant1, IROp->Size * 8)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant1));
}
}
break;
}
case OP_SELECT: {
uint64_t Constant1 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant1)) {
if (IsImmAddSub(Constant1)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant1));
}
}
uint64_t AllOnes = IROp->Size == 8 ? 0xffff'ffff'ffff'ffffull : 0xffff'ffffull;
uint64_t Constant2 {};
uint64_t Constant3 {};
if (IREmit->IsValueConstant(IROp->Args[2], &Constant2) && IREmit->IsValueConstant(IROp->Args[3], &Constant3) &&
(Constant2 == 1 || Constant2 == AllOnes) && Constant3 == 0) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[2]));
IREmit->ReplaceNodeArgument(CodeNode, 2, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 3, CreateInlineConstant(IREmit, Constant3));
}
break;
}
case OP_NZCVSELECT: {
uint64_t AllOnes = IROp->Size == 8 ? 0xffff'ffff'ffff'ffffull : 0xffff'ffffull;
// We always allow source 1 to be zero, but source 0 can only be a
// special 1/~0 constant if source 1 is 0.
uint64_t Constant0 {};
uint64_t Constant1 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant1) && Constant1 == 0) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant1));
if (IREmit->IsValueConstant(IROp->Args[0], &Constant0) && (Constant0 == 1 || Constant0 == AllOnes)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
IREmit->ReplaceNodeArgument(CodeNode, 0, CreateInlineConstant(IREmit, Constant0));
}
}
break;
}
case OP_CONDJUMP: {
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
}
}
break;
}
case OP_EXITFUNCTION: {
auto Op = IROp->C<IR::IROp_ExitFunction>();
uint64_t Constant {};
if (IREmit->IsValueConstant(Op->NewRIP, &Constant)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, CreateInlineConstant(IREmit, Constant));
} else {
auto NewRIP = IREmit->GetOpHeader(Op->NewRIP);
if (NewRIP->Op == OP_ENTRYPOINTOFFSET) {
auto EO = NewRIP->C<IR::IROp_EntrypointOffset>();
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineEntrypointOffset(IR::SizeToOpSize(EO->Header.Size), EO->Offset));
}
}
break;
}
case OP_OR:
case OP_XOR:
case OP_AND:
case OP_ANDWITHFLAGS:
case OP_ANDN: {
uint64_t Constant2 {};
if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
if (IsImmLogical(Constant2, IROp->Size * 8)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
}
}
break;
}
case OP_LOADMEM: {
auto Op = IROp->CW<IR::IROp_LoadMem>();
uint64_t Constant2 {};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Offset, &Constant2)) {
if (IsImmMemory(Constant2, IROp->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
}
}
break;
}
case OP_STOREMEM: {
auto Op = IROp->CW<IR::IROp_StoreMem>();
uint64_t Constant2 {};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Offset, &Constant2)) {
if (IsImmMemory(Constant2, IROp->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
}
}
break;
}
case OP_LOADMEMTSO: {
auto Op = IROp->CW<IR::IROp_LoadMemTSO>();
uint64_t Constant2 {};
if (SupportsTSOImm9) {
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Offset, &Constant2)) {
if (IsTSOImm9(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
}
}
}
break;
}
case OP_STOREMEMTSO: {
auto Op = IROp->CW<IR::IROp_StoreMemTSO>();
uint64_t Constant2 {};
if (SupportsTSOImm9) {
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Offset, &Constant2)) {
if (IsTSOImm9(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
}
}
}
break;
}
case OP_MEMCPY: {
auto Op = IROp->CW<IR::IROp_MemCpy>();
uint64_t Constant {};
if (IREmit->IsValueConstant(Op->Direction, &Constant)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Direction));
IREmit->ReplaceNodeArgument(CodeNode, Op->Direction_Index, CreateInlineConstant(IREmit, Constant));
}
break;
}
case OP_MEMSET: {
auto Op = IROp->CW<IR::IROp_MemSet>();
uint64_t Constant {};
if (IREmit->IsValueConstant(Op->Direction, &Constant)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Direction));
IREmit->ReplaceNodeArgument(CodeNode, Op->Direction_Index, CreateInlineConstant(IREmit, Constant));
}
break;
}
case OP_PREFETCH: {
auto Op = IROp->CW<IR::IROp_Prefetch>();
uint64_t Constant2 {};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Offset, &Constant2)) {
if (IsImmMemory(Constant2, IROp->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
}
}
break;
}
default: break;
}
}
}
void ConstProp::Run(IREmitter* IREmit) {
FEXCORE_PROFILE_SCOPED("PassManager::ConstProp");
auto CurrentIR = IREmit->ViewIR();
HandleConstantPools(IREmit, CurrentIR);
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
ConstantPropagation(IREmit, CurrentIR, CodeNode, IROp);
}
if (InlineConstants) {
ConstantInlining(IREmit, CurrentIR);
}
}
fextl::unique_ptr<FEXCore::IR::Pass> CreateConstProp(bool InlineConstants, bool SupportsTSOImm9, bool Is64BitMode) {
return fextl::make_unique<ConstProp>(InlineConstants, SupportsTSOImm9, Is64BitMode);
}
} // namespace FEXCore::IR
Reference in new issue View Git Blame Copy Permalink