122c310d78ce9630af15b0de94c18a409705b7496Tim Murray//===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===// 222c310d78ce9630af15b0de94c18a409705b7496Tim Murray// 322c310d78ce9630af15b0de94c18a409705b7496Tim Murray// The LLVM Compiler Infrastructure 422c310d78ce9630af15b0de94c18a409705b7496Tim Murray// 522c310d78ce9630af15b0de94c18a409705b7496Tim Murray// This file is distributed under the University of Illinois Open Source 622c310d78ce9630af15b0de94c18a409705b7496Tim Murray// License. See LICENSE.TXT for details. 722c310d78ce9630af15b0de94c18a409705b7496Tim Murray// 822c310d78ce9630af15b0de94c18a409705b7496Tim Murray//===----------------------------------------------------------------------===// 922c310d78ce9630af15b0de94c18a409705b7496Tim Murray// For best-case performance on Cortex-A57, we should try to use a balanced 1022c310d78ce9630af15b0de94c18a409705b7496Tim Murray// mix of odd and even D-registers when performing a critical sequence of 1122c310d78ce9630af15b0de94c18a409705b7496Tim Murray// independent, non-quadword FP/ASIMD floating-point multiply or 1222c310d78ce9630af15b0de94c18a409705b7496Tim Murray// multiply-accumulate operations. 1322c310d78ce9630af15b0de94c18a409705b7496Tim Murray// 1422c310d78ce9630af15b0de94c18a409705b7496Tim Murray// This pass attempts to detect situations where the register allocation may 1522c310d78ce9630af15b0de94c18a409705b7496Tim Murray// adversely affect this load balancing and to change the registers used so as 1622c310d78ce9630af15b0de94c18a409705b7496Tim Murray// to better utilize the CPU. 1722c310d78ce9630af15b0de94c18a409705b7496Tim Murray// 1822c310d78ce9630af15b0de94c18a409705b7496Tim Murray// Ideally we'd just take each multiply or multiply-accumulate in turn and 1922c310d78ce9630af15b0de94c18a409705b7496Tim Murray// allocate it alternating even or odd registers. However, multiply-accumulates 2022c310d78ce9630af15b0de94c18a409705b7496Tim Murray// are most efficiently performed in the same functional unit as their 2122c310d78ce9630af15b0de94c18a409705b7496Tim Murray// accumulation operand. Therefore this pass tries to find maximal sequences 2222c310d78ce9630af15b0de94c18a409705b7496Tim Murray// ("Chains") of multiply-accumulates linked via their accumulation operand, 2322c310d78ce9630af15b0de94c18a409705b7496Tim Murray// and assign them all the same "color" (oddness/evenness). 2422c310d78ce9630af15b0de94c18a409705b7496Tim Murray// 2522c310d78ce9630af15b0de94c18a409705b7496Tim Murray// This optimization affects S-register and D-register floating point 2622c310d78ce9630af15b0de94c18a409705b7496Tim Murray// multiplies and FMADD/FMAs, as well as vector (floating point only) muls and 2722c310d78ce9630af15b0de94c18a409705b7496Tim Murray// FMADD/FMA. Q register instructions (and 128-bit vector instructions) are 2822c310d78ce9630af15b0de94c18a409705b7496Tim Murray// not affected. 2922c310d78ce9630af15b0de94c18a409705b7496Tim Murray//===----------------------------------------------------------------------===// 3022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 3122c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "AArch64.h" 3222c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "AArch64InstrInfo.h" 3322c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "AArch64Subtarget.h" 3422c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/ADT/BitVector.h" 3522c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/ADT/EquivalenceClasses.h" 3622c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/CodeGen/MachineFunction.h" 3722c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/CodeGen/MachineFunctionPass.h" 3822c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/CodeGen/MachineInstr.h" 3922c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/CodeGen/MachineInstrBuilder.h" 4022c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/CodeGen/MachineRegisterInfo.h" 4122c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/CodeGen/RegisterScavenging.h" 4222c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/CodeGen/RegisterClassInfo.h" 4322c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/Support/CommandLine.h" 4422c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/Support/Debug.h" 4522c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include "llvm/Support/raw_ostream.h" 4622c310d78ce9630af15b0de94c18a409705b7496Tim Murray#include <list> 4722c310d78ce9630af15b0de94c18a409705b7496Tim Murrayusing namespace llvm; 4822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 4922c310d78ce9630af15b0de94c18a409705b7496Tim Murray#define DEBUG_TYPE "aarch64-a57-fp-load-balancing" 5022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 5122c310d78ce9630af15b0de94c18a409705b7496Tim Murray// Enforce the algorithm to use the scavenged register even when the original 5222c310d78ce9630af15b0de94c18a409705b7496Tim Murray// destination register is the correct color. Used for testing. 5322c310d78ce9630af15b0de94c18a409705b7496Tim Murraystatic cl::opt<bool> 5422c310d78ce9630af15b0de94c18a409705b7496Tim MurrayTransformAll("aarch64-a57-fp-load-balancing-force-all", 5522c310d78ce9630af15b0de94c18a409705b7496Tim Murray cl::desc("Always modify dest registers regardless of color"), 5622c310d78ce9630af15b0de94c18a409705b7496Tim Murray cl::init(false), cl::Hidden); 5722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 5822c310d78ce9630af15b0de94c18a409705b7496Tim Murray// Never use the balance information obtained from chains - return a specific 5922c310d78ce9630af15b0de94c18a409705b7496Tim Murray// color always. Used for testing. 6022c310d78ce9630af15b0de94c18a409705b7496Tim Murraystatic cl::opt<unsigned> 6122c310d78ce9630af15b0de94c18a409705b7496Tim MurrayOverrideBalance("aarch64-a57-fp-load-balancing-override", 6222c310d78ce9630af15b0de94c18a409705b7496Tim Murray cl::desc("Ignore balance information, always return " 6322c310d78ce9630af15b0de94c18a409705b7496Tim Murray "(1: Even, 2: Odd)."), 6422c310d78ce9630af15b0de94c18a409705b7496Tim Murray cl::init(0), cl::Hidden); 6522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 6622c310d78ce9630af15b0de94c18a409705b7496Tim Murray//===----------------------------------------------------------------------===// 6722c310d78ce9630af15b0de94c18a409705b7496Tim Murray// Helper functions 6822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 6922c310d78ce9630af15b0de94c18a409705b7496Tim Murray// Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs? 7022c310d78ce9630af15b0de94c18a409705b7496Tim Murraystatic bool isMul(MachineInstr *MI) { 7122c310d78ce9630af15b0de94c18a409705b7496Tim Murray switch (MI->getOpcode()) { 7222c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FMULSrr: 7322c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FNMULSrr: 7422c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FMULDrr: 7522c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FNMULDrr: 7622c310d78ce9630af15b0de94c18a409705b7496Tim Murray return true; 7722c310d78ce9630af15b0de94c18a409705b7496Tim Murray default: 7822c310d78ce9630af15b0de94c18a409705b7496Tim Murray return false; 7922c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 8022c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 8122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 8222c310d78ce9630af15b0de94c18a409705b7496Tim Murray// Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs? 8322c310d78ce9630af15b0de94c18a409705b7496Tim Murraystatic bool isMla(MachineInstr *MI) { 8422c310d78ce9630af15b0de94c18a409705b7496Tim Murray switch (MI->getOpcode()) { 8522c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FMSUBSrrr: 8622c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FMADDSrrr: 8722c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FNMSUBSrrr: 8822c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FNMADDSrrr: 8922c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FMSUBDrrr: 9022c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FMADDDrrr: 9122c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FNMSUBDrrr: 9222c310d78ce9630af15b0de94c18a409705b7496Tim Murray case AArch64::FNMADDDrrr: 9322c310d78ce9630af15b0de94c18a409705b7496Tim Murray return true; 9422c310d78ce9630af15b0de94c18a409705b7496Tim Murray default: 9522c310d78ce9630af15b0de94c18a409705b7496Tim Murray return false; 9622c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 9722c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 9822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 9922c310d78ce9630af15b0de94c18a409705b7496Tim Murray//===----------------------------------------------------------------------===// 10022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 10122c310d78ce9630af15b0de94c18a409705b7496Tim Murraynamespace { 10222c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// A "color", which is either even or odd. Yes, these aren't really colors 10322c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// but the algorithm is conceptually doing two-color graph coloring. 10422c310d78ce9630af15b0de94c18a409705b7496Tim Murrayenum class Color { Even, Odd }; 10522c310d78ce9630af15b0de94c18a409705b7496Tim Murraystatic const char *ColorNames[2] = { "Even", "Odd" }; 10622c310d78ce9630af15b0de94c18a409705b7496Tim Murray 10722c310d78ce9630af15b0de94c18a409705b7496Tim Murrayclass Chain; 10822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 10922c310d78ce9630af15b0de94c18a409705b7496Tim Murrayclass AArch64A57FPLoadBalancing : public MachineFunctionPass { 11022c310d78ce9630af15b0de94c18a409705b7496Tim Murray const AArch64InstrInfo *TII; 11122c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineRegisterInfo *MRI; 11222c310d78ce9630af15b0de94c18a409705b7496Tim Murray const TargetRegisterInfo *TRI; 11322c310d78ce9630af15b0de94c18a409705b7496Tim Murray RegisterClassInfo RCI; 11422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 11522c310d78ce9630af15b0de94c18a409705b7496Tim Murraypublic: 11622c310d78ce9630af15b0de94c18a409705b7496Tim Murray static char ID; 11722c310d78ce9630af15b0de94c18a409705b7496Tim Murray explicit AArch64A57FPLoadBalancing() : MachineFunctionPass(ID) {} 11822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 11922c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool runOnMachineFunction(MachineFunction &F) override; 12022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 12122c310d78ce9630af15b0de94c18a409705b7496Tim Murray const char *getPassName() const override { 12222c310d78ce9630af15b0de94c18a409705b7496Tim Murray return "A57 FP Anti-dependency breaker"; 12322c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 12422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 12522c310d78ce9630af15b0de94c18a409705b7496Tim Murray void getAnalysisUsage(AnalysisUsage &AU) const override { 12622c310d78ce9630af15b0de94c18a409705b7496Tim Murray AU.setPreservesCFG(); 12722c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineFunctionPass::getAnalysisUsage(AU); 12822c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 12922c310d78ce9630af15b0de94c18a409705b7496Tim Murray 13022c310d78ce9630af15b0de94c18a409705b7496Tim Murrayprivate: 13122c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool runOnBasicBlock(MachineBasicBlock &MBB); 13222c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool colorChainSet(std::vector<Chain*> GV, MachineBasicBlock &MBB, 13322c310d78ce9630af15b0de94c18a409705b7496Tim Murray int &Balance); 13422c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB); 13522c310d78ce9630af15b0de94c18a409705b7496Tim Murray int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB); 13622c310d78ce9630af15b0de94c18a409705b7496Tim Murray void scanInstruction(MachineInstr *MI, unsigned Idx, 13750721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines std::map<unsigned, Chain*> &Active, 13850721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines std::set<std::unique_ptr<Chain>> &AllChains); 13922c310d78ce9630af15b0de94c18a409705b7496Tim Murray void maybeKillChain(MachineOperand &MO, unsigned Idx, 14022c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::map<unsigned, Chain*> &RegChains); 14122c310d78ce9630af15b0de94c18a409705b7496Tim Murray Color getColor(unsigned Register); 14222c310d78ce9630af15b0de94c18a409705b7496Tim Murray Chain *getAndEraseNext(Color PreferredColor, std::vector<Chain*> &L); 14322c310d78ce9630af15b0de94c18a409705b7496Tim Murray}; 14422c310d78ce9630af15b0de94c18a409705b7496Tim Murraychar AArch64A57FPLoadBalancing::ID = 0; 14522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 14622c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// A Chain is a sequence of instructions that are linked together by 14722c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// an accumulation operand. For example: 14822c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// 14922c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// fmul d0<def>, ? 15022c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// fmla d1<def>, ?, ?, d0<kill> 15122c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// fmla d2<def>, ?, ?, d1<kill> 15222c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// 15322c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// There may be other instructions interleaved in the sequence that 15422c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// do not belong to the chain. These other instructions must not use 15522c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// the "chain" register at any point. 15622c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// 15722c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// We currently only support chains where the "chain" operand is killed 15822c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// at each link in the chain for simplicity. 15922c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// A chain has three important instructions - Start, Last and Kill. 16022c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// * The start instruction is the first instruction in the chain. 16122c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// * Last is the final instruction in the chain. 16222c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// * Kill may or may not be defined. If defined, Kill is the instruction 16322c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// where the outgoing value of the Last instruction is killed. 16422c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// This information is important as if we know the outgoing value is 16522c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// killed with no intervening uses, we can safely change its register. 16622c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// 16722c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// Without a kill instruction, we must assume the outgoing value escapes 16822c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// beyond our model and either must not change its register or must 16922c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// create a fixup FMOV to keep the old register value consistent. 17022c310d78ce9630af15b0de94c18a409705b7496Tim Murray/// 17122c310d78ce9630af15b0de94c18a409705b7496Tim Murrayclass Chain { 17222c310d78ce9630af15b0de94c18a409705b7496Tim Murraypublic: 17322c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// The important (marker) instructions. 17422c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineInstr *StartInst, *LastInst, *KillInst; 17522c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// The index, from the start of the basic block, that each marker 17622c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// appears. These are stored so we can do quick interval tests. 17722c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned StartInstIdx, LastInstIdx, KillInstIdx; 17822c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// All instructions in the chain. 17922c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::set<MachineInstr*> Insts; 18022c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// True if KillInst cannot be modified. If this is true, 18122c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// we cannot change LastInst's outgoing register. 18222c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// This will be true for tied values and regmasks. 18322c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool KillIsImmutable; 18422c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// The "color" of LastInst. This will be the preferred chain color, 18522c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// as changing intermediate nodes is easy but changing the last 18622c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// instruction can be more tricky. 18722c310d78ce9630af15b0de94c18a409705b7496Tim Murray Color LastColor; 18822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 18950721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines Chain(MachineInstr *MI, unsigned Idx, Color C) 19050721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines : StartInst(MI), LastInst(MI), KillInst(nullptr), 19150721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0), 19250721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines LastColor(C) { 19322c310d78ce9630af15b0de94c18a409705b7496Tim Murray Insts.insert(MI); 19422c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 19522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 19622c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Add a new instruction into the chain. The instruction's dest operand 19722c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// has the given color. 19822c310d78ce9630af15b0de94c18a409705b7496Tim Murray void add(MachineInstr *MI, unsigned Idx, Color C) { 19922c310d78ce9630af15b0de94c18a409705b7496Tim Murray LastInst = MI; 20022c310d78ce9630af15b0de94c18a409705b7496Tim Murray LastInstIdx = Idx; 20122c310d78ce9630af15b0de94c18a409705b7496Tim Murray LastColor = C; 20250721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && 20350721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines "Chain: broken invariant. A Chain can only be killed after its last " 20450721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines "def"); 20522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 20622c310d78ce9630af15b0de94c18a409705b7496Tim Murray Insts.insert(MI); 20722c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 20822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 20922c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return true if MI is a member of the chain. 21022c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool contains(MachineInstr *MI) { return Insts.count(MI) > 0; } 21122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 21222c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return the number of instructions in the chain. 21322c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned size() const { 21422c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Insts.size(); 21522c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 21622c310d78ce9630af15b0de94c18a409705b7496Tim Murray 21722c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Inform the chain that its last active register (the dest register of 21822c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// LastInst) is killed by MI with no intervening uses or defs. 21922c310d78ce9630af15b0de94c18a409705b7496Tim Murray void setKill(MachineInstr *MI, unsigned Idx, bool Immutable) { 22022c310d78ce9630af15b0de94c18a409705b7496Tim Murray KillInst = MI; 22122c310d78ce9630af15b0de94c18a409705b7496Tim Murray KillInstIdx = Idx; 22222c310d78ce9630af15b0de94c18a409705b7496Tim Murray KillIsImmutable = Immutable; 22350721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && 22450721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines "Chain: broken invariant. A Chain can only be killed after its last " 22550721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines "def"); 22622c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 22722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 22822c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return the first instruction in the chain. 22922c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineInstr *getStart() const { return StartInst; } 23022c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return the last instruction in the chain. 23122c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineInstr *getLast() const { return LastInst; } 23222c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return the "kill" instruction (as set with setKill()) or NULL. 23322c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineInstr *getKill() const { return KillInst; } 23422c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return an instruction that can be used as an iterator for the end 23522c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// of the chain. This is the maximum of KillInst (if set) and LastInst. 23650721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines MachineBasicBlock::iterator getEnd() const { 23722c310d78ce9630af15b0de94c18a409705b7496Tim Murray return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst); 23822c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 23922c310d78ce9630af15b0de94c18a409705b7496Tim Murray 24022c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Can the Kill instruction (assuming one exists) be modified? 24122c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool isKillImmutable() const { return KillIsImmutable; } 24222c310d78ce9630af15b0de94c18a409705b7496Tim Murray 24322c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return the preferred color of this chain. 24422c310d78ce9630af15b0de94c18a409705b7496Tim Murray Color getPreferredColor() { 24522c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (OverrideBalance != 0) 24622c310d78ce9630af15b0de94c18a409705b7496Tim Murray return OverrideBalance == 1 ? Color::Even : Color::Odd; 24722c310d78ce9630af15b0de94c18a409705b7496Tim Murray return LastColor; 24822c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 24922c310d78ce9630af15b0de94c18a409705b7496Tim Murray 25022c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return true if this chain (StartInst..KillInst) overlaps with Other. 25150721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines bool rangeOverlapsWith(const Chain &Other) const { 25222c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned End = KillInst ? KillInstIdx : LastInstIdx; 25350721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines unsigned OtherEnd = Other.KillInst ? 25450721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines Other.KillInstIdx : Other.LastInstIdx; 25522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 25650721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End; 25722c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 25822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 25922c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return true if this chain starts before Other. 26022c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool startsBefore(Chain *Other) { 26122c310d78ce9630af15b0de94c18a409705b7496Tim Murray return StartInstIdx < Other->StartInstIdx; 26222c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 26322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 26422c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return true if the group will require a fixup MOV at the end. 26522c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool requiresFixup() const { 26622c310d78ce9630af15b0de94c18a409705b7496Tim Murray return (getKill() && isKillImmutable()) || !getKill(); 26722c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 26822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 26922c310d78ce9630af15b0de94c18a409705b7496Tim Murray /// Return a simple string representation of the chain. 27022c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::string str() const { 27122c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::string S; 27222c310d78ce9630af15b0de94c18a409705b7496Tim Murray raw_string_ostream OS(S); 27322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 27422c310d78ce9630af15b0de94c18a409705b7496Tim Murray OS << "{"; 27522c310d78ce9630af15b0de94c18a409705b7496Tim Murray StartInst->print(OS, NULL, true); 27622c310d78ce9630af15b0de94c18a409705b7496Tim Murray OS << " -> "; 27722c310d78ce9630af15b0de94c18a409705b7496Tim Murray LastInst->print(OS, NULL, true); 27822c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (KillInst) { 27922c310d78ce9630af15b0de94c18a409705b7496Tim Murray OS << " (kill @ "; 28022c310d78ce9630af15b0de94c18a409705b7496Tim Murray KillInst->print(OS, NULL, true); 28122c310d78ce9630af15b0de94c18a409705b7496Tim Murray OS << ")"; 28222c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 28322c310d78ce9630af15b0de94c18a409705b7496Tim Murray OS << "}"; 28422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 28522c310d78ce9630af15b0de94c18a409705b7496Tim Murray return OS.str(); 28622c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 28722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 28822c310d78ce9630af15b0de94c18a409705b7496Tim Murray}; 28922c310d78ce9630af15b0de94c18a409705b7496Tim Murray 29022c310d78ce9630af15b0de94c18a409705b7496Tim Murray} // end anonymous namespace 29122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 29222c310d78ce9630af15b0de94c18a409705b7496Tim Murray//===----------------------------------------------------------------------===// 29322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 29422c310d78ce9630af15b0de94c18a409705b7496Tim Murraybool AArch64A57FPLoadBalancing::runOnMachineFunction(MachineFunction &F) { 29522c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool Changed = false; 29622c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "***** AArch64A57FPLoadBalancing *****\n"); 29722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 29822c310d78ce9630af15b0de94c18a409705b7496Tim Murray const TargetMachine &TM = F.getTarget(); 29922c310d78ce9630af15b0de94c18a409705b7496Tim Murray MRI = &F.getRegInfo(); 30022c310d78ce9630af15b0de94c18a409705b7496Tim Murray TRI = F.getRegInfo().getTargetRegisterInfo(); 30122c310d78ce9630af15b0de94c18a409705b7496Tim Murray TII = TM.getSubtarget<AArch64Subtarget>().getInstrInfo(); 30222c310d78ce9630af15b0de94c18a409705b7496Tim Murray RCI.runOnMachineFunction(F); 30322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 30422c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto &MBB : F) { 30522c310d78ce9630af15b0de94c18a409705b7496Tim Murray Changed |= runOnBasicBlock(MBB); 30622c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 30722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 30822c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Changed; 30922c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 31022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 31122c310d78ce9630af15b0de94c18a409705b7496Tim Murraybool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { 31222c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool Changed = false; 31322c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n"); 31422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 31522c310d78ce9630af15b0de94c18a409705b7496Tim Murray // First, scan the basic block producing a set of chains. 31622c310d78ce9630af15b0de94c18a409705b7496Tim Murray 31722c310d78ce9630af15b0de94c18a409705b7496Tim Murray // The currently "active" chains - chains that can be added to and haven't 31822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // been killed yet. This is keyed by register - all chains can only have one 31922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // "link" register between each inst in the chain. 32022c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::map<unsigned, Chain*> ActiveChains; 32150721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines std::set<std::unique_ptr<Chain>> AllChains; 32222c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned Idx = 0; 32322c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto &MI : MBB) 32422c310d78ce9630af15b0de94c18a409705b7496Tim Murray scanInstruction(&MI, Idx++, ActiveChains, AllChains); 32522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 32622c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Scan complete, "<< AllChains.size() << " chains created.\n"); 32722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 32822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Group the chains into disjoint sets based on their liveness range. This is 32922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // a poor-man's version of graph coloring. Ideally we'd create an interference 33022c310d78ce9630af15b0de94c18a409705b7496Tim Murray // graph and perform full-on graph coloring on that, but; 33122c310d78ce9630af15b0de94c18a409705b7496Tim Murray // (a) That's rather heavyweight for only two colors. 33222c310d78ce9630af15b0de94c18a409705b7496Tim Murray // (b) We expect multiple disjoint interference regions - in practice the live 33322c310d78ce9630af15b0de94c18a409705b7496Tim Murray // range of chains is quite small and they are clustered between loads 33422c310d78ce9630af15b0de94c18a409705b7496Tim Murray // and stores. 33522c310d78ce9630af15b0de94c18a409705b7496Tim Murray EquivalenceClasses<Chain*> EC; 33650721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines for (auto &I : AllChains) 33750721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines EC.insert(I.get()); 33822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 33950721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines for (auto &I : AllChains) 34050721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines for (auto &J : AllChains) 34150721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines if (I != J && I->rangeOverlapsWith(*J)) 34250721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines EC.unionSets(I.get(), J.get()); 34322c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n"); 34422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 34522c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Now we assume that every member of an equivalence class interferes 34622c310d78ce9630af15b0de94c18a409705b7496Tim Murray // with every other member of that class, and with no members of other classes. 34722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 34822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Convert the EquivalenceClasses to a simpler set of sets. 34922c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::vector<std::vector<Chain*> > V; 35022c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto I = EC.begin(), E = EC.end(); I != E; ++I) { 35122c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::vector<Chain*> Cs(EC.member_begin(I), EC.member_end()); 35222c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (Cs.empty()) continue; 35322c310d78ce9630af15b0de94c18a409705b7496Tim Murray V.push_back(Cs); 35422c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 35522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 35622c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Now we have a set of sets, order them by start address so 35722c310d78ce9630af15b0de94c18a409705b7496Tim Murray // we can iterate over them sequentially. 35822c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::sort(V.begin(), V.end(), 35922c310d78ce9630af15b0de94c18a409705b7496Tim Murray [](const std::vector<Chain*> &A, 36022c310d78ce9630af15b0de94c18a409705b7496Tim Murray const std::vector<Chain*> &B) { 36122c310d78ce9630af15b0de94c18a409705b7496Tim Murray return A.front()->startsBefore(B.front()); 36222c310d78ce9630af15b0de94c18a409705b7496Tim Murray }); 36322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 36422c310d78ce9630af15b0de94c18a409705b7496Tim Murray // As we only have two colors, we can track the global (BB-level) balance of 36522c310d78ce9630af15b0de94c18a409705b7496Tim Murray // odds versus evens. We aim to keep this near zero to keep both execution 36622c310d78ce9630af15b0de94c18a409705b7496Tim Murray // units fed. 36722c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Positive means we're even-heavy, negative we're odd-heavy. 36822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // 36922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // FIXME: If chains have interdependencies, for example: 37022c310d78ce9630af15b0de94c18a409705b7496Tim Murray // mul r0, r1, r2 37122c310d78ce9630af15b0de94c18a409705b7496Tim Murray // mul r3, r0, r1 37222c310d78ce9630af15b0de94c18a409705b7496Tim Murray // We do not model this and may color each one differently, assuming we'll 37322c310d78ce9630af15b0de94c18a409705b7496Tim Murray // get ILP when we obviously can't. This hasn't been seen to be a problem 37422c310d78ce9630af15b0de94c18a409705b7496Tim Murray // in practice so far, so we simplify the algorithm by ignoring it. 37522c310d78ce9630af15b0de94c18a409705b7496Tim Murray int Parity = 0; 37622c310d78ce9630af15b0de94c18a409705b7496Tim Murray 37722c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto &I : V) 37822c310d78ce9630af15b0de94c18a409705b7496Tim Murray Changed |= colorChainSet(I, MBB, Parity); 37922c310d78ce9630af15b0de94c18a409705b7496Tim Murray 38022c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Changed; 38122c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 38222c310d78ce9630af15b0de94c18a409705b7496Tim Murray 38322c310d78ce9630af15b0de94c18a409705b7496Tim MurrayChain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor, 38422c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::vector<Chain*> &L) { 38522c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (L.empty()) 38622c310d78ce9630af15b0de94c18a409705b7496Tim Murray return nullptr; 38722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 38822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // We try and get the best candidate from L to color next, given that our 38922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // preferred color is "PreferredColor". L is ordered from larger to smaller 39022c310d78ce9630af15b0de94c18a409705b7496Tim Murray // chains. It is beneficial to color the large chains before the small chains, 39122c310d78ce9630af15b0de94c18a409705b7496Tim Murray // but if we can't find a chain of the maximum length with the preferred color, 39222c310d78ce9630af15b0de94c18a409705b7496Tim Murray // we fuzz the size and look for slightly smaller chains before giving up and 39322c310d78ce9630af15b0de94c18a409705b7496Tim Murray // returning a chain that must be recolored. 39422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 39522c310d78ce9630af15b0de94c18a409705b7496Tim Murray // FIXME: Does this need to be configurable? 39622c310d78ce9630af15b0de94c18a409705b7496Tim Murray const unsigned SizeFuzz = 1; 39722c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned MinSize = L.front()->size() - SizeFuzz; 39822c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto I = L.begin(), E = L.end(); I != E; ++I) { 39922c310d78ce9630af15b0de94c18a409705b7496Tim Murray if ((*I)->size() <= MinSize) { 40022c310d78ce9630af15b0de94c18a409705b7496Tim Murray // We've gone past the size limit. Return the previous item. 40122c310d78ce9630af15b0de94c18a409705b7496Tim Murray Chain *Ch = *--I; 40222c310d78ce9630af15b0de94c18a409705b7496Tim Murray L.erase(I); 40322c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Ch; 40422c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 40522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 40622c310d78ce9630af15b0de94c18a409705b7496Tim Murray if ((*I)->getPreferredColor() == PreferredColor) { 40722c310d78ce9630af15b0de94c18a409705b7496Tim Murray Chain *Ch = *I; 40822c310d78ce9630af15b0de94c18a409705b7496Tim Murray L.erase(I); 40922c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Ch; 41022c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 41122c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 41222c310d78ce9630af15b0de94c18a409705b7496Tim Murray 41322c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Bailout case - just return the first item. 41422c310d78ce9630af15b0de94c18a409705b7496Tim Murray Chain *Ch = L.front(); 41522c310d78ce9630af15b0de94c18a409705b7496Tim Murray L.erase(L.begin()); 41622c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Ch; 41722c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 41822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 41922c310d78ce9630af15b0de94c18a409705b7496Tim Murraybool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV, 42022c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineBasicBlock &MBB, 42122c310d78ce9630af15b0de94c18a409705b7496Tim Murray int &Parity) { 42222c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool Changed = false; 42322c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "colorChainSet(): #sets=" << GV.size() << "\n"); 42422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 42522c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Sort by descending size order so that we allocate the most important 42622c310d78ce9630af15b0de94c18a409705b7496Tim Murray // sets first. 42722c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Tie-break equivalent sizes by sorting chains requiring fixups before 42822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // those without fixups. The logic here is that we should look at the 42922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // chains that we cannot change before we look at those we can, 43022c310d78ce9630af15b0de94c18a409705b7496Tim Murray // so the parity counter is updated and we know what color we should 43122c310d78ce9630af15b0de94c18a409705b7496Tim Murray // change them to! 43222c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::sort(GV.begin(), GV.end(), [](const Chain *G1, const Chain *G2) { 43322c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (G1->size() != G2->size()) 43422c310d78ce9630af15b0de94c18a409705b7496Tim Murray return G1->size() > G2->size(); 43522c310d78ce9630af15b0de94c18a409705b7496Tim Murray return G1->requiresFixup() > G2->requiresFixup(); 43622c310d78ce9630af15b0de94c18a409705b7496Tim Murray }); 43722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 43822c310d78ce9630af15b0de94c18a409705b7496Tim Murray Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd; 43922c310d78ce9630af15b0de94c18a409705b7496Tim Murray while (Chain *G = getAndEraseNext(PreferredColor, GV)) { 44022c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Start off by assuming we'll color to our own preferred color. 44122c310d78ce9630af15b0de94c18a409705b7496Tim Murray Color C = PreferredColor; 44222c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (Parity == 0) 44322c310d78ce9630af15b0de94c18a409705b7496Tim Murray // But if we really don't care, use the chain's preferred color. 44422c310d78ce9630af15b0de94c18a409705b7496Tim Murray C = G->getPreferredColor(); 44522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 44622c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << " - Parity=" << Parity << ", Color=" 44722c310d78ce9630af15b0de94c18a409705b7496Tim Murray << ColorNames[(int)C] << "\n"); 44822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 44922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // If we'll need a fixup FMOV, don't bother. Testing has shown that this 45022c310d78ce9630af15b0de94c18a409705b7496Tim Murray // happens infrequently and when it does it has at least a 50% chance of 45122c310d78ce9630af15b0de94c18a409705b7496Tim Murray // slowing code down instead of speeding it up. 45222c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (G->requiresFixup() && C != G->getPreferredColor()) { 45322c310d78ce9630af15b0de94c18a409705b7496Tim Murray C = G->getPreferredColor(); 45422c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << " - " << G->str() << " - not worthwhile changing; " 45522c310d78ce9630af15b0de94c18a409705b7496Tim Murray "color remains " << ColorNames[(int)C] << "\n"); 45622c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 45722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 45822c310d78ce9630af15b0de94c18a409705b7496Tim Murray Changed |= colorChain(G, C, MBB); 45922c310d78ce9630af15b0de94c18a409705b7496Tim Murray 46022c310d78ce9630af15b0de94c18a409705b7496Tim Murray Parity += (C == Color::Even) ? G->size() : -G->size(); 46122c310d78ce9630af15b0de94c18a409705b7496Tim Murray PreferredColor = Parity < 0 ? Color::Even : Color::Odd; 46222c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 46322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 46422c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Changed; 46522c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 46622c310d78ce9630af15b0de94c18a409705b7496Tim Murray 46722c310d78ce9630af15b0de94c18a409705b7496Tim Murrayint AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C, 46822c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineBasicBlock &MBB) { 46922c310d78ce9630af15b0de94c18a409705b7496Tim Murray RegScavenger RS; 47022c310d78ce9630af15b0de94c18a409705b7496Tim Murray RS.enterBasicBlock(&MBB); 47122c310d78ce9630af15b0de94c18a409705b7496Tim Murray RS.forward(MachineBasicBlock::iterator(G->getStart())); 47222c310d78ce9630af15b0de94c18a409705b7496Tim Murray 47322c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Can we find an appropriate register that is available throughout the life 47422c310d78ce9630af15b0de94c18a409705b7496Tim Murray // of the chain? 47522c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned RegClassID = G->getStart()->getDesc().OpInfo[0].RegClass; 47622c310d78ce9630af15b0de94c18a409705b7496Tim Murray BitVector AvailableRegs = RS.getRegsAvailable(TRI->getRegClass(RegClassID)); 47722c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); 47822c310d78ce9630af15b0de94c18a409705b7496Tim Murray I != E; ++I) { 47922c310d78ce9630af15b0de94c18a409705b7496Tim Murray RS.forward(I); 48022c310d78ce9630af15b0de94c18a409705b7496Tim Murray AvailableRegs &= RS.getRegsAvailable(TRI->getRegClass(RegClassID)); 48122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 48222c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Remove any registers clobbered by a regmask. 48322c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto J : I->operands()) { 48422c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (J.isRegMask()) 48522c310d78ce9630af15b0de94c18a409705b7496Tim Murray AvailableRegs.clearBitsNotInMask(J.getRegMask()); 48622c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 48722c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 48822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 48922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Make sure we allocate in-order, to get the cheapest registers first. 49022c310d78ce9630af15b0de94c18a409705b7496Tim Murray auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID)); 49122c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto Reg : Ord) { 49222c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (!AvailableRegs[Reg]) 49322c310d78ce9630af15b0de94c18a409705b7496Tim Murray continue; 49422c310d78ce9630af15b0de94c18a409705b7496Tim Murray if ((C == Color::Even && (Reg % 2) == 0) || 49522c310d78ce9630af15b0de94c18a409705b7496Tim Murray (C == Color::Odd && (Reg % 2) == 1)) 49622c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Reg; 49722c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 49822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 49922c310d78ce9630af15b0de94c18a409705b7496Tim Murray return -1; 50022c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 50122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 50222c310d78ce9630af15b0de94c18a409705b7496Tim Murraybool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C, 50322c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineBasicBlock &MBB) { 50422c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool Changed = false; 50522c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << " - colorChain(" << G->str() << ", " 50622c310d78ce9630af15b0de94c18a409705b7496Tim Murray << ColorNames[(int)C] << ")\n"); 50722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 50822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Try and obtain a free register of the right class. Without a register 50922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // to play with we cannot continue. 51022c310d78ce9630af15b0de94c18a409705b7496Tim Murray int Reg = scavengeRegister(G, C, MBB); 51122c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (Reg == -1) { 51222c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n"); 51322c310d78ce9630af15b0de94c18a409705b7496Tim Murray return false; 51422c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 51522c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << " - Scavenged register: " << TRI->getName(Reg) << "\n"); 51622c310d78ce9630af15b0de94c18a409705b7496Tim Murray 51722c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::map<unsigned, unsigned> Substs; 51822c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); 51922c310d78ce9630af15b0de94c18a409705b7496Tim Murray I != E; ++I) { 52022c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (!G->contains(I) && 52122c310d78ce9630af15b0de94c18a409705b7496Tim Murray (&*I != G->getKill() || G->isKillImmutable())) 52222c310d78ce9630af15b0de94c18a409705b7496Tim Murray continue; 52322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 52422c310d78ce9630af15b0de94c18a409705b7496Tim Murray // I is a member of G, or I is a mutable instruction that kills G. 52522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 52622c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::vector<unsigned> ToErase; 52722c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto &U : I->operands()) { 52822c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) { 52922c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned OrigReg = U.getReg(); 53022c310d78ce9630af15b0de94c18a409705b7496Tim Murray U.setReg(Substs[OrigReg]); 53122c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (U.isKill()) 53222c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Don't erase straight away, because there may be other operands 53322c310d78ce9630af15b0de94c18a409705b7496Tim Murray // that also reference this substitution! 53422c310d78ce9630af15b0de94c18a409705b7496Tim Murray ToErase.push_back(OrigReg); 53522c310d78ce9630af15b0de94c18a409705b7496Tim Murray } else if (U.isRegMask()) { 53622c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto J : Substs) { 53722c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (U.clobbersPhysReg(J.first)) 53822c310d78ce9630af15b0de94c18a409705b7496Tim Murray ToErase.push_back(J.first); 53922c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 54022c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 54122c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 54222c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Now it's safe to remove the substs identified earlier. 54322c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto J : ToErase) 54422c310d78ce9630af15b0de94c18a409705b7496Tim Murray Substs.erase(J); 54522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 54622c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Only change the def if this isn't the last instruction. 54722c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (&*I != G->getKill()) { 54822c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineOperand &MO = I->getOperand(0); 54922c310d78ce9630af15b0de94c18a409705b7496Tim Murray 55022c310d78ce9630af15b0de94c18a409705b7496Tim Murray bool Change = TransformAll || getColor(MO.getReg()) != C; 55122c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (G->requiresFixup() && &*I == G->getLast()) 55222c310d78ce9630af15b0de94c18a409705b7496Tim Murray Change = false; 55322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 55422c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (Change) { 55522c310d78ce9630af15b0de94c18a409705b7496Tim Murray Substs[MO.getReg()] = Reg; 55622c310d78ce9630af15b0de94c18a409705b7496Tim Murray MO.setReg(Reg); 55722c310d78ce9630af15b0de94c18a409705b7496Tim Murray MRI->setPhysRegUsed(Reg); 55822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 55922c310d78ce9630af15b0de94c18a409705b7496Tim Murray Changed = true; 56022c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 56122c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 56222c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 56322c310d78ce9630af15b0de94c18a409705b7496Tim Murray assert(Substs.size() == 0 && "No substitutions should be left active!"); 56422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 56522c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (G->getKill()) { 56622c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << " - Kill instruction seen.\n"); 56722c310d78ce9630af15b0de94c18a409705b7496Tim Murray } else { 56822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // We didn't have a kill instruction, but we didn't seem to need to change 56922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // the destination register anyway. 57022c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << " - Destination register not changed.\n"); 57122c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 57222c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Changed; 57322c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 57422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 57522c310d78ce9630af15b0de94c18a409705b7496Tim Murrayvoid AArch64A57FPLoadBalancing:: 57622c310d78ce9630af15b0de94c18a409705b7496Tim MurrayscanInstruction(MachineInstr *MI, unsigned Idx, 57722c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::map<unsigned, Chain*> &ActiveChains, 57850721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines std::set<std::unique_ptr<Chain>> &AllChains) { 57922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Inspect "MI", updating ActiveChains and AllChains. 58022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 58122c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (isMul(MI)) { 58222c310d78ce9630af15b0de94c18a409705b7496Tim Murray 58350721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines for (auto &I : MI->uses()) 58450721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines maybeKillChain(I, Idx, ActiveChains); 58550721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines for (auto &I : MI->defs()) 58622c310d78ce9630af15b0de94c18a409705b7496Tim Murray maybeKillChain(I, Idx, ActiveChains); 58722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 58822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Create a new chain. Multiplies don't require forwarding so can go on any 58922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // unit. 59022c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned DestReg = MI->getOperand(0).getReg(); 59122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 59222c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "New chain started for register " 59322c310d78ce9630af15b0de94c18a409705b7496Tim Murray << TRI->getName(DestReg) << " at " << *MI); 59422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 59550721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg)); 59650721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines ActiveChains[DestReg] = G.get(); 59750721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines AllChains.insert(std::move(G)); 59822c310d78ce9630af15b0de94c18a409705b7496Tim Murray 59922c310d78ce9630af15b0de94c18a409705b7496Tim Murray } else if (isMla(MI)) { 60022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 60122c310d78ce9630af15b0de94c18a409705b7496Tim Murray // It is beneficial to keep MLAs on the same functional unit as their 60222c310d78ce9630af15b0de94c18a409705b7496Tim Murray // accumulator operand. 60322c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned DestReg = MI->getOperand(0).getReg(); 60422c310d78ce9630af15b0de94c18a409705b7496Tim Murray unsigned AccumReg = MI->getOperand(3).getReg(); 60522c310d78ce9630af15b0de94c18a409705b7496Tim Murray 60622c310d78ce9630af15b0de94c18a409705b7496Tim Murray maybeKillChain(MI->getOperand(1), Idx, ActiveChains); 60722c310d78ce9630af15b0de94c18a409705b7496Tim Murray maybeKillChain(MI->getOperand(2), Idx, ActiveChains); 60822c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (DestReg != AccumReg) 60922c310d78ce9630af15b0de94c18a409705b7496Tim Murray maybeKillChain(MI->getOperand(0), Idx, ActiveChains); 61022c310d78ce9630af15b0de94c18a409705b7496Tim Murray 61122c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (ActiveChains.find(AccumReg) != ActiveChains.end()) { 61222c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Chain found for accumulator register " 61322c310d78ce9630af15b0de94c18a409705b7496Tim Murray << TRI->getName(AccumReg) << " in MI " << *MI); 61422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 61522c310d78ce9630af15b0de94c18a409705b7496Tim Murray // For simplicity we only chain together sequences of MULs/MLAs where the 61622c310d78ce9630af15b0de94c18a409705b7496Tim Murray // accumulator register is killed on each instruction. This means we don't 61722c310d78ce9630af15b0de94c18a409705b7496Tim Murray // need to track other uses of the registers we want to rewrite. 61822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // 61922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // FIXME: We could extend to handle the non-kill cases for more coverage. 62022c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (MI->getOperand(3).isKill()) { 62122c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Add to chain. 62222c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Instruction was successfully added to chain.\n"); 62322c310d78ce9630af15b0de94c18a409705b7496Tim Murray ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg)); 62422c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Handle cases where the destination is not the same as the accumulator. 62550721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines if (DestReg != AccumReg) { 62650721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines ActiveChains[DestReg] = ActiveChains[AccumReg]; 62750721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines ActiveChains.erase(AccumReg); 62850721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines } 62922c310d78ce9630af15b0de94c18a409705b7496Tim Murray return; 63022c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 63122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 63222c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Cannot add to chain because accumulator operand wasn't " 63322c310d78ce9630af15b0de94c18a409705b7496Tim Murray << "marked <kill>!\n"); 63422c310d78ce9630af15b0de94c18a409705b7496Tim Murray maybeKillChain(MI->getOperand(3), Idx, ActiveChains); 63522c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 63622c310d78ce9630af15b0de94c18a409705b7496Tim Murray 63722c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Creating new chain for dest register " 63822c310d78ce9630af15b0de94c18a409705b7496Tim Murray << TRI->getName(DestReg) << "\n"); 63950721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg)); 64050721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines ActiveChains[DestReg] = G.get(); 64150721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines AllChains.insert(std::move(G)); 64222c310d78ce9630af15b0de94c18a409705b7496Tim Murray 64322c310d78ce9630af15b0de94c18a409705b7496Tim Murray } else { 64422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 64522c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs 64622c310d78ce9630af15b0de94c18a409705b7496Tim Murray // lists. 64750721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines for (auto &I : MI->uses()) 64850721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines maybeKillChain(I, Idx, ActiveChains); 64950721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines for (auto &I : MI->defs()) 65022c310d78ce9630af15b0de94c18a409705b7496Tim Murray maybeKillChain(I, Idx, ActiveChains); 65122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 65222c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 65322c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 65422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 65522c310d78ce9630af15b0de94c18a409705b7496Tim Murrayvoid AArch64A57FPLoadBalancing:: 65622c310d78ce9630af15b0de94c18a409705b7496Tim MurraymaybeKillChain(MachineOperand &MO, unsigned Idx, 65722c310d78ce9630af15b0de94c18a409705b7496Tim Murray std::map<unsigned, Chain*> &ActiveChains) { 65822c310d78ce9630af15b0de94c18a409705b7496Tim Murray // Given an operand and the set of active chains (keyed by register), 65922c310d78ce9630af15b0de94c18a409705b7496Tim Murray // determine if a chain should be ended and remove from ActiveChains. 66022c310d78ce9630af15b0de94c18a409705b7496Tim Murray MachineInstr *MI = MO.getParent(); 66122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 66222c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (MO.isReg()) { 66322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 66422c310d78ce9630af15b0de94c18a409705b7496Tim Murray // If this is a KILL of a current chain, record it. 66522c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (MO.isKill() && ActiveChains.find(MO.getReg()) != ActiveChains.end()) { 66622c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Kill seen for chain " << TRI->getName(MO.getReg()) 66722c310d78ce9630af15b0de94c18a409705b7496Tim Murray << "\n"); 66822c310d78ce9630af15b0de94c18a409705b7496Tim Murray ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied()); 66922c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 67022c310d78ce9630af15b0de94c18a409705b7496Tim Murray ActiveChains.erase(MO.getReg()); 67122c310d78ce9630af15b0de94c18a409705b7496Tim Murray 67222c310d78ce9630af15b0de94c18a409705b7496Tim Murray } else if (MO.isRegMask()) { 67322c310d78ce9630af15b0de94c18a409705b7496Tim Murray 67422c310d78ce9630af15b0de94c18a409705b7496Tim Murray for (auto I = ActiveChains.begin(), E = ActiveChains.end(); 67550721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines I != E;) { 67622c310d78ce9630af15b0de94c18a409705b7496Tim Murray if (MO.clobbersPhysReg(I->first)) { 67722c310d78ce9630af15b0de94c18a409705b7496Tim Murray DEBUG(dbgs() << "Kill (regmask) seen for chain " 67822c310d78ce9630af15b0de94c18a409705b7496Tim Murray << TRI->getName(I->first) << "\n"); 67922c310d78ce9630af15b0de94c18a409705b7496Tim Murray I->second->setKill(MI, Idx, /*Immutable=*/true); 68050721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines ActiveChains.erase(I++); 68150721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines } else 68250721a7a8e3c7094a828e29e7697ea52855c6542Stephen Hines ++I; 68322c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 68422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 68522c310d78ce9630af15b0de94c18a409705b7496Tim Murray } 68622c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 68722c310d78ce9630af15b0de94c18a409705b7496Tim Murray 68822c310d78ce9630af15b0de94c18a409705b7496Tim MurrayColor AArch64A57FPLoadBalancing::getColor(unsigned Reg) { 68922c310d78ce9630af15b0de94c18a409705b7496Tim Murray if ((TRI->getEncodingValue(Reg) % 2) == 0) 69022c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Color::Even; 69122c310d78ce9630af15b0de94c18a409705b7496Tim Murray else 69222c310d78ce9630af15b0de94c18a409705b7496Tim Murray return Color::Odd; 69322c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 69422c310d78ce9630af15b0de94c18a409705b7496Tim Murray 69522c310d78ce9630af15b0de94c18a409705b7496Tim Murray// Factory function used by AArch64TargetMachine to add the pass to the passmanager. 69622c310d78ce9630af15b0de94c18a409705b7496Tim MurrayFunctionPass *llvm::createAArch64A57FPLoadBalancing() { 69722c310d78ce9630af15b0de94c18a409705b7496Tim Murray return new AArch64A57FPLoadBalancing(); 69822c310d78ce9630af15b0de94c18a409705b7496Tim Murray} 699