LiveIntervalAnalysis.cpp revision 0ab7103e06ee1da7bde5b196a68be77ab49a005d 
1//===-- LiveIntervalAnalysis.cpp - Live Interval Analysis -----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the LiveInterval analysis pass which is used
11// by the Linear Scan Register allocator. This pass linearizes the
12// basic blocks of the function in DFS order and uses the
13// LiveVariables pass to conservatively compute live intervals for
14// each virtual and physical register.
15//
16//===----------------------------------------------------------------------===//
17
18#define DEBUG_TYPE "regalloc"
19#include "llvm/CodeGen/LiveIntervalAnalysis.h"
20#include "llvm/Value.h"
21#include "llvm/Analysis/AliasAnalysis.h"
22#include "llvm/CodeGen/LiveVariables.h"
23#include "llvm/CodeGen/MachineDominators.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/Passes.h"
27#include "llvm/Target/TargetRegisterInfo.h"
28#include "llvm/Target/TargetInstrInfo.h"
29#include "llvm/Target/TargetMachine.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Debug.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/raw_ostream.h"
34#include "llvm/ADT/DenseSet.h"
35#include "llvm/ADT/STLExtras.h"
36#include "LiveRangeCalc.h"
37#include <algorithm>
38#include <limits>
39#include <cmath>
40using namespace llvm;
41
42// Switch to the new experimental algorithm for computing live intervals.
43static cl::opt<bool>
44NewLiveIntervals("new-live-intervals", cl::Hidden,
45                 cl::desc("Use new algorithm forcomputing live intervals"));
46
47char LiveIntervals::ID = 0;
48INITIALIZE_PASS_BEGIN(LiveIntervals, "liveintervals",
49                "Live Interval Analysis", false, false)
50INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
51INITIALIZE_PASS_DEPENDENCY(LiveVariables)
52INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
53INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
54INITIALIZE_PASS_END(LiveIntervals, "liveintervals",
55                "Live Interval Analysis", false, false)
56
57void LiveIntervals::getAnalysisUsage(AnalysisUsage &AU) const {
58  AU.setPreservesCFG();
59  AU.addRequired<AliasAnalysis>();
60  AU.addPreserved<AliasAnalysis>();
61  AU.addRequired<LiveVariables>();
62  AU.addPreserved<LiveVariables>();
63  AU.addPreservedID(MachineLoopInfoID);
64  AU.addRequiredTransitiveID(MachineDominatorsID);
65  AU.addPreservedID(MachineDominatorsID);
66  AU.addPreserved<SlotIndexes>();
67  AU.addRequiredTransitive<SlotIndexes>();
68  MachineFunctionPass::getAnalysisUsage(AU);
69}
70
71LiveIntervals::LiveIntervals() : MachineFunctionPass(ID),
72  DomTree(0), LRCalc(0) {
73  initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
74}
75
76LiveIntervals::~LiveIntervals() {
77  delete LRCalc;
78}
79
80void LiveIntervals::releaseMemory() {
81  // Free the live intervals themselves.
82  for (unsigned i = 0, e = VirtRegIntervals.size(); i != e; ++i)
83    delete VirtRegIntervals[TargetRegisterInfo::index2VirtReg(i)];
84  VirtRegIntervals.clear();
85  RegMaskSlots.clear();
86  RegMaskBits.clear();
87  RegMaskBlocks.clear();
88
89  for (unsigned i = 0, e = RegUnitIntervals.size(); i != e; ++i)
90    delete RegUnitIntervals[i];
91  RegUnitIntervals.clear();
92
93  // Release VNInfo memory regions, VNInfo objects don't need to be dtor'd.
94  VNInfoAllocator.Reset();
95}
96
97/// runOnMachineFunction - Register allocate the whole function
98///
99bool LiveIntervals::runOnMachineFunction(MachineFunction &fn) {
100  MF = &fn;
101  MRI = &MF->getRegInfo();
102  TM = &fn.getTarget();
103  TRI = TM->getRegisterInfo();
104  TII = TM->getInstrInfo();
105  AA = &getAnalysis<AliasAnalysis>();
106  LV = &getAnalysis<LiveVariables>();
107  Indexes = &getAnalysis<SlotIndexes>();
108  DomTree = &getAnalysis<MachineDominatorTree>();
109  if (!LRCalc)
110    LRCalc = new LiveRangeCalc();
111  AllocatableRegs = TRI->getAllocatableSet(fn);
112  ReservedRegs = TRI->getReservedRegs(fn);
113
114  // Allocate space for all virtual registers.
115  VirtRegIntervals.resize(MRI->getNumVirtRegs());
116
117  if (NewLiveIntervals) {
118    // This is the new way of computing live intervals.
119    // It is independent of LiveVariables, and it can run at any time.
120    computeVirtRegs();
121    computeRegMasks();
122  } else {
123    // This is the old way of computing live intervals.
124    // It depends on LiveVariables.
125    computeIntervals();
126  }
127  computeLiveInRegUnits();
128
129  DEBUG(dump());
130  return true;
131}
132
133/// print - Implement the dump method.
134void LiveIntervals::print(raw_ostream &OS, const Module* ) const {
135  OS << "********** INTERVALS **********\n";
136
137  // Dump the regunits.
138  for (unsigned i = 0, e = RegUnitIntervals.size(); i != e; ++i)
139    if (LiveInterval *LI = RegUnitIntervals[i])
140      OS << PrintRegUnit(i, TRI) << " = " << *LI << '\n';
141
142  // Dump the virtregs.
143  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
144    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
145    if (hasInterval(Reg))
146      OS << PrintReg(Reg) << " = " << getInterval(Reg) << '\n';
147  }
148
149  printInstrs(OS);
150}
151
152void LiveIntervals::printInstrs(raw_ostream &OS) const {
153  OS << "********** MACHINEINSTRS **********\n";
154  MF->print(OS, Indexes);
155}
156
157void LiveIntervals::dumpInstrs() const {
158  printInstrs(dbgs());
159}
160
161static
162bool MultipleDefsBySameMI(const MachineInstr &MI, unsigned MOIdx) {
163  unsigned Reg = MI.getOperand(MOIdx).getReg();
164  for (unsigned i = MOIdx+1, e = MI.getNumOperands(); i < e; ++i) {
165    const MachineOperand &MO = MI.getOperand(i);
166    if (!MO.isReg())
167      continue;
168    if (MO.getReg() == Reg && MO.isDef()) {
169      assert(MI.getOperand(MOIdx).getSubReg() != MO.getSubReg() &&
170             MI.getOperand(MOIdx).getSubReg() &&
171             (MO.getSubReg() || MO.isImplicit()));
172      return true;
173    }
174  }
175  return false;
176}
177
178/// isPartialRedef - Return true if the specified def at the specific index is
179/// partially re-defining the specified live interval. A common case of this is
180/// a definition of the sub-register.
181bool LiveIntervals::isPartialRedef(SlotIndex MIIdx, MachineOperand &MO,
182                                   LiveInterval &interval) {
183  if (!MO.getSubReg() || MO.isEarlyClobber())
184    return false;
185
186  SlotIndex RedefIndex = MIIdx.getRegSlot();
187  const LiveRange *OldLR =
188    interval.getLiveRangeContaining(RedefIndex.getRegSlot(true));
189  MachineInstr *DefMI = getInstructionFromIndex(OldLR->valno->def);
190  if (DefMI != 0) {
191    return DefMI->findRegisterDefOperandIdx(interval.reg) != -1;
192  }
193  return false;
194}
195
196void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
197                                             MachineBasicBlock::iterator mi,
198                                             SlotIndex MIIdx,
199                                             MachineOperand& MO,
200                                             unsigned MOIdx,
201                                             LiveInterval &interval) {
202  DEBUG(dbgs() << "\t\tregister: " << PrintReg(interval.reg, TRI));
203
204  // Virtual registers may be defined multiple times (due to phi
205  // elimination and 2-addr elimination).  Much of what we do only has to be
206  // done once for the vreg.  We use an empty interval to detect the first
207  // time we see a vreg.
208  LiveVariables::VarInfo& vi = LV->getVarInfo(interval.reg);
209  if (interval.empty()) {
210    // Get the Idx of the defining instructions.
211    SlotIndex defIndex = MIIdx.getRegSlot(MO.isEarlyClobber());
212
213    // Make sure the first definition is not a partial redefinition.
214    assert(!MO.readsReg() && "First def cannot also read virtual register "
215           "missing <undef> flag?");
216
217    VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator);
218    assert(ValNo->id == 0 && "First value in interval is not 0?");
219
220    // Loop over all of the blocks that the vreg is defined in.  There are
221    // two cases we have to handle here.  The most common case is a vreg
222    // whose lifetime is contained within a basic block.  In this case there
223    // will be a single kill, in MBB, which comes after the definition.
224    if (vi.Kills.size() == 1 && vi.Kills[0]->getParent() == mbb) {
225      // FIXME: what about dead vars?
226      SlotIndex killIdx;
227      if (vi.Kills[0] != mi)
228        killIdx = getInstructionIndex(vi.Kills[0]).getRegSlot();
229      else
230        killIdx = defIndex.getDeadSlot();
231
232      // If the kill happens after the definition, we have an intra-block
233      // live range.
234      if (killIdx > defIndex) {
235        assert(vi.AliveBlocks.empty() &&
236               "Shouldn't be alive across any blocks!");
237        LiveRange LR(defIndex, killIdx, ValNo);
238        interval.addRange(LR);
239        DEBUG(dbgs() << " +" << LR << "\n");
240        return;
241      }
242    }
243
244    // The other case we handle is when a virtual register lives to the end
245    // of the defining block, potentially live across some blocks, then is
246    // live into some number of blocks, but gets killed.  Start by adding a
247    // range that goes from this definition to the end of the defining block.
248    LiveRange NewLR(defIndex, getMBBEndIdx(mbb), ValNo);
249    DEBUG(dbgs() << " +" << NewLR);
250    interval.addRange(NewLR);
251
252    bool PHIJoin = LV->isPHIJoin(interval.reg);
253
254    if (PHIJoin) {
255      // A phi join register is killed at the end of the MBB and revived as a
256      // new valno in the killing blocks.
257      assert(vi.AliveBlocks.empty() && "Phi join can't pass through blocks");
258      DEBUG(dbgs() << " phi-join");
259      ValNo->setHasPHIKill(true);
260    } else {
261      // Iterate over all of the blocks that the variable is completely
262      // live in, adding [insrtIndex(begin), instrIndex(end)+4) to the
263      // live interval.
264      for (SparseBitVector<>::iterator I = vi.AliveBlocks.begin(),
265               E = vi.AliveBlocks.end(); I != E; ++I) {
266        MachineBasicBlock *aliveBlock = MF->getBlockNumbered(*I);
267        LiveRange LR(getMBBStartIdx(aliveBlock), getMBBEndIdx(aliveBlock),
268                     ValNo);
269        interval.addRange(LR);
270        DEBUG(dbgs() << " +" << LR);
271      }
272    }
273
274    // Finally, this virtual register is live from the start of any killing
275    // block to the 'use' slot of the killing instruction.
276    for (unsigned i = 0, e = vi.Kills.size(); i != e; ++i) {
277      MachineInstr *Kill = vi.Kills[i];
278      SlotIndex Start = getMBBStartIdx(Kill->getParent());
279      SlotIndex killIdx = getInstructionIndex(Kill).getRegSlot();
280
281      // Create interval with one of a NEW value number.  Note that this value
282      // number isn't actually defined by an instruction, weird huh? :)
283      if (PHIJoin) {
284        assert(getInstructionFromIndex(Start) == 0 &&
285               "PHI def index points at actual instruction.");
286        ValNo = interval.getNextValue(Start, VNInfoAllocator);
287      }
288      LiveRange LR(Start, killIdx, ValNo);
289      interval.addRange(LR);
290      DEBUG(dbgs() << " +" << LR);
291    }
292
293  } else {
294    if (MultipleDefsBySameMI(*mi, MOIdx))
295      // Multiple defs of the same virtual register by the same instruction.
296      // e.g. %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ...
297      // This is likely due to elimination of REG_SEQUENCE instructions. Return
298      // here since there is nothing to do.
299      return;
300
301    // If this is the second time we see a virtual register definition, it
302    // must be due to phi elimination or two addr elimination.  If this is
303    // the result of two address elimination, then the vreg is one of the
304    // def-and-use register operand.
305
306    // It may also be partial redef like this:
307    // 80  %reg1041:6<def> = VSHRNv4i16 %reg1034<kill>, 12, pred:14, pred:%reg0
308    // 120 %reg1041:5<def> = VSHRNv4i16 %reg1039<kill>, 12, pred:14, pred:%reg0
309    bool PartReDef = isPartialRedef(MIIdx, MO, interval);
310    if (PartReDef || mi->isRegTiedToUseOperand(MOIdx)) {
311      // If this is a two-address definition, then we have already processed
312      // the live range.  The only problem is that we didn't realize there
313      // are actually two values in the live interval.  Because of this we
314      // need to take the LiveRegion that defines this register and split it
315      // into two values.
316      SlotIndex RedefIndex = MIIdx.getRegSlot(MO.isEarlyClobber());
317
318      const LiveRange *OldLR =
319        interval.getLiveRangeContaining(RedefIndex.getRegSlot(true));
320      VNInfo *OldValNo = OldLR->valno;
321      SlotIndex DefIndex = OldValNo->def.getRegSlot();
322
323      // Delete the previous value, which should be short and continuous,
324      // because the 2-addr copy must be in the same MBB as the redef.
325      interval.removeRange(DefIndex, RedefIndex);
326
327      // The new value number (#1) is defined by the instruction we claimed
328      // defined value #0.
329      VNInfo *ValNo = interval.createValueCopy(OldValNo, VNInfoAllocator);
330
331      // Value#0 is now defined by the 2-addr instruction.
332      OldValNo->def = RedefIndex;
333
334      // Add the new live interval which replaces the range for the input copy.
335      LiveRange LR(DefIndex, RedefIndex, ValNo);
336      DEBUG(dbgs() << " replace range with " << LR);
337      interval.addRange(LR);
338
339      // If this redefinition is dead, we need to add a dummy unit live
340      // range covering the def slot.
341      if (MO.isDead())
342        interval.addRange(LiveRange(RedefIndex, RedefIndex.getDeadSlot(),
343                                    OldValNo));
344
345      DEBUG(dbgs() << " RESULT: " << interval);
346    } else if (LV->isPHIJoin(interval.reg)) {
347      // In the case of PHI elimination, each variable definition is only
348      // live until the end of the block.  We've already taken care of the
349      // rest of the live range.
350
351      SlotIndex defIndex = MIIdx.getRegSlot();
352      if (MO.isEarlyClobber())
353        defIndex = MIIdx.getRegSlot(true);
354
355      VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator);
356
357      SlotIndex killIndex = getMBBEndIdx(mbb);
358      LiveRange LR(defIndex, killIndex, ValNo);
359      interval.addRange(LR);
360      ValNo->setHasPHIKill(true);
361      DEBUG(dbgs() << " phi-join +" << LR);
362    } else {
363      llvm_unreachable("Multiply defined register");
364    }
365  }
366
367  DEBUG(dbgs() << '\n');
368}
369
370void LiveIntervals::handleRegisterDef(MachineBasicBlock *MBB,
371                                      MachineBasicBlock::iterator MI,
372                                      SlotIndex MIIdx,
373                                      MachineOperand& MO,
374                                      unsigned MOIdx) {
375  if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
376    handleVirtualRegisterDef(MBB, MI, MIIdx, MO, MOIdx,
377                             getOrCreateInterval(MO.getReg()));
378}
379
380/// computeIntervals - computes the live intervals for virtual
381/// registers. for some ordering of the machine instructions [1,N] a
382/// live interval is an interval [i, j) where 1 <= i <= j < N for
383/// which a variable is live
384void LiveIntervals::computeIntervals() {
385  DEBUG(dbgs() << "********** COMPUTING LIVE INTERVALS **********\n"
386               << "********** Function: "
387               << ((Value*)MF->getFunction())->getName() << '\n');
388
389  RegMaskBlocks.resize(MF->getNumBlockIDs());
390
391  SmallVector<unsigned, 8> UndefUses;
392  for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end();
393       MBBI != E; ++MBBI) {
394    MachineBasicBlock *MBB = MBBI;
395    RegMaskBlocks[MBB->getNumber()].first = RegMaskSlots.size();
396
397    if (MBB->empty())
398      continue;
399
400    // Track the index of the current machine instr.
401    SlotIndex MIIndex = getMBBStartIdx(MBB);
402    DEBUG(dbgs() << "BB#" << MBB->getNumber()
403          << ":\t\t# derived from " << MBB->getName() << "\n");
404
405    // Skip over empty initial indices.
406    if (getInstructionFromIndex(MIIndex) == 0)
407      MIIndex = Indexes->getNextNonNullIndex(MIIndex);
408
409    for (MachineBasicBlock::iterator MI = MBB->begin(), miEnd = MBB->end();
410         MI != miEnd; ++MI) {
411      DEBUG(dbgs() << MIIndex << "\t" << *MI);
412      if (MI->isDebugValue())
413        continue;
414      assert(Indexes->getInstructionFromIndex(MIIndex) == MI &&
415             "Lost SlotIndex synchronization");
416
417      // Handle defs.
418      for (int i = MI->getNumOperands() - 1; i >= 0; --i) {
419        MachineOperand &MO = MI->getOperand(i);
420
421        // Collect register masks.
422        if (MO.isRegMask()) {
423          RegMaskSlots.push_back(MIIndex.getRegSlot());
424          RegMaskBits.push_back(MO.getRegMask());
425          continue;
426        }
427
428        if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg()))
429          continue;
430
431        // handle register defs - build intervals
432        if (MO.isDef())
433          handleRegisterDef(MBB, MI, MIIndex, MO, i);
434        else if (MO.isUndef())
435          UndefUses.push_back(MO.getReg());
436      }
437
438      // Move to the next instr slot.
439      MIIndex = Indexes->getNextNonNullIndex(MIIndex);
440    }
441
442    // Compute the number of register mask instructions in this block.
443    std::pair<unsigned, unsigned> &RMB = RegMaskBlocks[MBB->getNumber()];
444    RMB.second = RegMaskSlots.size() - RMB.first;;
445  }
446
447  // Create empty intervals for registers defined by implicit_def's (except
448  // for those implicit_def that define values which are liveout of their
449  // blocks.
450  for (unsigned i = 0, e = UndefUses.size(); i != e; ++i) {
451    unsigned UndefReg = UndefUses[i];
452    (void)getOrCreateInterval(UndefReg);
453  }
454}
455
456LiveInterval* LiveIntervals::createInterval(unsigned reg) {
457  float Weight = TargetRegisterInfo::isPhysicalRegister(reg) ? HUGE_VALF : 0.0F;
458  return new LiveInterval(reg, Weight);
459}
460
461
462/// computeVirtRegInterval - Compute the live interval of a virtual register,
463/// based on defs and uses.
464void LiveIntervals::computeVirtRegInterval(LiveInterval *LI) {
465  assert(LRCalc && "LRCalc not initialized.");
466  assert(LI->empty() && "Should only compute empty intervals.");
467  LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
468  LRCalc->createDeadDefs(LI);
469  LRCalc->extendToUses(LI);
470}
471
472void LiveIntervals::computeVirtRegs() {
473  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
474    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
475    if (MRI->reg_nodbg_empty(Reg))
476      continue;
477    LiveInterval *LI = createInterval(Reg);
478    VirtRegIntervals[Reg] = LI;
479    computeVirtRegInterval(LI);
480  }
481}
482
483void LiveIntervals::computeRegMasks() {
484  RegMaskBlocks.resize(MF->getNumBlockIDs());
485
486  // Find all instructions with regmask operands.
487  for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end();
488       MBBI != E; ++MBBI) {
489    MachineBasicBlock *MBB = MBBI;
490    std::pair<unsigned, unsigned> &RMB = RegMaskBlocks[MBB->getNumber()];
491    RMB.first = RegMaskSlots.size();
492    for (MachineBasicBlock::iterator MI = MBB->begin(), ME = MBB->end();
493         MI != ME; ++MI)
494      for (MIOperands MO(MI); MO.isValid(); ++MO) {
495        if (!MO->isRegMask())
496          continue;
497          RegMaskSlots.push_back(Indexes->getInstructionIndex(MI).getRegSlot());
498          RegMaskBits.push_back(MO->getRegMask());
499      }
500    // Compute the number of register mask instructions in this block.
501    RMB.second = RegMaskSlots.size() - RMB.first;;
502  }
503}
504
505//===----------------------------------------------------------------------===//
506//                           Register Unit Liveness
507//===----------------------------------------------------------------------===//
508//
509// Fixed interference typically comes from ABI boundaries: Function arguments
510// and return values are passed in fixed registers, and so are exception
511// pointers entering landing pads. Certain instructions require values to be
512// present in specific registers. That is also represented through fixed
513// interference.
514//
515
516/// computeRegUnitInterval - Compute the live interval of a register unit, based
517/// on the uses and defs of aliasing registers.  The interval should be empty,
518/// or contain only dead phi-defs from ABI blocks.
519void LiveIntervals::computeRegUnitInterval(LiveInterval *LI) {
520  unsigned Unit = LI->reg;
521
522  assert(LRCalc && "LRCalc not initialized.");
523  LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
524
525  // The physregs aliasing Unit are the roots and their super-registers.
526  // Create all values as dead defs before extending to uses. Note that roots
527  // may share super-registers. That's OK because createDeadDefs() is
528  // idempotent. It is very rare for a register unit to have multiple roots, so
529  // uniquing super-registers is probably not worthwhile.
530  for (MCRegUnitRootIterator Roots(Unit, TRI); Roots.isValid(); ++Roots) {
531    unsigned Root = *Roots;
532    if (!MRI->reg_empty(Root))
533      LRCalc->createDeadDefs(LI, Root);
534    for (MCSuperRegIterator Supers(Root, TRI); Supers.isValid(); ++Supers) {
535      if (!MRI->reg_empty(*Supers))
536        LRCalc->createDeadDefs(LI, *Supers);
537    }
538  }
539
540  // Now extend LI to reach all uses.
541  // Ignore uses of reserved registers. We only track defs of those.
542  for (MCRegUnitRootIterator Roots(Unit, TRI); Roots.isValid(); ++Roots) {
543    unsigned Root = *Roots;
544    if (!isReserved(Root) && !MRI->reg_empty(Root))
545      LRCalc->extendToUses(LI, Root);
546    for (MCSuperRegIterator Supers(Root, TRI); Supers.isValid(); ++Supers) {
547      unsigned Reg = *Supers;
548      if (!isReserved(Reg) && !MRI->reg_empty(Reg))
549        LRCalc->extendToUses(LI, Reg);
550    }
551  }
552}
553
554
555/// computeLiveInRegUnits - Precompute the live ranges of any register units
556/// that are live-in to an ABI block somewhere. Register values can appear
557/// without a corresponding def when entering the entry block or a landing pad.
558///
559void LiveIntervals::computeLiveInRegUnits() {
560  RegUnitIntervals.resize(TRI->getNumRegUnits());
561  DEBUG(dbgs() << "Computing live-in reg-units in ABI blocks.\n");
562
563  // Keep track of the intervals allocated.
564  SmallVector<LiveInterval*, 8> NewIntvs;
565
566  // Check all basic blocks for live-ins.
567  for (MachineFunction::const_iterator MFI = MF->begin(), MFE = MF->end();
568       MFI != MFE; ++MFI) {
569    const MachineBasicBlock *MBB = MFI;
570
571    // We only care about ABI blocks: Entry + landing pads.
572    if ((MFI != MF->begin() && !MBB->isLandingPad()) || MBB->livein_empty())
573      continue;
574
575    // Create phi-defs at Begin for all live-in registers.
576    SlotIndex Begin = Indexes->getMBBStartIdx(MBB);
577    DEBUG(dbgs() << Begin << "\tBB#" << MBB->getNumber());
578    for (MachineBasicBlock::livein_iterator LII = MBB->livein_begin(),
579         LIE = MBB->livein_end(); LII != LIE; ++LII) {
580      for (MCRegUnitIterator Units(*LII, TRI); Units.isValid(); ++Units) {
581        unsigned Unit = *Units;
582        LiveInterval *Intv = RegUnitIntervals[Unit];
583        if (!Intv) {
584          Intv = RegUnitIntervals[Unit] = new LiveInterval(Unit, HUGE_VALF);
585          NewIntvs.push_back(Intv);
586        }
587        VNInfo *VNI = Intv->createDeadDef(Begin, getVNInfoAllocator());
588        (void)VNI;
589        DEBUG(dbgs() << ' ' << PrintRegUnit(Unit, TRI) << '#' << VNI->id);
590      }
591    }
592    DEBUG(dbgs() << '\n');
593  }
594  DEBUG(dbgs() << "Created " << NewIntvs.size() << " new intervals.\n");
595
596  // Compute the 'normal' part of the intervals.
597  for (unsigned i = 0, e = NewIntvs.size(); i != e; ++i)
598    computeRegUnitInterval(NewIntvs[i]);
599}
600
601
602/// shrinkToUses - After removing some uses of a register, shrink its live
603/// range to just the remaining uses. This method does not compute reaching
604/// defs for new uses, and it doesn't remove dead defs.
605bool LiveIntervals::shrinkToUses(LiveInterval *li,
606                                 SmallVectorImpl<MachineInstr*> *dead) {
607  DEBUG(dbgs() << "Shrink: " << *li << '\n');
608  assert(TargetRegisterInfo::isVirtualRegister(li->reg)
609         && "Can only shrink virtual registers");
610  // Find all the values used, including PHI kills.
611  SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList;
612
613  // Blocks that have already been added to WorkList as live-out.
614  SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
615
616  // Visit all instructions reading li->reg.
617  for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(li->reg);
618       MachineInstr *UseMI = I.skipInstruction();) {
619    if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg))
620      continue;
621    SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
622    LiveRangeQuery LRQ(*li, Idx);
623    VNInfo *VNI = LRQ.valueIn();
624    if (!VNI) {
625      // This shouldn't happen: readsVirtualRegister returns true, but there is
626      // no live value. It is likely caused by a target getting <undef> flags
627      // wrong.
628      DEBUG(dbgs() << Idx << '\t' << *UseMI
629                   << "Warning: Instr claims to read non-existent value in "
630                    << *li << '\n');
631      continue;
632    }
633    // Special case: An early-clobber tied operand reads and writes the
634    // register one slot early.
635    if (VNInfo *DefVNI = LRQ.valueDefined())
636      Idx = DefVNI->def;
637
638    WorkList.push_back(std::make_pair(Idx, VNI));
639  }
640
641  // Create a new live interval with only minimal live segments per def.
642  LiveInterval NewLI(li->reg, 0);
643  for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
644       I != E; ++I) {
645    VNInfo *VNI = *I;
646    if (VNI->isUnused())
647      continue;
648    NewLI.addRange(LiveRange(VNI->def, VNI->def.getDeadSlot(), VNI));
649  }
650
651  // Keep track of the PHIs that are in use.
652  SmallPtrSet<VNInfo*, 8> UsedPHIs;
653
654  // Extend intervals to reach all uses in WorkList.
655  while (!WorkList.empty()) {
656    SlotIndex Idx = WorkList.back().first;
657    VNInfo *VNI = WorkList.back().second;
658    WorkList.pop_back();
659    const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot());
660    SlotIndex BlockStart = getMBBStartIdx(MBB);
661
662    // Extend the live range for VNI to be live at Idx.
663    if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) {
664      (void)ExtVNI;
665      assert(ExtVNI == VNI && "Unexpected existing value number");
666      // Is this a PHIDef we haven't seen before?
667      if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI))
668        continue;
669      // The PHI is live, make sure the predecessors are live-out.
670      for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
671           PE = MBB->pred_end(); PI != PE; ++PI) {
672        if (!LiveOut.insert(*PI))
673          continue;
674        SlotIndex Stop = getMBBEndIdx(*PI);
675        // A predecessor is not required to have a live-out value for a PHI.
676        if (VNInfo *PVNI = li->getVNInfoBefore(Stop))
677          WorkList.push_back(std::make_pair(Stop, PVNI));
678      }
679      continue;
680    }
681
682    // VNI is live-in to MBB.
683    DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
684    NewLI.addRange(LiveRange(BlockStart, Idx, VNI));
685
686    // Make sure VNI is live-out from the predecessors.
687    for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
688         PE = MBB->pred_end(); PI != PE; ++PI) {
689      if (!LiveOut.insert(*PI))
690        continue;
691      SlotIndex Stop = getMBBEndIdx(*PI);
692      assert(li->getVNInfoBefore(Stop) == VNI &&
693             "Wrong value out of predecessor");
694      WorkList.push_back(std::make_pair(Stop, VNI));
695    }
696  }
697
698  // Handle dead values.
699  bool CanSeparate = false;
700  for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
701       I != E; ++I) {
702    VNInfo *VNI = *I;
703    if (VNI->isUnused())
704      continue;
705    LiveInterval::iterator LII = NewLI.FindLiveRangeContaining(VNI->def);
706    assert(LII != NewLI.end() && "Missing live range for PHI");
707    if (LII->end != VNI->def.getDeadSlot())
708      continue;
709    if (VNI->isPHIDef()) {
710      // This is a dead PHI. Remove it.
711      VNI->setIsUnused(true);
712      NewLI.removeRange(*LII);
713      DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n");
714      CanSeparate = true;
715    } else {
716      // This is a dead def. Make sure the instruction knows.
717      MachineInstr *MI = getInstructionFromIndex(VNI->def);
718      assert(MI && "No instruction defining live value");
719      MI->addRegisterDead(li->reg, TRI);
720      if (dead && MI->allDefsAreDead()) {
721        DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI);
722        dead->push_back(MI);
723      }
724    }
725  }
726
727  // Move the trimmed ranges back.
728  li->ranges.swap(NewLI.ranges);
729  DEBUG(dbgs() << "Shrunk: " << *li << '\n');
730  return CanSeparate;
731}
732
733
734//===----------------------------------------------------------------------===//
735// Register allocator hooks.
736//
737
738void LiveIntervals::addKillFlags() {
739  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
740    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
741    if (MRI->reg_nodbg_empty(Reg))
742      continue;
743    LiveInterval *LI = &getInterval(Reg);
744
745    // Every instruction that kills Reg corresponds to a live range end point.
746    for (LiveInterval::iterator RI = LI->begin(), RE = LI->end(); RI != RE;
747         ++RI) {
748      // A block index indicates an MBB edge.
749      if (RI->end.isBlock())
750        continue;
751      MachineInstr *MI = getInstructionFromIndex(RI->end);
752      if (!MI)
753        continue;
754      MI->addRegisterKilled(Reg, NULL);
755    }
756  }
757}
758
759MachineBasicBlock*
760LiveIntervals::intervalIsInOneMBB(const LiveInterval &LI) const {
761  // A local live range must be fully contained inside the block, meaning it is
762  // defined and killed at instructions, not at block boundaries. It is not
763  // live in or or out of any block.
764  //
765  // It is technically possible to have a PHI-defined live range identical to a
766  // single block, but we are going to return false in that case.
767
768  SlotIndex Start = LI.beginIndex();
769  if (Start.isBlock())
770    return NULL;
771
772  SlotIndex Stop = LI.endIndex();
773  if (Stop.isBlock())
774    return NULL;
775
776  // getMBBFromIndex doesn't need to search the MBB table when both indexes
777  // belong to proper instructions.
778  MachineBasicBlock *MBB1 = Indexes->getMBBFromIndex(Start);
779  MachineBasicBlock *MBB2 = Indexes->getMBBFromIndex(Stop);
780  return MBB1 == MBB2 ? MBB1 : NULL;
781}
782
783bool
784LiveIntervals::hasPHIKill(const LiveInterval &LI, const VNInfo *VNI) const {
785  for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
786       I != E; ++I) {
787    const VNInfo *PHI = *I;
788    if (PHI->isUnused() || !PHI->isPHIDef())
789      continue;
790    const MachineBasicBlock *PHIMBB = getMBBFromIndex(PHI->def);
791    // Conservatively return true instead of scanning huge predecessor lists.
792    if (PHIMBB->pred_size() > 100)
793      return true;
794    for (MachineBasicBlock::const_pred_iterator
795         PI = PHIMBB->pred_begin(), PE = PHIMBB->pred_end(); PI != PE; ++PI)
796      if (VNI == LI.getVNInfoBefore(Indexes->getMBBEndIdx(*PI)))
797        return true;
798  }
799  return false;
800}
801
802float
803LiveIntervals::getSpillWeight(bool isDef, bool isUse, unsigned loopDepth) {
804  // Limit the loop depth ridiculousness.
805  if (loopDepth > 200)
806    loopDepth = 200;
807
808  // The loop depth is used to roughly estimate the number of times the
809  // instruction is executed. Something like 10^d is simple, but will quickly
810  // overflow a float. This expression behaves like 10^d for small d, but is
811  // more tempered for large d. At d=200 we get 6.7e33 which leaves a bit of
812  // headroom before overflow.
813  // By the way, powf() might be unavailable here. For consistency,
814  // We may take pow(double,double).
815  float lc = std::pow(1 + (100.0 / (loopDepth + 10)), (double)loopDepth);
816
817  return (isDef + isUse) * lc;
818}
819
820LiveRange LiveIntervals::addLiveRangeToEndOfBlock(unsigned reg,
821                                                  MachineInstr* startInst) {
822  LiveInterval& Interval = getOrCreateInterval(reg);
823  VNInfo* VN = Interval.getNextValue(
824    SlotIndex(getInstructionIndex(startInst).getRegSlot()),
825    getVNInfoAllocator());
826  VN->setHasPHIKill(true);
827  LiveRange LR(
828     SlotIndex(getInstructionIndex(startInst).getRegSlot()),
829     getMBBEndIdx(startInst->getParent()), VN);
830  Interval.addRange(LR);
831
832  return LR;
833}
834
835
836//===----------------------------------------------------------------------===//
837//                          Register mask functions
838//===----------------------------------------------------------------------===//
839
840bool LiveIntervals::checkRegMaskInterference(LiveInterval &LI,
841                                             BitVector &UsableRegs) {
842  if (LI.empty())
843    return false;
844  LiveInterval::iterator LiveI = LI.begin(), LiveE = LI.end();
845
846  // Use a smaller arrays for local live ranges.
847  ArrayRef<SlotIndex> Slots;
848  ArrayRef<const uint32_t*> Bits;
849  if (MachineBasicBlock *MBB = intervalIsInOneMBB(LI)) {
850    Slots = getRegMaskSlotsInBlock(MBB->getNumber());
851    Bits = getRegMaskBitsInBlock(MBB->getNumber());
852  } else {
853    Slots = getRegMaskSlots();
854    Bits = getRegMaskBits();
855  }
856
857  // We are going to enumerate all the register mask slots contained in LI.
858  // Start with a binary search of RegMaskSlots to find a starting point.
859  ArrayRef<SlotIndex>::iterator SlotI =
860    std::lower_bound(Slots.begin(), Slots.end(), LiveI->start);
861  ArrayRef<SlotIndex>::iterator SlotE = Slots.end();
862
863  // No slots in range, LI begins after the last call.
864  if (SlotI == SlotE)
865    return false;
866
867  bool Found = false;
868  for (;;) {
869    assert(*SlotI >= LiveI->start);
870    // Loop over all slots overlapping this segment.
871    while (*SlotI < LiveI->end) {
872      // *SlotI overlaps LI. Collect mask bits.
873      if (!Found) {
874        // This is the first overlap. Initialize UsableRegs to all ones.
875        UsableRegs.clear();
876        UsableRegs.resize(TRI->getNumRegs(), true);
877        Found = true;
878      }
879      // Remove usable registers clobbered by this mask.
880      UsableRegs.clearBitsNotInMask(Bits[SlotI-Slots.begin()]);
881      if (++SlotI == SlotE)
882        return Found;
883    }
884    // *SlotI is beyond the current LI segment.
885    LiveI = LI.advanceTo(LiveI, *SlotI);
886    if (LiveI == LiveE)
887      return Found;
888    // Advance SlotI until it overlaps.
889    while (*SlotI < LiveI->start)
890      if (++SlotI == SlotE)
891        return Found;
892  }
893}
894
895//===----------------------------------------------------------------------===//
896//                         IntervalUpdate class.
897//===----------------------------------------------------------------------===//
898
899// HMEditor is a toolkit used by handleMove to trim or extend live intervals.
900class LiveIntervals::HMEditor {
901private:
902  LiveIntervals& LIS;
903  const MachineRegisterInfo& MRI;
904  const TargetRegisterInfo& TRI;
905  SlotIndex NewIdx;
906
907  typedef std::pair<LiveInterval*, LiveRange*> IntRangePair;
908  typedef DenseSet<IntRangePair> RangeSet;
909
910  struct RegRanges {
911    LiveRange* Use;
912    LiveRange* EC;
913    LiveRange* Dead;
914    LiveRange* Def;
915    RegRanges() : Use(0), EC(0), Dead(0), Def(0) {}
916  };
917  typedef DenseMap<unsigned, RegRanges> BundleRanges;
918
919public:
920  HMEditor(LiveIntervals& LIS, const MachineRegisterInfo& MRI,
921           const TargetRegisterInfo& TRI, SlotIndex NewIdx)
922    : LIS(LIS), MRI(MRI), TRI(TRI), NewIdx(NewIdx) {}
923
924  // Update intervals for all operands of MI from OldIdx to NewIdx.
925  // This assumes that MI used to be at OldIdx, and now resides at
926  // NewIdx.
927  void moveAllRangesFrom(MachineInstr* MI, SlotIndex OldIdx) {
928    assert(NewIdx != OldIdx && "No-op move? That's a bit strange.");
929
930    // Collect the operands.
931    RangeSet Entering, Internal, Exiting;
932    bool hasRegMaskOp = false;
933    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
934
935    // To keep the LiveRanges valid within an interval, move the ranges closest
936    // to the destination first. This prevents ranges from overlapping, to that
937    // APIs like removeRange still work.
938    if (NewIdx < OldIdx) {
939      moveAllEnteringFrom(OldIdx, Entering);
940      moveAllInternalFrom(OldIdx, Internal);
941      moveAllExitingFrom(OldIdx, Exiting);
942    }
943    else {
944      moveAllExitingFrom(OldIdx, Exiting);
945      moveAllInternalFrom(OldIdx, Internal);
946      moveAllEnteringFrom(OldIdx, Entering);
947    }
948
949    if (hasRegMaskOp)
950      updateRegMaskSlots(OldIdx);
951
952#ifndef NDEBUG
953    LIValidator validator;
954    validator = std::for_each(Entering.begin(), Entering.end(), validator);
955    validator = std::for_each(Internal.begin(), Internal.end(), validator);
956    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
957    assert(validator.rangesOk() && "moveAllOperandsFrom broke liveness.");
958#endif
959
960  }
961
962  // Update intervals for all operands of MI to refer to BundleStart's
963  // SlotIndex.
964  void moveAllRangesInto(MachineInstr* MI, MachineInstr* BundleStart) {
965    if (MI == BundleStart)
966      return; // Bundling instr with itself - nothing to do.
967
968    SlotIndex OldIdx = LIS.getSlotIndexes()->getInstructionIndex(MI);
969    assert(LIS.getSlotIndexes()->getInstructionFromIndex(OldIdx) == MI &&
970           "SlotIndex <-> Instruction mapping broken for MI");
971
972    // Collect all ranges already in the bundle.
973    MachineBasicBlock::instr_iterator BII(BundleStart);
974    RangeSet Entering, Internal, Exiting;
975    bool hasRegMaskOp = false;
976    collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
977    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
978    for (++BII; &*BII == MI || BII->isInsideBundle(); ++BII) {
979      if (&*BII == MI)
980        continue;
981      collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
982      assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
983    }
984
985    BundleRanges BR = createBundleRanges(Entering, Internal, Exiting);
986
987    Entering.clear();
988    Internal.clear();
989    Exiting.clear();
990    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
991    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
992
993    DEBUG(dbgs() << "Entering: " << Entering.size() << "\n");
994    DEBUG(dbgs() << "Internal: " << Internal.size() << "\n");
995    DEBUG(dbgs() << "Exiting: " << Exiting.size() << "\n");
996
997    moveAllEnteringFromInto(OldIdx, Entering, BR);
998    moveAllInternalFromInto(OldIdx, Internal, BR);
999    moveAllExitingFromInto(OldIdx, Exiting, BR);
1000
1001
1002#ifndef NDEBUG
1003    LIValidator validator;
1004    validator = std::for_each(Entering.begin(), Entering.end(), validator);
1005    validator = std::for_each(Internal.begin(), Internal.end(), validator);
1006    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
1007    assert(validator.rangesOk() && "moveAllOperandsInto broke liveness.");
1008#endif
1009  }
1010
1011private:
1012
1013#ifndef NDEBUG
1014  class LIValidator {
1015  private:
1016    DenseSet<const LiveInterval*> Checked, Bogus;
1017  public:
1018    void operator()(const IntRangePair& P) {
1019      const LiveInterval* LI = P.first;
1020      if (Checked.count(LI))
1021        return;
1022      Checked.insert(LI);
1023      if (LI->empty())
1024        return;
1025      SlotIndex LastEnd = LI->begin()->start;
1026      for (LiveInterval::const_iterator LRI = LI->begin(), LRE = LI->end();
1027           LRI != LRE; ++LRI) {
1028        const LiveRange& LR = *LRI;
1029        if (LastEnd > LR.start || LR.start >= LR.end)
1030          Bogus.insert(LI);
1031        LastEnd = LR.end;
1032      }
1033    }
1034
1035    bool rangesOk() const {
1036      return Bogus.empty();
1037    }
1038  };
1039#endif
1040
1041  // Collect IntRangePairs for all operands of MI that may need fixing.
1042  // Treat's MI's index as OldIdx (regardless of what it is in SlotIndexes'
1043  // maps).
1044  void collectRanges(MachineInstr* MI, RangeSet& Entering, RangeSet& Internal,
1045                     RangeSet& Exiting, bool& hasRegMaskOp, SlotIndex OldIdx) {
1046    hasRegMaskOp = false;
1047    for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
1048                                    MOE = MI->operands_end();
1049         MOI != MOE; ++MOI) {
1050      const MachineOperand& MO = *MOI;
1051
1052      if (MO.isRegMask()) {
1053        hasRegMaskOp = true;
1054        continue;
1055      }
1056
1057      if (!MO.isReg() || MO.getReg() == 0)
1058        continue;
1059
1060      unsigned Reg = MO.getReg();
1061
1062      // TODO: Currently we're skipping uses that are reserved or have no
1063      // interval, but we're not updating their kills. This should be
1064      // fixed.
1065      if (TargetRegisterInfo::isPhysicalRegister(Reg) && LIS.isReserved(Reg))
1066        continue;
1067
1068      // Collect ranges for register units. These live ranges are computed on
1069      // demand, so just skip any that haven't been computed yet.
1070      if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
1071        for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
1072          if (LiveInterval *LI = LIS.getCachedRegUnit(*Units))
1073            collectRanges(MO, LI, Entering, Internal, Exiting, OldIdx);
1074      } else {
1075        // Collect ranges for individual virtual registers.
1076        collectRanges(MO, &LIS.getInterval(Reg),
1077                      Entering, Internal, Exiting, OldIdx);
1078      }
1079    }
1080  }
1081
1082  void collectRanges(const MachineOperand &MO, LiveInterval *LI,
1083                     RangeSet &Entering, RangeSet &Internal, RangeSet &Exiting,
1084                     SlotIndex OldIdx) {
1085    if (MO.readsReg()) {
1086      LiveRange* LR = LI->getLiveRangeContaining(OldIdx);
1087      if (LR != 0)
1088        Entering.insert(std::make_pair(LI, LR));
1089    }
1090    if (MO.isDef()) {
1091      LiveRange* LR = LI->getLiveRangeContaining(OldIdx.getRegSlot());
1092      assert(LR != 0 && "No live range for def?");
1093      if (LR->end > OldIdx.getDeadSlot())
1094        Exiting.insert(std::make_pair(LI, LR));
1095      else
1096        Internal.insert(std::make_pair(LI, LR));
1097    }
1098  }
1099
1100  BundleRanges createBundleRanges(RangeSet& Entering,
1101                                  RangeSet& Internal,
1102                                  RangeSet& Exiting) {
1103    BundleRanges BR;
1104
1105    for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1106         EI != EE; ++EI) {
1107      LiveInterval* LI = EI->first;
1108      LiveRange* LR = EI->second;
1109      BR[LI->reg].Use = LR;
1110    }
1111
1112    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1113         II != IE; ++II) {
1114      LiveInterval* LI = II->first;
1115      LiveRange* LR = II->second;
1116      if (LR->end.isDead()) {
1117        BR[LI->reg].Dead = LR;
1118      } else {
1119        BR[LI->reg].EC = LR;
1120      }
1121    }
1122
1123    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1124         EI != EE; ++EI) {
1125      LiveInterval* LI = EI->first;
1126      LiveRange* LR = EI->second;
1127      BR[LI->reg].Def = LR;
1128    }
1129
1130    return BR;
1131  }
1132
1133  void moveKillFlags(unsigned reg, SlotIndex OldIdx, SlotIndex newKillIdx) {
1134    MachineInstr* OldKillMI = LIS.getInstructionFromIndex(OldIdx);
1135    if (!OldKillMI->killsRegister(reg))
1136      return; // Bail out if we don't have kill flags on the old register.
1137    MachineInstr* NewKillMI = LIS.getInstructionFromIndex(newKillIdx);
1138    assert(OldKillMI->killsRegister(reg) && "Old 'kill' instr isn't a kill.");
1139    assert(!NewKillMI->killsRegister(reg) &&
1140           "New kill instr is already a kill.");
1141    OldKillMI->clearRegisterKills(reg, &TRI);
1142    NewKillMI->addRegisterKilled(reg, &TRI);
1143  }
1144
1145  void updateRegMaskSlots(SlotIndex OldIdx) {
1146    SmallVectorImpl<SlotIndex>::iterator RI =
1147      std::lower_bound(LIS.RegMaskSlots.begin(), LIS.RegMaskSlots.end(),
1148                       OldIdx);
1149    assert(*RI == OldIdx && "No RegMask at OldIdx.");
1150    *RI = NewIdx;
1151    assert(*prior(RI) < *RI && *RI < *next(RI) &&
1152           "RegSlots out of order. Did you move one call across another?");
1153  }
1154
1155  // Return the last use of reg between NewIdx and OldIdx.
1156  SlotIndex findLastUseBefore(unsigned Reg, SlotIndex OldIdx) {
1157    SlotIndex LastUse = NewIdx;
1158    for (MachineRegisterInfo::use_nodbg_iterator
1159           UI = MRI.use_nodbg_begin(Reg),
1160           UE = MRI.use_nodbg_end();
1161         UI != UE; UI.skipInstruction()) {
1162      const MachineInstr* MI = &*UI;
1163      SlotIndex InstSlot = LIS.getSlotIndexes()->getInstructionIndex(MI);
1164      if (InstSlot > LastUse && InstSlot < OldIdx)
1165        LastUse = InstSlot;
1166    }
1167    return LastUse;
1168  }
1169
1170  void moveEnteringUpFrom(SlotIndex OldIdx, IntRangePair& P) {
1171    LiveInterval* LI = P.first;
1172    LiveRange* LR = P.second;
1173    bool LiveThrough = LR->end > OldIdx.getRegSlot();
1174    if (LiveThrough)
1175      return;
1176    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
1177    if (LastUse != NewIdx)
1178      moveKillFlags(LI->reg, NewIdx, LastUse);
1179    LR->end = LastUse.getRegSlot();
1180  }
1181
1182  void moveEnteringDownFrom(SlotIndex OldIdx, IntRangePair& P) {
1183    LiveInterval* LI = P.first;
1184    LiveRange* LR = P.second;
1185    // Extend the LiveRange if NewIdx is past the end.
1186    if (NewIdx > LR->end) {
1187      // Move kill flags if OldIdx was not originally the end
1188      // (otherwise LR->end points to an invalid slot).
1189      if (LR->end.getRegSlot() != OldIdx.getRegSlot()) {
1190        assert(LR->end > OldIdx && "LiveRange does not cover original slot");
1191        moveKillFlags(LI->reg, LR->end, NewIdx);
1192      }
1193      LR->end = NewIdx.getRegSlot();
1194    }
1195  }
1196
1197  void moveAllEnteringFrom(SlotIndex OldIdx, RangeSet& Entering) {
1198    bool GoingUp = NewIdx < OldIdx;
1199
1200    if (GoingUp) {
1201      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1202           EI != EE; ++EI)
1203        moveEnteringUpFrom(OldIdx, *EI);
1204    } else {
1205      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1206           EI != EE; ++EI)
1207        moveEnteringDownFrom(OldIdx, *EI);
1208    }
1209  }
1210
1211  void moveInternalFrom(SlotIndex OldIdx, IntRangePair& P) {
1212    LiveInterval* LI = P.first;
1213    LiveRange* LR = P.second;
1214    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
1215           LR->end <= OldIdx.getDeadSlot() &&
1216           "Range should be internal to OldIdx.");
1217    LiveRange Tmp(*LR);
1218    Tmp.start = NewIdx.getRegSlot(LR->start.isEarlyClobber());
1219    Tmp.valno->def = Tmp.start;
1220    Tmp.end = LR->end.isDead() ? NewIdx.getDeadSlot() : NewIdx.getRegSlot();
1221    LI->removeRange(*LR);
1222    LI->addRange(Tmp);
1223  }
1224
1225  void moveAllInternalFrom(SlotIndex OldIdx, RangeSet& Internal) {
1226    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1227         II != IE; ++II)
1228      moveInternalFrom(OldIdx, *II);
1229  }
1230
1231  void moveExitingFrom(SlotIndex OldIdx, IntRangePair& P) {
1232    LiveRange* LR = P.second;
1233    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
1234           "Range should start in OldIdx.");
1235    assert(LR->end > OldIdx.getDeadSlot() && "Range should exit OldIdx.");
1236    SlotIndex NewStart = NewIdx.getRegSlot(LR->start.isEarlyClobber());
1237    LR->start = NewStart;
1238    LR->valno->def = NewStart;
1239  }
1240
1241  void moveAllExitingFrom(SlotIndex OldIdx, RangeSet& Exiting) {
1242    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1243         EI != EE; ++EI)
1244      moveExitingFrom(OldIdx, *EI);
1245  }
1246
1247  void moveEnteringUpFromInto(SlotIndex OldIdx, IntRangePair& P,
1248                              BundleRanges& BR) {
1249    LiveInterval* LI = P.first;
1250    LiveRange* LR = P.second;
1251    bool LiveThrough = LR->end > OldIdx.getRegSlot();
1252    if (LiveThrough) {
1253      assert((LR->start < NewIdx || BR[LI->reg].Def == LR) &&
1254             "Def in bundle should be def range.");
1255      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
1256             "If bundle has use for this reg it should be LR.");
1257      BR[LI->reg].Use = LR;
1258      return;
1259    }
1260
1261    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
1262    moveKillFlags(LI->reg, OldIdx, LastUse);
1263
1264    if (LR->start < NewIdx) {
1265      // Becoming a new entering range.
1266      assert(BR[LI->reg].Dead == 0 && BR[LI->reg].Def == 0 &&
1267             "Bundle shouldn't be re-defining reg mid-range.");
1268      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
1269             "Bundle shouldn't have different use range for same reg.");
1270      LR->end = LastUse.getRegSlot();
1271      BR[LI->reg].Use = LR;
1272    } else {
1273      // Becoming a new Dead-def.
1274      assert(LR->start == NewIdx.getRegSlot(LR->start.isEarlyClobber()) &&
1275             "Live range starting at unexpected slot.");
1276      assert(BR[LI->reg].Def == LR && "Reg should have def range.");
1277      assert(BR[LI->reg].Dead == 0 &&
1278               "Can't have def and dead def of same reg in a bundle.");
1279      LR->end = LastUse.getDeadSlot();
1280      BR[LI->reg].Dead = BR[LI->reg].Def;
1281      BR[LI->reg].Def = 0;
1282    }
1283  }
1284
1285  void moveEnteringDownFromInto(SlotIndex OldIdx, IntRangePair& P,
1286                                BundleRanges& BR) {
1287    LiveInterval* LI = P.first;
1288    LiveRange* LR = P.second;
1289    if (NewIdx > LR->end) {
1290      // Range extended to bundle. Add to bundle uses.
1291      // Note: Currently adds kill flags to bundle start.
1292      assert(BR[LI->reg].Use == 0 &&
1293             "Bundle already has use range for reg.");
1294      moveKillFlags(LI->reg, LR->end, NewIdx);
1295      LR->end = NewIdx.getRegSlot();
1296      BR[LI->reg].Use = LR;
1297    } else {
1298      assert(BR[LI->reg].Use != 0 &&
1299             "Bundle should already have a use range for reg.");
1300    }
1301  }
1302
1303  void moveAllEnteringFromInto(SlotIndex OldIdx, RangeSet& Entering,
1304                               BundleRanges& BR) {
1305    bool GoingUp = NewIdx < OldIdx;
1306
1307    if (GoingUp) {
1308      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1309           EI != EE; ++EI)
1310        moveEnteringUpFromInto(OldIdx, *EI, BR);
1311    } else {
1312      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1313           EI != EE; ++EI)
1314        moveEnteringDownFromInto(OldIdx, *EI, BR);
1315    }
1316  }
1317
1318  void moveInternalFromInto(SlotIndex OldIdx, IntRangePair& P,
1319                            BundleRanges& BR) {
1320    // TODO: Sane rules for moving ranges into bundles.
1321  }
1322
1323  void moveAllInternalFromInto(SlotIndex OldIdx, RangeSet& Internal,
1324                               BundleRanges& BR) {
1325    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1326         II != IE; ++II)
1327      moveInternalFromInto(OldIdx, *II, BR);
1328  }
1329
1330  void moveExitingFromInto(SlotIndex OldIdx, IntRangePair& P,
1331                           BundleRanges& BR) {
1332    LiveInterval* LI = P.first;
1333    LiveRange* LR = P.second;
1334
1335    assert(LR->start.isRegister() &&
1336           "Don't know how to merge exiting ECs into bundles yet.");
1337
1338    if (LR->end > NewIdx.getDeadSlot()) {
1339      // This range is becoming an exiting range on the bundle.
1340      // If there was an old dead-def of this reg, delete it.
1341      if (BR[LI->reg].Dead != 0) {
1342        LI->removeRange(*BR[LI->reg].Dead);
1343        BR[LI->reg].Dead = 0;
1344      }
1345      assert(BR[LI->reg].Def == 0 &&
1346             "Can't have two defs for the same variable exiting a bundle.");
1347      LR->start = NewIdx.getRegSlot();
1348      LR->valno->def = LR->start;
1349      BR[LI->reg].Def = LR;
1350    } else {
1351      // This range is becoming internal to the bundle.
1352      assert(LR->end == NewIdx.getRegSlot() &&
1353             "Can't bundle def whose kill is before the bundle");
1354      if (BR[LI->reg].Dead || BR[LI->reg].Def) {
1355        // Already have a def for this. Just delete range.
1356        LI->removeRange(*LR);
1357      } else {
1358        // Make range dead, record.
1359        LR->end = NewIdx.getDeadSlot();
1360        BR[LI->reg].Dead = LR;
1361        assert(BR[LI->reg].Use == LR &&
1362               "Range becoming dead should currently be use.");
1363      }
1364      // In both cases the range is no longer a use on the bundle.
1365      BR[LI->reg].Use = 0;
1366    }
1367  }
1368
1369  void moveAllExitingFromInto(SlotIndex OldIdx, RangeSet& Exiting,
1370                              BundleRanges& BR) {
1371    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1372         EI != EE; ++EI)
1373      moveExitingFromInto(OldIdx, *EI, BR);
1374  }
1375
1376};
1377
1378void LiveIntervals::handleMove(MachineInstr* MI) {
1379  SlotIndex OldIndex = Indexes->getInstructionIndex(MI);
1380  Indexes->removeMachineInstrFromMaps(MI);
1381  SlotIndex NewIndex = MI->isInsideBundle() ?
1382                        Indexes->getInstructionIndex(MI) :
1383                        Indexes->insertMachineInstrInMaps(MI);
1384  assert(getMBBStartIdx(MI->getParent()) <= OldIndex &&
1385         OldIndex < getMBBEndIdx(MI->getParent()) &&
1386         "Cannot handle moves across basic block boundaries.");
1387  assert(!MI->isBundled() && "Can't handle bundled instructions yet.");
1388
1389  HMEditor HME(*this, *MRI, *TRI, NewIndex);
1390  HME.moveAllRangesFrom(MI, OldIndex);
1391}
1392
1393void LiveIntervals::handleMoveIntoBundle(MachineInstr* MI,
1394                                         MachineInstr* BundleStart) {
1395  SlotIndex NewIndex = Indexes->getInstructionIndex(BundleStart);
1396  HMEditor HME(*this, *MRI, *TRI, NewIndex);
1397  HME.moveAllRangesInto(MI, BundleStart);
1398}
1399