LiveIntervalAnalysis.cpp revision b18d779b35909cd5b753871f8bf2ff4f6c17ace1 
1//===-- LiveIntervalAnalysis.cpp - Live Interval Analysis -----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the LiveInterval analysis pass which is used
11// by the Linear Scan Register allocator. This pass linearizes the
12// basic blocks of the function in DFS order and uses the
13// LiveVariables pass to conservatively compute live intervals for
14// each virtual and physical register.
15//
16//===----------------------------------------------------------------------===//
17
18#define DEBUG_TYPE "regalloc"
19#include "llvm/CodeGen/LiveIntervalAnalysis.h"
20#include "llvm/Value.h"
21#include "llvm/Analysis/AliasAnalysis.h"
22#include "llvm/CodeGen/LiveVariables.h"
23#include "llvm/CodeGen/MachineDominators.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/Passes.h"
27#include "llvm/Target/TargetRegisterInfo.h"
28#include "llvm/Target/TargetInstrInfo.h"
29#include "llvm/Target/TargetMachine.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Debug.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/raw_ostream.h"
34#include "llvm/ADT/DenseSet.h"
35#include "llvm/ADT/STLExtras.h"
36#include "LiveRangeCalc.h"
37#include <algorithm>
38#include <limits>
39#include <cmath>
40using namespace llvm;
41
42// Switch to the new experimental algorithm for computing live intervals.
43static cl::opt<bool>
44NewLiveIntervals("new-live-intervals", cl::Hidden,
45                 cl::desc("Use new algorithm forcomputing live intervals"));
46
47char LiveIntervals::ID = 0;
48INITIALIZE_PASS_BEGIN(LiveIntervals, "liveintervals",
49                "Live Interval Analysis", false, false)
50INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
51INITIALIZE_PASS_DEPENDENCY(LiveVariables)
52INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
53INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
54INITIALIZE_PASS_END(LiveIntervals, "liveintervals",
55                "Live Interval Analysis", false, false)
56
57void LiveIntervals::getAnalysisUsage(AnalysisUsage &AU) const {
58  AU.setPreservesCFG();
59  AU.addRequired<AliasAnalysis>();
60  AU.addPreserved<AliasAnalysis>();
61  AU.addRequired<LiveVariables>();
62  AU.addPreserved<LiveVariables>();
63  AU.addPreservedID(MachineLoopInfoID);
64  AU.addRequiredTransitiveID(MachineDominatorsID);
65  AU.addPreservedID(MachineDominatorsID);
66  AU.addPreserved<SlotIndexes>();
67  AU.addRequiredTransitive<SlotIndexes>();
68  MachineFunctionPass::getAnalysisUsage(AU);
69}
70
71LiveIntervals::LiveIntervals() : MachineFunctionPass(ID),
72  DomTree(0), LRCalc(0) {
73  initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
74}
75
76LiveIntervals::~LiveIntervals() {
77  delete LRCalc;
78}
79
80void LiveIntervals::releaseMemory() {
81  // Free the live intervals themselves.
82  for (unsigned i = 0, e = VirtRegIntervals.size(); i != e; ++i)
83    delete VirtRegIntervals[TargetRegisterInfo::index2VirtReg(i)];
84  VirtRegIntervals.clear();
85  RegMaskSlots.clear();
86  RegMaskBits.clear();
87  RegMaskBlocks.clear();
88
89  for (unsigned i = 0, e = RegUnitIntervals.size(); i != e; ++i)
90    delete RegUnitIntervals[i];
91  RegUnitIntervals.clear();
92
93  // Release VNInfo memory regions, VNInfo objects don't need to be dtor'd.
94  VNInfoAllocator.Reset();
95}
96
97/// runOnMachineFunction - Register allocate the whole function
98///
99bool LiveIntervals::runOnMachineFunction(MachineFunction &fn) {
100  MF = &fn;
101  MRI = &MF->getRegInfo();
102  TM = &fn.getTarget();
103  TRI = TM->getRegisterInfo();
104  TII = TM->getInstrInfo();
105  AA = &getAnalysis<AliasAnalysis>();
106  LV = &getAnalysis<LiveVariables>();
107  Indexes = &getAnalysis<SlotIndexes>();
108  DomTree = &getAnalysis<MachineDominatorTree>();
109  if (!LRCalc)
110    LRCalc = new LiveRangeCalc();
111  AllocatableRegs = TRI->getAllocatableSet(fn);
112  ReservedRegs = TRI->getReservedRegs(fn);
113
114  // Allocate space for all virtual registers.
115  VirtRegIntervals.resize(MRI->getNumVirtRegs());
116
117  if (NewLiveIntervals) {
118    // This is the new way of computing live intervals.
119    // It is independent of LiveVariables, and it can run at any time.
120    for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
121      unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
122      if (MRI->reg_nodbg_empty(Reg))
123        continue;
124      LiveInterval *LI = createInterval(Reg);
125      VirtRegIntervals[Reg] = LI;
126      computeVirtRegInterval(LI);
127    }
128  } else {
129    // This is the old way of computing live intervals.
130    // It depends on LiveVariables.
131    computeIntervals();
132  }
133  computeLiveInRegUnits();
134
135  DEBUG(dump());
136  return true;
137}
138
139/// print - Implement the dump method.
140void LiveIntervals::print(raw_ostream &OS, const Module* ) const {
141  OS << "********** INTERVALS **********\n";
142
143  // Dump the regunits.
144  for (unsigned i = 0, e = RegUnitIntervals.size(); i != e; ++i)
145    if (LiveInterval *LI = RegUnitIntervals[i])
146      OS << PrintRegUnit(i, TRI) << " = " << *LI << '\n';
147
148  // Dump the virtregs.
149  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
150    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
151    if (hasInterval(Reg))
152      OS << PrintReg(Reg) << " = " << getInterval(Reg) << '\n';
153  }
154
155  printInstrs(OS);
156}
157
158void LiveIntervals::printInstrs(raw_ostream &OS) const {
159  OS << "********** MACHINEINSTRS **********\n";
160  MF->print(OS, Indexes);
161}
162
163void LiveIntervals::dumpInstrs() const {
164  printInstrs(dbgs());
165}
166
167static
168bool MultipleDefsBySameMI(const MachineInstr &MI, unsigned MOIdx) {
169  unsigned Reg = MI.getOperand(MOIdx).getReg();
170  for (unsigned i = MOIdx+1, e = MI.getNumOperands(); i < e; ++i) {
171    const MachineOperand &MO = MI.getOperand(i);
172    if (!MO.isReg())
173      continue;
174    if (MO.getReg() == Reg && MO.isDef()) {
175      assert(MI.getOperand(MOIdx).getSubReg() != MO.getSubReg() &&
176             MI.getOperand(MOIdx).getSubReg() &&
177             (MO.getSubReg() || MO.isImplicit()));
178      return true;
179    }
180  }
181  return false;
182}
183
184/// isPartialRedef - Return true if the specified def at the specific index is
185/// partially re-defining the specified live interval. A common case of this is
186/// a definition of the sub-register.
187bool LiveIntervals::isPartialRedef(SlotIndex MIIdx, MachineOperand &MO,
188                                   LiveInterval &interval) {
189  if (!MO.getSubReg() || MO.isEarlyClobber())
190    return false;
191
192  SlotIndex RedefIndex = MIIdx.getRegSlot();
193  const LiveRange *OldLR =
194    interval.getLiveRangeContaining(RedefIndex.getRegSlot(true));
195  MachineInstr *DefMI = getInstructionFromIndex(OldLR->valno->def);
196  if (DefMI != 0) {
197    return DefMI->findRegisterDefOperandIdx(interval.reg) != -1;
198  }
199  return false;
200}
201
202void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
203                                             MachineBasicBlock::iterator mi,
204                                             SlotIndex MIIdx,
205                                             MachineOperand& MO,
206                                             unsigned MOIdx,
207                                             LiveInterval &interval) {
208  DEBUG(dbgs() << "\t\tregister: " << PrintReg(interval.reg, TRI));
209
210  // Virtual registers may be defined multiple times (due to phi
211  // elimination and 2-addr elimination).  Much of what we do only has to be
212  // done once for the vreg.  We use an empty interval to detect the first
213  // time we see a vreg.
214  LiveVariables::VarInfo& vi = LV->getVarInfo(interval.reg);
215  if (interval.empty()) {
216    // Get the Idx of the defining instructions.
217    SlotIndex defIndex = MIIdx.getRegSlot(MO.isEarlyClobber());
218
219    // Make sure the first definition is not a partial redefinition.
220    assert(!MO.readsReg() && "First def cannot also read virtual register "
221           "missing <undef> flag?");
222
223    VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator);
224    assert(ValNo->id == 0 && "First value in interval is not 0?");
225
226    // Loop over all of the blocks that the vreg is defined in.  There are
227    // two cases we have to handle here.  The most common case is a vreg
228    // whose lifetime is contained within a basic block.  In this case there
229    // will be a single kill, in MBB, which comes after the definition.
230    if (vi.Kills.size() == 1 && vi.Kills[0]->getParent() == mbb) {
231      // FIXME: what about dead vars?
232      SlotIndex killIdx;
233      if (vi.Kills[0] != mi)
234        killIdx = getInstructionIndex(vi.Kills[0]).getRegSlot();
235      else
236        killIdx = defIndex.getDeadSlot();
237
238      // If the kill happens after the definition, we have an intra-block
239      // live range.
240      if (killIdx > defIndex) {
241        assert(vi.AliveBlocks.empty() &&
242               "Shouldn't be alive across any blocks!");
243        LiveRange LR(defIndex, killIdx, ValNo);
244        interval.addRange(LR);
245        DEBUG(dbgs() << " +" << LR << "\n");
246        return;
247      }
248    }
249
250    // The other case we handle is when a virtual register lives to the end
251    // of the defining block, potentially live across some blocks, then is
252    // live into some number of blocks, but gets killed.  Start by adding a
253    // range that goes from this definition to the end of the defining block.
254    LiveRange NewLR(defIndex, getMBBEndIdx(mbb), ValNo);
255    DEBUG(dbgs() << " +" << NewLR);
256    interval.addRange(NewLR);
257
258    bool PHIJoin = LV->isPHIJoin(interval.reg);
259
260    if (PHIJoin) {
261      // A phi join register is killed at the end of the MBB and revived as a
262      // new valno in the killing blocks.
263      assert(vi.AliveBlocks.empty() && "Phi join can't pass through blocks");
264      DEBUG(dbgs() << " phi-join");
265      ValNo->setHasPHIKill(true);
266    } else {
267      // Iterate over all of the blocks that the variable is completely
268      // live in, adding [insrtIndex(begin), instrIndex(end)+4) to the
269      // live interval.
270      for (SparseBitVector<>::iterator I = vi.AliveBlocks.begin(),
271               E = vi.AliveBlocks.end(); I != E; ++I) {
272        MachineBasicBlock *aliveBlock = MF->getBlockNumbered(*I);
273        LiveRange LR(getMBBStartIdx(aliveBlock), getMBBEndIdx(aliveBlock),
274                     ValNo);
275        interval.addRange(LR);
276        DEBUG(dbgs() << " +" << LR);
277      }
278    }
279
280    // Finally, this virtual register is live from the start of any killing
281    // block to the 'use' slot of the killing instruction.
282    for (unsigned i = 0, e = vi.Kills.size(); i != e; ++i) {
283      MachineInstr *Kill = vi.Kills[i];
284      SlotIndex Start = getMBBStartIdx(Kill->getParent());
285      SlotIndex killIdx = getInstructionIndex(Kill).getRegSlot();
286
287      // Create interval with one of a NEW value number.  Note that this value
288      // number isn't actually defined by an instruction, weird huh? :)
289      if (PHIJoin) {
290        assert(getInstructionFromIndex(Start) == 0 &&
291               "PHI def index points at actual instruction.");
292        ValNo = interval.getNextValue(Start, VNInfoAllocator);
293      }
294      LiveRange LR(Start, killIdx, ValNo);
295      interval.addRange(LR);
296      DEBUG(dbgs() << " +" << LR);
297    }
298
299  } else {
300    if (MultipleDefsBySameMI(*mi, MOIdx))
301      // Multiple defs of the same virtual register by the same instruction.
302      // e.g. %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ...
303      // This is likely due to elimination of REG_SEQUENCE instructions. Return
304      // here since there is nothing to do.
305      return;
306
307    // If this is the second time we see a virtual register definition, it
308    // must be due to phi elimination or two addr elimination.  If this is
309    // the result of two address elimination, then the vreg is one of the
310    // def-and-use register operand.
311
312    // It may also be partial redef like this:
313    // 80  %reg1041:6<def> = VSHRNv4i16 %reg1034<kill>, 12, pred:14, pred:%reg0
314    // 120 %reg1041:5<def> = VSHRNv4i16 %reg1039<kill>, 12, pred:14, pred:%reg0
315    bool PartReDef = isPartialRedef(MIIdx, MO, interval);
316    if (PartReDef || mi->isRegTiedToUseOperand(MOIdx)) {
317      // If this is a two-address definition, then we have already processed
318      // the live range.  The only problem is that we didn't realize there
319      // are actually two values in the live interval.  Because of this we
320      // need to take the LiveRegion that defines this register and split it
321      // into two values.
322      SlotIndex RedefIndex = MIIdx.getRegSlot(MO.isEarlyClobber());
323
324      const LiveRange *OldLR =
325        interval.getLiveRangeContaining(RedefIndex.getRegSlot(true));
326      VNInfo *OldValNo = OldLR->valno;
327      SlotIndex DefIndex = OldValNo->def.getRegSlot();
328
329      // Delete the previous value, which should be short and continuous,
330      // because the 2-addr copy must be in the same MBB as the redef.
331      interval.removeRange(DefIndex, RedefIndex);
332
333      // The new value number (#1) is defined by the instruction we claimed
334      // defined value #0.
335      VNInfo *ValNo = interval.createValueCopy(OldValNo, VNInfoAllocator);
336
337      // Value#0 is now defined by the 2-addr instruction.
338      OldValNo->def = RedefIndex;
339
340      // Add the new live interval which replaces the range for the input copy.
341      LiveRange LR(DefIndex, RedefIndex, ValNo);
342      DEBUG(dbgs() << " replace range with " << LR);
343      interval.addRange(LR);
344
345      // If this redefinition is dead, we need to add a dummy unit live
346      // range covering the def slot.
347      if (MO.isDead())
348        interval.addRange(LiveRange(RedefIndex, RedefIndex.getDeadSlot(),
349                                    OldValNo));
350
351      DEBUG(dbgs() << " RESULT: " << interval);
352    } else if (LV->isPHIJoin(interval.reg)) {
353      // In the case of PHI elimination, each variable definition is only
354      // live until the end of the block.  We've already taken care of the
355      // rest of the live range.
356
357      SlotIndex defIndex = MIIdx.getRegSlot();
358      if (MO.isEarlyClobber())
359        defIndex = MIIdx.getRegSlot(true);
360
361      VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator);
362
363      SlotIndex killIndex = getMBBEndIdx(mbb);
364      LiveRange LR(defIndex, killIndex, ValNo);
365      interval.addRange(LR);
366      ValNo->setHasPHIKill(true);
367      DEBUG(dbgs() << " phi-join +" << LR);
368    } else {
369      llvm_unreachable("Multiply defined register");
370    }
371  }
372
373  DEBUG(dbgs() << '\n');
374}
375
376void LiveIntervals::handleRegisterDef(MachineBasicBlock *MBB,
377                                      MachineBasicBlock::iterator MI,
378                                      SlotIndex MIIdx,
379                                      MachineOperand& MO,
380                                      unsigned MOIdx) {
381  if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
382    handleVirtualRegisterDef(MBB, MI, MIIdx, MO, MOIdx,
383                             getOrCreateInterval(MO.getReg()));
384}
385
386/// computeIntervals - computes the live intervals for virtual
387/// registers. for some ordering of the machine instructions [1,N] a
388/// live interval is an interval [i, j) where 1 <= i <= j < N for
389/// which a variable is live
390void LiveIntervals::computeIntervals() {
391  DEBUG(dbgs() << "********** COMPUTING LIVE INTERVALS **********\n"
392               << "********** Function: "
393               << ((Value*)MF->getFunction())->getName() << '\n');
394
395  RegMaskBlocks.resize(MF->getNumBlockIDs());
396
397  SmallVector<unsigned, 8> UndefUses;
398  for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end();
399       MBBI != E; ++MBBI) {
400    MachineBasicBlock *MBB = MBBI;
401    RegMaskBlocks[MBB->getNumber()].first = RegMaskSlots.size();
402
403    if (MBB->empty())
404      continue;
405
406    // Track the index of the current machine instr.
407    SlotIndex MIIndex = getMBBStartIdx(MBB);
408    DEBUG(dbgs() << "BB#" << MBB->getNumber()
409          << ":\t\t# derived from " << MBB->getName() << "\n");
410
411    // Skip over empty initial indices.
412    if (getInstructionFromIndex(MIIndex) == 0)
413      MIIndex = Indexes->getNextNonNullIndex(MIIndex);
414
415    for (MachineBasicBlock::iterator MI = MBB->begin(), miEnd = MBB->end();
416         MI != miEnd; ++MI) {
417      DEBUG(dbgs() << MIIndex << "\t" << *MI);
418      if (MI->isDebugValue())
419        continue;
420      assert(Indexes->getInstructionFromIndex(MIIndex) == MI &&
421             "Lost SlotIndex synchronization");
422
423      // Handle defs.
424      for (int i = MI->getNumOperands() - 1; i >= 0; --i) {
425        MachineOperand &MO = MI->getOperand(i);
426
427        // Collect register masks.
428        if (MO.isRegMask()) {
429          RegMaskSlots.push_back(MIIndex.getRegSlot());
430          RegMaskBits.push_back(MO.getRegMask());
431          continue;
432        }
433
434        if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg()))
435          continue;
436
437        // handle register defs - build intervals
438        if (MO.isDef())
439          handleRegisterDef(MBB, MI, MIIndex, MO, i);
440        else if (MO.isUndef())
441          UndefUses.push_back(MO.getReg());
442      }
443
444      // Move to the next instr slot.
445      MIIndex = Indexes->getNextNonNullIndex(MIIndex);
446    }
447
448    // Compute the number of register mask instructions in this block.
449    std::pair<unsigned, unsigned> &RMB = RegMaskBlocks[MBB->getNumber()];
450    RMB.second = RegMaskSlots.size() - RMB.first;;
451  }
452
453  // Create empty intervals for registers defined by implicit_def's (except
454  // for those implicit_def that define values which are liveout of their
455  // blocks.
456  for (unsigned i = 0, e = UndefUses.size(); i != e; ++i) {
457    unsigned UndefReg = UndefUses[i];
458    (void)getOrCreateInterval(UndefReg);
459  }
460}
461
462LiveInterval* LiveIntervals::createInterval(unsigned reg) {
463  float Weight = TargetRegisterInfo::isPhysicalRegister(reg) ? HUGE_VALF : 0.0F;
464  return new LiveInterval(reg, Weight);
465}
466
467
468/// computeVirtRegInterval - Compute the live interval of a virtual register,
469/// based on defs and uses.
470void LiveIntervals::computeVirtRegInterval(LiveInterval *LI) {
471  assert(LRCalc && "LRCalc not initialized.");
472  assert(LI->empty() && "Should only compute empty intervals.");
473  LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
474  LRCalc->createDeadDefs(LI);
475  LRCalc->extendToUses(LI);
476}
477
478
479//===----------------------------------------------------------------------===//
480//                           Register Unit Liveness
481//===----------------------------------------------------------------------===//
482//
483// Fixed interference typically comes from ABI boundaries: Function arguments
484// and return values are passed in fixed registers, and so are exception
485// pointers entering landing pads. Certain instructions require values to be
486// present in specific registers. That is also represented through fixed
487// interference.
488//
489
490/// computeRegUnitInterval - Compute the live interval of a register unit, based
491/// on the uses and defs of aliasing registers.  The interval should be empty,
492/// or contain only dead phi-defs from ABI blocks.
493void LiveIntervals::computeRegUnitInterval(LiveInterval *LI) {
494  unsigned Unit = LI->reg;
495
496  assert(LRCalc && "LRCalc not initialized.");
497  LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
498
499  // The physregs aliasing Unit are the roots and their super-registers.
500  // Create all values as dead defs before extending to uses. Note that roots
501  // may share super-registers. That's OK because createDeadDefs() is
502  // idempotent. It is very rare for a register unit to have multiple roots, so
503  // uniquing super-registers is probably not worthwhile.
504  for (MCRegUnitRootIterator Roots(Unit, TRI); Roots.isValid(); ++Roots) {
505    unsigned Root = *Roots;
506    if (!MRI->reg_empty(Root))
507      LRCalc->createDeadDefs(LI, Root);
508    for (MCSuperRegIterator Supers(Root, TRI); Supers.isValid(); ++Supers) {
509      if (!MRI->reg_empty(*Supers))
510        LRCalc->createDeadDefs(LI, *Supers);
511    }
512  }
513
514  // Now extend LI to reach all uses.
515  // Ignore uses of reserved registers. We only track defs of those.
516  for (MCRegUnitRootIterator Roots(Unit, TRI); Roots.isValid(); ++Roots) {
517    unsigned Root = *Roots;
518    if (!isReserved(Root) && !MRI->reg_empty(Root))
519      LRCalc->extendToUses(LI, Root);
520    for (MCSuperRegIterator Supers(Root, TRI); Supers.isValid(); ++Supers) {
521      unsigned Reg = *Supers;
522      if (!isReserved(Reg) && !MRI->reg_empty(Reg))
523        LRCalc->extendToUses(LI, Reg);
524    }
525  }
526}
527
528
529/// computeLiveInRegUnits - Precompute the live ranges of any register units
530/// that are live-in to an ABI block somewhere. Register values can appear
531/// without a corresponding def when entering the entry block or a landing pad.
532///
533void LiveIntervals::computeLiveInRegUnits() {
534  RegUnitIntervals.resize(TRI->getNumRegUnits());
535  DEBUG(dbgs() << "Computing live-in reg-units in ABI blocks.\n");
536
537  // Keep track of the intervals allocated.
538  SmallVector<LiveInterval*, 8> NewIntvs;
539
540  // Check all basic blocks for live-ins.
541  for (MachineFunction::const_iterator MFI = MF->begin(), MFE = MF->end();
542       MFI != MFE; ++MFI) {
543    const MachineBasicBlock *MBB = MFI;
544
545    // We only care about ABI blocks: Entry + landing pads.
546    if ((MFI != MF->begin() && !MBB->isLandingPad()) || MBB->livein_empty())
547      continue;
548
549    // Create phi-defs at Begin for all live-in registers.
550    SlotIndex Begin = Indexes->getMBBStartIdx(MBB);
551    DEBUG(dbgs() << Begin << "\tBB#" << MBB->getNumber());
552    for (MachineBasicBlock::livein_iterator LII = MBB->livein_begin(),
553         LIE = MBB->livein_end(); LII != LIE; ++LII) {
554      for (MCRegUnitIterator Units(*LII, TRI); Units.isValid(); ++Units) {
555        unsigned Unit = *Units;
556        LiveInterval *Intv = RegUnitIntervals[Unit];
557        if (!Intv) {
558          Intv = RegUnitIntervals[Unit] = new LiveInterval(Unit, HUGE_VALF);
559          NewIntvs.push_back(Intv);
560        }
561        VNInfo *VNI = Intv->createDeadDef(Begin, getVNInfoAllocator());
562        (void)VNI;
563        DEBUG(dbgs() << ' ' << PrintRegUnit(Unit, TRI) << '#' << VNI->id);
564      }
565    }
566    DEBUG(dbgs() << '\n');
567  }
568  DEBUG(dbgs() << "Created " << NewIntvs.size() << " new intervals.\n");
569
570  // Compute the 'normal' part of the intervals.
571  for (unsigned i = 0, e = NewIntvs.size(); i != e; ++i)
572    computeRegUnitInterval(NewIntvs[i]);
573}
574
575
576/// shrinkToUses - After removing some uses of a register, shrink its live
577/// range to just the remaining uses. This method does not compute reaching
578/// defs for new uses, and it doesn't remove dead defs.
579bool LiveIntervals::shrinkToUses(LiveInterval *li,
580                                 SmallVectorImpl<MachineInstr*> *dead) {
581  DEBUG(dbgs() << "Shrink: " << *li << '\n');
582  assert(TargetRegisterInfo::isVirtualRegister(li->reg)
583         && "Can only shrink virtual registers");
584  // Find all the values used, including PHI kills.
585  SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList;
586
587  // Blocks that have already been added to WorkList as live-out.
588  SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
589
590  // Visit all instructions reading li->reg.
591  for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(li->reg);
592       MachineInstr *UseMI = I.skipInstruction();) {
593    if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg))
594      continue;
595    SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
596    LiveRangeQuery LRQ(*li, Idx);
597    VNInfo *VNI = LRQ.valueIn();
598    if (!VNI) {
599      // This shouldn't happen: readsVirtualRegister returns true, but there is
600      // no live value. It is likely caused by a target getting <undef> flags
601      // wrong.
602      DEBUG(dbgs() << Idx << '\t' << *UseMI
603                   << "Warning: Instr claims to read non-existent value in "
604                    << *li << '\n');
605      continue;
606    }
607    // Special case: An early-clobber tied operand reads and writes the
608    // register one slot early.
609    if (VNInfo *DefVNI = LRQ.valueDefined())
610      Idx = DefVNI->def;
611
612    WorkList.push_back(std::make_pair(Idx, VNI));
613  }
614
615  // Create a new live interval with only minimal live segments per def.
616  LiveInterval NewLI(li->reg, 0);
617  for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
618       I != E; ++I) {
619    VNInfo *VNI = *I;
620    if (VNI->isUnused())
621      continue;
622    NewLI.addRange(LiveRange(VNI->def, VNI->def.getDeadSlot(), VNI));
623  }
624
625  // Keep track of the PHIs that are in use.
626  SmallPtrSet<VNInfo*, 8> UsedPHIs;
627
628  // Extend intervals to reach all uses in WorkList.
629  while (!WorkList.empty()) {
630    SlotIndex Idx = WorkList.back().first;
631    VNInfo *VNI = WorkList.back().second;
632    WorkList.pop_back();
633    const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot());
634    SlotIndex BlockStart = getMBBStartIdx(MBB);
635
636    // Extend the live range for VNI to be live at Idx.
637    if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) {
638      (void)ExtVNI;
639      assert(ExtVNI == VNI && "Unexpected existing value number");
640      // Is this a PHIDef we haven't seen before?
641      if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI))
642        continue;
643      // The PHI is live, make sure the predecessors are live-out.
644      for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
645           PE = MBB->pred_end(); PI != PE; ++PI) {
646        if (!LiveOut.insert(*PI))
647          continue;
648        SlotIndex Stop = getMBBEndIdx(*PI);
649        // A predecessor is not required to have a live-out value for a PHI.
650        if (VNInfo *PVNI = li->getVNInfoBefore(Stop))
651          WorkList.push_back(std::make_pair(Stop, PVNI));
652      }
653      continue;
654    }
655
656    // VNI is live-in to MBB.
657    DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
658    NewLI.addRange(LiveRange(BlockStart, Idx, VNI));
659
660    // Make sure VNI is live-out from the predecessors.
661    for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
662         PE = MBB->pred_end(); PI != PE; ++PI) {
663      if (!LiveOut.insert(*PI))
664        continue;
665      SlotIndex Stop = getMBBEndIdx(*PI);
666      assert(li->getVNInfoBefore(Stop) == VNI &&
667             "Wrong value out of predecessor");
668      WorkList.push_back(std::make_pair(Stop, VNI));
669    }
670  }
671
672  // Handle dead values.
673  bool CanSeparate = false;
674  for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
675       I != E; ++I) {
676    VNInfo *VNI = *I;
677    if (VNI->isUnused())
678      continue;
679    LiveInterval::iterator LII = NewLI.FindLiveRangeContaining(VNI->def);
680    assert(LII != NewLI.end() && "Missing live range for PHI");
681    if (LII->end != VNI->def.getDeadSlot())
682      continue;
683    if (VNI->isPHIDef()) {
684      // This is a dead PHI. Remove it.
685      VNI->setIsUnused(true);
686      NewLI.removeRange(*LII);
687      DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n");
688      CanSeparate = true;
689    } else {
690      // This is a dead def. Make sure the instruction knows.
691      MachineInstr *MI = getInstructionFromIndex(VNI->def);
692      assert(MI && "No instruction defining live value");
693      MI->addRegisterDead(li->reg, TRI);
694      if (dead && MI->allDefsAreDead()) {
695        DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI);
696        dead->push_back(MI);
697      }
698    }
699  }
700
701  // Move the trimmed ranges back.
702  li->ranges.swap(NewLI.ranges);
703  DEBUG(dbgs() << "Shrunk: " << *li << '\n');
704  return CanSeparate;
705}
706
707
708//===----------------------------------------------------------------------===//
709// Register allocator hooks.
710//
711
712void LiveIntervals::addKillFlags() {
713  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
714    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
715    if (MRI->reg_nodbg_empty(Reg))
716      continue;
717    LiveInterval *LI = &getInterval(Reg);
718
719    // Every instruction that kills Reg corresponds to a live range end point.
720    for (LiveInterval::iterator RI = LI->begin(), RE = LI->end(); RI != RE;
721         ++RI) {
722      // A block index indicates an MBB edge.
723      if (RI->end.isBlock())
724        continue;
725      MachineInstr *MI = getInstructionFromIndex(RI->end);
726      if (!MI)
727        continue;
728      MI->addRegisterKilled(Reg, NULL);
729    }
730  }
731}
732
733MachineBasicBlock*
734LiveIntervals::intervalIsInOneMBB(const LiveInterval &LI) const {
735  // A local live range must be fully contained inside the block, meaning it is
736  // defined and killed at instructions, not at block boundaries. It is not
737  // live in or or out of any block.
738  //
739  // It is technically possible to have a PHI-defined live range identical to a
740  // single block, but we are going to return false in that case.
741
742  SlotIndex Start = LI.beginIndex();
743  if (Start.isBlock())
744    return NULL;
745
746  SlotIndex Stop = LI.endIndex();
747  if (Stop.isBlock())
748    return NULL;
749
750  // getMBBFromIndex doesn't need to search the MBB table when both indexes
751  // belong to proper instructions.
752  MachineBasicBlock *MBB1 = Indexes->getMBBFromIndex(Start);
753  MachineBasicBlock *MBB2 = Indexes->getMBBFromIndex(Stop);
754  return MBB1 == MBB2 ? MBB1 : NULL;
755}
756
757float
758LiveIntervals::getSpillWeight(bool isDef, bool isUse, unsigned loopDepth) {
759  // Limit the loop depth ridiculousness.
760  if (loopDepth > 200)
761    loopDepth = 200;
762
763  // The loop depth is used to roughly estimate the number of times the
764  // instruction is executed. Something like 10^d is simple, but will quickly
765  // overflow a float. This expression behaves like 10^d for small d, but is
766  // more tempered for large d. At d=200 we get 6.7e33 which leaves a bit of
767  // headroom before overflow.
768  // By the way, powf() might be unavailable here. For consistency,
769  // We may take pow(double,double).
770  float lc = std::pow(1 + (100.0 / (loopDepth + 10)), (double)loopDepth);
771
772  return (isDef + isUse) * lc;
773}
774
775LiveRange LiveIntervals::addLiveRangeToEndOfBlock(unsigned reg,
776                                                  MachineInstr* startInst) {
777  LiveInterval& Interval = getOrCreateInterval(reg);
778  VNInfo* VN = Interval.getNextValue(
779    SlotIndex(getInstructionIndex(startInst).getRegSlot()),
780    getVNInfoAllocator());
781  VN->setHasPHIKill(true);
782  LiveRange LR(
783     SlotIndex(getInstructionIndex(startInst).getRegSlot()),
784     getMBBEndIdx(startInst->getParent()), VN);
785  Interval.addRange(LR);
786
787  return LR;
788}
789
790
791//===----------------------------------------------------------------------===//
792//                          Register mask functions
793//===----------------------------------------------------------------------===//
794
795bool LiveIntervals::checkRegMaskInterference(LiveInterval &LI,
796                                             BitVector &UsableRegs) {
797  if (LI.empty())
798    return false;
799  LiveInterval::iterator LiveI = LI.begin(), LiveE = LI.end();
800
801  // Use a smaller arrays for local live ranges.
802  ArrayRef<SlotIndex> Slots;
803  ArrayRef<const uint32_t*> Bits;
804  if (MachineBasicBlock *MBB = intervalIsInOneMBB(LI)) {
805    Slots = getRegMaskSlotsInBlock(MBB->getNumber());
806    Bits = getRegMaskBitsInBlock(MBB->getNumber());
807  } else {
808    Slots = getRegMaskSlots();
809    Bits = getRegMaskBits();
810  }
811
812  // We are going to enumerate all the register mask slots contained in LI.
813  // Start with a binary search of RegMaskSlots to find a starting point.
814  ArrayRef<SlotIndex>::iterator SlotI =
815    std::lower_bound(Slots.begin(), Slots.end(), LiveI->start);
816  ArrayRef<SlotIndex>::iterator SlotE = Slots.end();
817
818  // No slots in range, LI begins after the last call.
819  if (SlotI == SlotE)
820    return false;
821
822  bool Found = false;
823  for (;;) {
824    assert(*SlotI >= LiveI->start);
825    // Loop over all slots overlapping this segment.
826    while (*SlotI < LiveI->end) {
827      // *SlotI overlaps LI. Collect mask bits.
828      if (!Found) {
829        // This is the first overlap. Initialize UsableRegs to all ones.
830        UsableRegs.clear();
831        UsableRegs.resize(TRI->getNumRegs(), true);
832        Found = true;
833      }
834      // Remove usable registers clobbered by this mask.
835      UsableRegs.clearBitsNotInMask(Bits[SlotI-Slots.begin()]);
836      if (++SlotI == SlotE)
837        return Found;
838    }
839    // *SlotI is beyond the current LI segment.
840    LiveI = LI.advanceTo(LiveI, *SlotI);
841    if (LiveI == LiveE)
842      return Found;
843    // Advance SlotI until it overlaps.
844    while (*SlotI < LiveI->start)
845      if (++SlotI == SlotE)
846        return Found;
847  }
848}
849
850//===----------------------------------------------------------------------===//
851//                         IntervalUpdate class.
852//===----------------------------------------------------------------------===//
853
854// HMEditor is a toolkit used by handleMove to trim or extend live intervals.
855class LiveIntervals::HMEditor {
856private:
857  LiveIntervals& LIS;
858  const MachineRegisterInfo& MRI;
859  const TargetRegisterInfo& TRI;
860  SlotIndex NewIdx;
861
862  typedef std::pair<LiveInterval*, LiveRange*> IntRangePair;
863  typedef DenseSet<IntRangePair> RangeSet;
864
865  struct RegRanges {
866    LiveRange* Use;
867    LiveRange* EC;
868    LiveRange* Dead;
869    LiveRange* Def;
870    RegRanges() : Use(0), EC(0), Dead(0), Def(0) {}
871  };
872  typedef DenseMap<unsigned, RegRanges> BundleRanges;
873
874public:
875  HMEditor(LiveIntervals& LIS, const MachineRegisterInfo& MRI,
876           const TargetRegisterInfo& TRI, SlotIndex NewIdx)
877    : LIS(LIS), MRI(MRI), TRI(TRI), NewIdx(NewIdx) {}
878
879  // Update intervals for all operands of MI from OldIdx to NewIdx.
880  // This assumes that MI used to be at OldIdx, and now resides at
881  // NewIdx.
882  void moveAllRangesFrom(MachineInstr* MI, SlotIndex OldIdx) {
883    assert(NewIdx != OldIdx && "No-op move? That's a bit strange.");
884
885    // Collect the operands.
886    RangeSet Entering, Internal, Exiting;
887    bool hasRegMaskOp = false;
888    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
889
890    // To keep the LiveRanges valid within an interval, move the ranges closest
891    // to the destination first. This prevents ranges from overlapping, to that
892    // APIs like removeRange still work.
893    if (NewIdx < OldIdx) {
894      moveAllEnteringFrom(OldIdx, Entering);
895      moveAllInternalFrom(OldIdx, Internal);
896      moveAllExitingFrom(OldIdx, Exiting);
897    }
898    else {
899      moveAllExitingFrom(OldIdx, Exiting);
900      moveAllInternalFrom(OldIdx, Internal);
901      moveAllEnteringFrom(OldIdx, Entering);
902    }
903
904    if (hasRegMaskOp)
905      updateRegMaskSlots(OldIdx);
906
907#ifndef NDEBUG
908    LIValidator validator;
909    validator = std::for_each(Entering.begin(), Entering.end(), validator);
910    validator = std::for_each(Internal.begin(), Internal.end(), validator);
911    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
912    assert(validator.rangesOk() && "moveAllOperandsFrom broke liveness.");
913#endif
914
915  }
916
917  // Update intervals for all operands of MI to refer to BundleStart's
918  // SlotIndex.
919  void moveAllRangesInto(MachineInstr* MI, MachineInstr* BundleStart) {
920    if (MI == BundleStart)
921      return; // Bundling instr with itself - nothing to do.
922
923    SlotIndex OldIdx = LIS.getSlotIndexes()->getInstructionIndex(MI);
924    assert(LIS.getSlotIndexes()->getInstructionFromIndex(OldIdx) == MI &&
925           "SlotIndex <-> Instruction mapping broken for MI");
926
927    // Collect all ranges already in the bundle.
928    MachineBasicBlock::instr_iterator BII(BundleStart);
929    RangeSet Entering, Internal, Exiting;
930    bool hasRegMaskOp = false;
931    collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
932    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
933    for (++BII; &*BII == MI || BII->isInsideBundle(); ++BII) {
934      if (&*BII == MI)
935        continue;
936      collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
937      assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
938    }
939
940    BundleRanges BR = createBundleRanges(Entering, Internal, Exiting);
941
942    Entering.clear();
943    Internal.clear();
944    Exiting.clear();
945    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
946    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
947
948    DEBUG(dbgs() << "Entering: " << Entering.size() << "\n");
949    DEBUG(dbgs() << "Internal: " << Internal.size() << "\n");
950    DEBUG(dbgs() << "Exiting: " << Exiting.size() << "\n");
951
952    moveAllEnteringFromInto(OldIdx, Entering, BR);
953    moveAllInternalFromInto(OldIdx, Internal, BR);
954    moveAllExitingFromInto(OldIdx, Exiting, BR);
955
956
957#ifndef NDEBUG
958    LIValidator validator;
959    validator = std::for_each(Entering.begin(), Entering.end(), validator);
960    validator = std::for_each(Internal.begin(), Internal.end(), validator);
961    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
962    assert(validator.rangesOk() && "moveAllOperandsInto broke liveness.");
963#endif
964  }
965
966private:
967
968#ifndef NDEBUG
969  class LIValidator {
970  private:
971    DenseSet<const LiveInterval*> Checked, Bogus;
972  public:
973    void operator()(const IntRangePair& P) {
974      const LiveInterval* LI = P.first;
975      if (Checked.count(LI))
976        return;
977      Checked.insert(LI);
978      if (LI->empty())
979        return;
980      SlotIndex LastEnd = LI->begin()->start;
981      for (LiveInterval::const_iterator LRI = LI->begin(), LRE = LI->end();
982           LRI != LRE; ++LRI) {
983        const LiveRange& LR = *LRI;
984        if (LastEnd > LR.start || LR.start >= LR.end)
985          Bogus.insert(LI);
986        LastEnd = LR.end;
987      }
988    }
989
990    bool rangesOk() const {
991      return Bogus.empty();
992    }
993  };
994#endif
995
996  // Collect IntRangePairs for all operands of MI that may need fixing.
997  // Treat's MI's index as OldIdx (regardless of what it is in SlotIndexes'
998  // maps).
999  void collectRanges(MachineInstr* MI, RangeSet& Entering, RangeSet& Internal,
1000                     RangeSet& Exiting, bool& hasRegMaskOp, SlotIndex OldIdx) {
1001    hasRegMaskOp = false;
1002    for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
1003                                    MOE = MI->operands_end();
1004         MOI != MOE; ++MOI) {
1005      const MachineOperand& MO = *MOI;
1006
1007      if (MO.isRegMask()) {
1008        hasRegMaskOp = true;
1009        continue;
1010      }
1011
1012      if (!MO.isReg() || MO.getReg() == 0)
1013        continue;
1014
1015      unsigned Reg = MO.getReg();
1016
1017      // TODO: Currently we're skipping uses that are reserved or have no
1018      // interval, but we're not updating their kills. This should be
1019      // fixed.
1020      if (TargetRegisterInfo::isPhysicalRegister(Reg) && LIS.isReserved(Reg))
1021        continue;
1022
1023      // Collect ranges for register units. These live ranges are computed on
1024      // demand, so just skip any that haven't been computed yet.
1025      if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
1026        for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
1027          if (LiveInterval *LI = LIS.getCachedRegUnit(*Units))
1028            collectRanges(MO, LI, Entering, Internal, Exiting, OldIdx);
1029      } else {
1030        // Collect ranges for individual virtual registers.
1031        collectRanges(MO, &LIS.getInterval(Reg),
1032                      Entering, Internal, Exiting, OldIdx);
1033      }
1034    }
1035  }
1036
1037  void collectRanges(const MachineOperand &MO, LiveInterval *LI,
1038                     RangeSet &Entering, RangeSet &Internal, RangeSet &Exiting,
1039                     SlotIndex OldIdx) {
1040    if (MO.readsReg()) {
1041      LiveRange* LR = LI->getLiveRangeContaining(OldIdx);
1042      if (LR != 0)
1043        Entering.insert(std::make_pair(LI, LR));
1044    }
1045    if (MO.isDef()) {
1046      LiveRange* LR = LI->getLiveRangeContaining(OldIdx.getRegSlot());
1047      assert(LR != 0 && "No live range for def?");
1048      if (LR->end > OldIdx.getDeadSlot())
1049        Exiting.insert(std::make_pair(LI, LR));
1050      else
1051        Internal.insert(std::make_pair(LI, LR));
1052    }
1053  }
1054
1055  BundleRanges createBundleRanges(RangeSet& Entering,
1056                                  RangeSet& Internal,
1057                                  RangeSet& Exiting) {
1058    BundleRanges BR;
1059
1060    for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1061         EI != EE; ++EI) {
1062      LiveInterval* LI = EI->first;
1063      LiveRange* LR = EI->second;
1064      BR[LI->reg].Use = LR;
1065    }
1066
1067    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1068         II != IE; ++II) {
1069      LiveInterval* LI = II->first;
1070      LiveRange* LR = II->second;
1071      if (LR->end.isDead()) {
1072        BR[LI->reg].Dead = LR;
1073      } else {
1074        BR[LI->reg].EC = LR;
1075      }
1076    }
1077
1078    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1079         EI != EE; ++EI) {
1080      LiveInterval* LI = EI->first;
1081      LiveRange* LR = EI->second;
1082      BR[LI->reg].Def = LR;
1083    }
1084
1085    return BR;
1086  }
1087
1088  void moveKillFlags(unsigned reg, SlotIndex OldIdx, SlotIndex newKillIdx) {
1089    MachineInstr* OldKillMI = LIS.getInstructionFromIndex(OldIdx);
1090    if (!OldKillMI->killsRegister(reg))
1091      return; // Bail out if we don't have kill flags on the old register.
1092    MachineInstr* NewKillMI = LIS.getInstructionFromIndex(newKillIdx);
1093    assert(OldKillMI->killsRegister(reg) && "Old 'kill' instr isn't a kill.");
1094    assert(!NewKillMI->killsRegister(reg) &&
1095           "New kill instr is already a kill.");
1096    OldKillMI->clearRegisterKills(reg, &TRI);
1097    NewKillMI->addRegisterKilled(reg, &TRI);
1098  }
1099
1100  void updateRegMaskSlots(SlotIndex OldIdx) {
1101    SmallVectorImpl<SlotIndex>::iterator RI =
1102      std::lower_bound(LIS.RegMaskSlots.begin(), LIS.RegMaskSlots.end(),
1103                       OldIdx);
1104    assert(*RI == OldIdx && "No RegMask at OldIdx.");
1105    *RI = NewIdx;
1106    assert(*prior(RI) < *RI && *RI < *next(RI) &&
1107           "RegSlots out of order. Did you move one call across another?");
1108  }
1109
1110  // Return the last use of reg between NewIdx and OldIdx.
1111  SlotIndex findLastUseBefore(unsigned Reg, SlotIndex OldIdx) {
1112    SlotIndex LastUse = NewIdx;
1113    for (MachineRegisterInfo::use_nodbg_iterator
1114           UI = MRI.use_nodbg_begin(Reg),
1115           UE = MRI.use_nodbg_end();
1116         UI != UE; UI.skipInstruction()) {
1117      const MachineInstr* MI = &*UI;
1118      SlotIndex InstSlot = LIS.getSlotIndexes()->getInstructionIndex(MI);
1119      if (InstSlot > LastUse && InstSlot < OldIdx)
1120        LastUse = InstSlot;
1121    }
1122    return LastUse;
1123  }
1124
1125  void moveEnteringUpFrom(SlotIndex OldIdx, IntRangePair& P) {
1126    LiveInterval* LI = P.first;
1127    LiveRange* LR = P.second;
1128    bool LiveThrough = LR->end > OldIdx.getRegSlot();
1129    if (LiveThrough)
1130      return;
1131    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
1132    if (LastUse != NewIdx)
1133      moveKillFlags(LI->reg, NewIdx, LastUse);
1134    LR->end = LastUse.getRegSlot();
1135  }
1136
1137  void moveEnteringDownFrom(SlotIndex OldIdx, IntRangePair& P) {
1138    LiveInterval* LI = P.first;
1139    LiveRange* LR = P.second;
1140    // Extend the LiveRange if NewIdx is past the end.
1141    if (NewIdx > LR->end) {
1142      // Move kill flags if OldIdx was not originally the end
1143      // (otherwise LR->end points to an invalid slot).
1144      if (LR->end.getRegSlot() != OldIdx.getRegSlot()) {
1145        assert(LR->end > OldIdx && "LiveRange does not cover original slot");
1146        moveKillFlags(LI->reg, LR->end, NewIdx);
1147      }
1148      LR->end = NewIdx.getRegSlot();
1149    }
1150  }
1151
1152  void moveAllEnteringFrom(SlotIndex OldIdx, RangeSet& Entering) {
1153    bool GoingUp = NewIdx < OldIdx;
1154
1155    if (GoingUp) {
1156      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1157           EI != EE; ++EI)
1158        moveEnteringUpFrom(OldIdx, *EI);
1159    } else {
1160      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1161           EI != EE; ++EI)
1162        moveEnteringDownFrom(OldIdx, *EI);
1163    }
1164  }
1165
1166  void moveInternalFrom(SlotIndex OldIdx, IntRangePair& P) {
1167    LiveInterval* LI = P.first;
1168    LiveRange* LR = P.second;
1169    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
1170           LR->end <= OldIdx.getDeadSlot() &&
1171           "Range should be internal to OldIdx.");
1172    LiveRange Tmp(*LR);
1173    Tmp.start = NewIdx.getRegSlot(LR->start.isEarlyClobber());
1174    Tmp.valno->def = Tmp.start;
1175    Tmp.end = LR->end.isDead() ? NewIdx.getDeadSlot() : NewIdx.getRegSlot();
1176    LI->removeRange(*LR);
1177    LI->addRange(Tmp);
1178  }
1179
1180  void moveAllInternalFrom(SlotIndex OldIdx, RangeSet& Internal) {
1181    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1182         II != IE; ++II)
1183      moveInternalFrom(OldIdx, *II);
1184  }
1185
1186  void moveExitingFrom(SlotIndex OldIdx, IntRangePair& P) {
1187    LiveRange* LR = P.second;
1188    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
1189           "Range should start in OldIdx.");
1190    assert(LR->end > OldIdx.getDeadSlot() && "Range should exit OldIdx.");
1191    SlotIndex NewStart = NewIdx.getRegSlot(LR->start.isEarlyClobber());
1192    LR->start = NewStart;
1193    LR->valno->def = NewStart;
1194  }
1195
1196  void moveAllExitingFrom(SlotIndex OldIdx, RangeSet& Exiting) {
1197    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1198         EI != EE; ++EI)
1199      moveExitingFrom(OldIdx, *EI);
1200  }
1201
1202  void moveEnteringUpFromInto(SlotIndex OldIdx, IntRangePair& P,
1203                              BundleRanges& BR) {
1204    LiveInterval* LI = P.first;
1205    LiveRange* LR = P.second;
1206    bool LiveThrough = LR->end > OldIdx.getRegSlot();
1207    if (LiveThrough) {
1208      assert((LR->start < NewIdx || BR[LI->reg].Def == LR) &&
1209             "Def in bundle should be def range.");
1210      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
1211             "If bundle has use for this reg it should be LR.");
1212      BR[LI->reg].Use = LR;
1213      return;
1214    }
1215
1216    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
1217    moveKillFlags(LI->reg, OldIdx, LastUse);
1218
1219    if (LR->start < NewIdx) {
1220      // Becoming a new entering range.
1221      assert(BR[LI->reg].Dead == 0 && BR[LI->reg].Def == 0 &&
1222             "Bundle shouldn't be re-defining reg mid-range.");
1223      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
1224             "Bundle shouldn't have different use range for same reg.");
1225      LR->end = LastUse.getRegSlot();
1226      BR[LI->reg].Use = LR;
1227    } else {
1228      // Becoming a new Dead-def.
1229      assert(LR->start == NewIdx.getRegSlot(LR->start.isEarlyClobber()) &&
1230             "Live range starting at unexpected slot.");
1231      assert(BR[LI->reg].Def == LR && "Reg should have def range.");
1232      assert(BR[LI->reg].Dead == 0 &&
1233               "Can't have def and dead def of same reg in a bundle.");
1234      LR->end = LastUse.getDeadSlot();
1235      BR[LI->reg].Dead = BR[LI->reg].Def;
1236      BR[LI->reg].Def = 0;
1237    }
1238  }
1239
1240  void moveEnteringDownFromInto(SlotIndex OldIdx, IntRangePair& P,
1241                                BundleRanges& BR) {
1242    LiveInterval* LI = P.first;
1243    LiveRange* LR = P.second;
1244    if (NewIdx > LR->end) {
1245      // Range extended to bundle. Add to bundle uses.
1246      // Note: Currently adds kill flags to bundle start.
1247      assert(BR[LI->reg].Use == 0 &&
1248             "Bundle already has use range for reg.");
1249      moveKillFlags(LI->reg, LR->end, NewIdx);
1250      LR->end = NewIdx.getRegSlot();
1251      BR[LI->reg].Use = LR;
1252    } else {
1253      assert(BR[LI->reg].Use != 0 &&
1254             "Bundle should already have a use range for reg.");
1255    }
1256  }
1257
1258  void moveAllEnteringFromInto(SlotIndex OldIdx, RangeSet& Entering,
1259                               BundleRanges& BR) {
1260    bool GoingUp = NewIdx < OldIdx;
1261
1262    if (GoingUp) {
1263      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1264           EI != EE; ++EI)
1265        moveEnteringUpFromInto(OldIdx, *EI, BR);
1266    } else {
1267      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1268           EI != EE; ++EI)
1269        moveEnteringDownFromInto(OldIdx, *EI, BR);
1270    }
1271  }
1272
1273  void moveInternalFromInto(SlotIndex OldIdx, IntRangePair& P,
1274                            BundleRanges& BR) {
1275    // TODO: Sane rules for moving ranges into bundles.
1276  }
1277
1278  void moveAllInternalFromInto(SlotIndex OldIdx, RangeSet& Internal,
1279                               BundleRanges& BR) {
1280    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1281         II != IE; ++II)
1282      moveInternalFromInto(OldIdx, *II, BR);
1283  }
1284
1285  void moveExitingFromInto(SlotIndex OldIdx, IntRangePair& P,
1286                           BundleRanges& BR) {
1287    LiveInterval* LI = P.first;
1288    LiveRange* LR = P.second;
1289
1290    assert(LR->start.isRegister() &&
1291           "Don't know how to merge exiting ECs into bundles yet.");
1292
1293    if (LR->end > NewIdx.getDeadSlot()) {
1294      // This range is becoming an exiting range on the bundle.
1295      // If there was an old dead-def of this reg, delete it.
1296      if (BR[LI->reg].Dead != 0) {
1297        LI->removeRange(*BR[LI->reg].Dead);
1298        BR[LI->reg].Dead = 0;
1299      }
1300      assert(BR[LI->reg].Def == 0 &&
1301             "Can't have two defs for the same variable exiting a bundle.");
1302      LR->start = NewIdx.getRegSlot();
1303      LR->valno->def = LR->start;
1304      BR[LI->reg].Def = LR;
1305    } else {
1306      // This range is becoming internal to the bundle.
1307      assert(LR->end == NewIdx.getRegSlot() &&
1308             "Can't bundle def whose kill is before the bundle");
1309      if (BR[LI->reg].Dead || BR[LI->reg].Def) {
1310        // Already have a def for this. Just delete range.
1311        LI->removeRange(*LR);
1312      } else {
1313        // Make range dead, record.
1314        LR->end = NewIdx.getDeadSlot();
1315        BR[LI->reg].Dead = LR;
1316        assert(BR[LI->reg].Use == LR &&
1317               "Range becoming dead should currently be use.");
1318      }
1319      // In both cases the range is no longer a use on the bundle.
1320      BR[LI->reg].Use = 0;
1321    }
1322  }
1323
1324  void moveAllExitingFromInto(SlotIndex OldIdx, RangeSet& Exiting,
1325                              BundleRanges& BR) {
1326    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1327         EI != EE; ++EI)
1328      moveExitingFromInto(OldIdx, *EI, BR);
1329  }
1330
1331};
1332
1333void LiveIntervals::handleMove(MachineInstr* MI) {
1334  SlotIndex OldIndex = Indexes->getInstructionIndex(MI);
1335  Indexes->removeMachineInstrFromMaps(MI);
1336  SlotIndex NewIndex = MI->isInsideBundle() ?
1337                        Indexes->getInstructionIndex(MI) :
1338                        Indexes->insertMachineInstrInMaps(MI);
1339  assert(getMBBStartIdx(MI->getParent()) <= OldIndex &&
1340         OldIndex < getMBBEndIdx(MI->getParent()) &&
1341         "Cannot handle moves across basic block boundaries.");
1342  assert(!MI->isBundled() && "Can't handle bundled instructions yet.");
1343
1344  HMEditor HME(*this, *MRI, *TRI, NewIndex);
1345  HME.moveAllRangesFrom(MI, OldIndex);
1346}
1347
1348void LiveIntervals::handleMoveIntoBundle(MachineInstr* MI,
1349                                         MachineInstr* BundleStart) {
1350  SlotIndex NewIndex = Indexes->getInstructionIndex(BundleStart);
1351  HMEditor HME(*this, *MRI, *TRI, NewIndex);
1352  HME.moveAllRangesInto(MI, BundleStart);
1353}
1354