LiveIntervalAnalysis.cpp revision 2769e9338429377f0d06fb78ec884bf69b0c7d41 
1//===-- LiveIntervalAnalysis.cpp - Live Interval Analysis -----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the LiveInterval analysis pass which is used
11// by the Linear Scan Register allocator. This pass linearizes the
12// basic blocks of the function in DFS order and uses the
13// LiveVariables pass to conservatively compute live intervals for
14// each virtual and physical register.
15//
16//===----------------------------------------------------------------------===//
17
18#define DEBUG_TYPE "regalloc"
19#include "llvm/CodeGen/LiveIntervalAnalysis.h"
20#include "llvm/Value.h"
21#include "llvm/Analysis/AliasAnalysis.h"
22#include "llvm/CodeGen/LiveVariables.h"
23#include "llvm/CodeGen/MachineDominators.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/Passes.h"
27#include "llvm/Target/TargetRegisterInfo.h"
28#include "llvm/Target/TargetInstrInfo.h"
29#include "llvm/Target/TargetMachine.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Debug.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/raw_ostream.h"
34#include "llvm/ADT/DenseSet.h"
35#include "llvm/ADT/Statistic.h"
36#include "llvm/ADT/STLExtras.h"
37#include "LiveRangeCalc.h"
38#include <algorithm>
39#include <limits>
40#include <cmath>
41using namespace llvm;
42
43// Temporary option to enable regunit liveness.
44static cl::opt<bool> LiveRegUnits("live-regunits", cl::Hidden);
45
46STATISTIC(numIntervals , "Number of original intervals");
47
48char LiveIntervals::ID = 0;
49INITIALIZE_PASS_BEGIN(LiveIntervals, "liveintervals",
50                "Live Interval Analysis", false, false)
51INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
52INITIALIZE_PASS_DEPENDENCY(LiveVariables)
53INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
54INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
55INITIALIZE_PASS_END(LiveIntervals, "liveintervals",
56                "Live Interval Analysis", false, false)
57
58void LiveIntervals::getAnalysisUsage(AnalysisUsage &AU) const {
59  AU.setPreservesCFG();
60  AU.addRequired<AliasAnalysis>();
61  AU.addPreserved<AliasAnalysis>();
62  AU.addRequired<LiveVariables>();
63  AU.addPreserved<LiveVariables>();
64  AU.addPreservedID(MachineLoopInfoID);
65  if (LiveRegUnits)
66    AU.addRequiredTransitiveID(MachineDominatorsID);
67  AU.addPreservedID(MachineDominatorsID);
68  AU.addPreserved<SlotIndexes>();
69  AU.addRequiredTransitive<SlotIndexes>();
70  MachineFunctionPass::getAnalysisUsage(AU);
71}
72
73LiveIntervals::LiveIntervals() : MachineFunctionPass(ID),
74  DomTree(0), LRCalc(0) {
75  initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
76}
77
78LiveIntervals::~LiveIntervals() {
79  delete LRCalc;
80}
81
82void LiveIntervals::releaseMemory() {
83  // Free the live intervals themselves.
84  for (DenseMap<unsigned, LiveInterval*>::iterator I = R2IMap.begin(),
85       E = R2IMap.end(); I != E; ++I)
86    delete I->second;
87
88  R2IMap.clear();
89  RegMaskSlots.clear();
90  RegMaskBits.clear();
91  RegMaskBlocks.clear();
92
93  for (unsigned i = 0, e = RegUnitIntervals.size(); i != e; ++i)
94    delete RegUnitIntervals[i];
95  RegUnitIntervals.clear();
96
97  // Release VNInfo memory regions, VNInfo objects don't need to be dtor'd.
98  VNInfoAllocator.Reset();
99}
100
101/// runOnMachineFunction - Register allocate the whole function
102///
103bool LiveIntervals::runOnMachineFunction(MachineFunction &fn) {
104  MF = &fn;
105  MRI = &MF->getRegInfo();
106  TM = &fn.getTarget();
107  TRI = TM->getRegisterInfo();
108  TII = TM->getInstrInfo();
109  AA = &getAnalysis<AliasAnalysis>();
110  LV = &getAnalysis<LiveVariables>();
111  Indexes = &getAnalysis<SlotIndexes>();
112  if (LiveRegUnits)
113    DomTree = &getAnalysis<MachineDominatorTree>();
114  if (LiveRegUnits && !LRCalc)
115    LRCalc = new LiveRangeCalc();
116  AllocatableRegs = TRI->getAllocatableSet(fn);
117  ReservedRegs = TRI->getReservedRegs(fn);
118
119  computeIntervals();
120
121  numIntervals += getNumIntervals();
122
123  if (LiveRegUnits) {
124    computeLiveInRegUnits();
125  }
126
127  DEBUG(dump());
128  return true;
129}
130
131/// print - Implement the dump method.
132void LiveIntervals::print(raw_ostream &OS, const Module* ) const {
133  OS << "********** INTERVALS **********\n";
134
135  // Dump the physregs.
136  for (unsigned Reg = 1, RegE = TRI->getNumRegs(); Reg != RegE; ++Reg)
137    if (const LiveInterval *LI = R2IMap.lookup(Reg))
138      OS << PrintReg(Reg, TRI) << '\t' << *LI << '\n';
139
140  // Dump the regunits.
141  for (unsigned i = 0, e = RegUnitIntervals.size(); i != e; ++i)
142    if (LiveInterval *LI = RegUnitIntervals[i])
143      OS << PrintRegUnit(i, TRI) << " = " << *LI << '\n';
144
145  // Dump the virtregs.
146  for (unsigned Reg = 0, RegE = MRI->getNumVirtRegs(); Reg != RegE; ++Reg)
147    if (const LiveInterval *LI =
148        R2IMap.lookup(TargetRegisterInfo::index2VirtReg(Reg)))
149      OS << PrintReg(LI->reg) << '\t' << *LI << '\n';
150
151  printInstrs(OS);
152}
153
154void LiveIntervals::printInstrs(raw_ostream &OS) const {
155  OS << "********** MACHINEINSTRS **********\n";
156  MF->print(OS, Indexes);
157}
158
159void LiveIntervals::dumpInstrs() const {
160  printInstrs(dbgs());
161}
162
163static
164bool MultipleDefsBySameMI(const MachineInstr &MI, unsigned MOIdx) {
165  unsigned Reg = MI.getOperand(MOIdx).getReg();
166  for (unsigned i = MOIdx+1, e = MI.getNumOperands(); i < e; ++i) {
167    const MachineOperand &MO = MI.getOperand(i);
168    if (!MO.isReg())
169      continue;
170    if (MO.getReg() == Reg && MO.isDef()) {
171      assert(MI.getOperand(MOIdx).getSubReg() != MO.getSubReg() &&
172             MI.getOperand(MOIdx).getSubReg() &&
173             (MO.getSubReg() || MO.isImplicit()));
174      return true;
175    }
176  }
177  return false;
178}
179
180/// isPartialRedef - Return true if the specified def at the specific index is
181/// partially re-defining the specified live interval. A common case of this is
182/// a definition of the sub-register.
183bool LiveIntervals::isPartialRedef(SlotIndex MIIdx, MachineOperand &MO,
184                                   LiveInterval &interval) {
185  if (!MO.getSubReg() || MO.isEarlyClobber())
186    return false;
187
188  SlotIndex RedefIndex = MIIdx.getRegSlot();
189  const LiveRange *OldLR =
190    interval.getLiveRangeContaining(RedefIndex.getRegSlot(true));
191  MachineInstr *DefMI = getInstructionFromIndex(OldLR->valno->def);
192  if (DefMI != 0) {
193    return DefMI->findRegisterDefOperandIdx(interval.reg) != -1;
194  }
195  return false;
196}
197
198void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
199                                             MachineBasicBlock::iterator mi,
200                                             SlotIndex MIIdx,
201                                             MachineOperand& MO,
202                                             unsigned MOIdx,
203                                             LiveInterval &interval) {
204  DEBUG(dbgs() << "\t\tregister: " << PrintReg(interval.reg, TRI));
205
206  // Virtual registers may be defined multiple times (due to phi
207  // elimination and 2-addr elimination).  Much of what we do only has to be
208  // done once for the vreg.  We use an empty interval to detect the first
209  // time we see a vreg.
210  LiveVariables::VarInfo& vi = LV->getVarInfo(interval.reg);
211  if (interval.empty()) {
212    // Get the Idx of the defining instructions.
213    SlotIndex defIndex = MIIdx.getRegSlot(MO.isEarlyClobber());
214
215    // Make sure the first definition is not a partial redefinition.
216    assert(!MO.readsReg() && "First def cannot also read virtual register "
217           "missing <undef> flag?");
218
219    VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator);
220    assert(ValNo->id == 0 && "First value in interval is not 0?");
221
222    // Loop over all of the blocks that the vreg is defined in.  There are
223    // two cases we have to handle here.  The most common case is a vreg
224    // whose lifetime is contained within a basic block.  In this case there
225    // will be a single kill, in MBB, which comes after the definition.
226    if (vi.Kills.size() == 1 && vi.Kills[0]->getParent() == mbb) {
227      // FIXME: what about dead vars?
228      SlotIndex killIdx;
229      if (vi.Kills[0] != mi)
230        killIdx = getInstructionIndex(vi.Kills[0]).getRegSlot();
231      else
232        killIdx = defIndex.getDeadSlot();
233
234      // If the kill happens after the definition, we have an intra-block
235      // live range.
236      if (killIdx > defIndex) {
237        assert(vi.AliveBlocks.empty() &&
238               "Shouldn't be alive across any blocks!");
239        LiveRange LR(defIndex, killIdx, ValNo);
240        interval.addRange(LR);
241        DEBUG(dbgs() << " +" << LR << "\n");
242        return;
243      }
244    }
245
246    // The other case we handle is when a virtual register lives to the end
247    // of the defining block, potentially live across some blocks, then is
248    // live into some number of blocks, but gets killed.  Start by adding a
249    // range that goes from this definition to the end of the defining block.
250    LiveRange NewLR(defIndex, getMBBEndIdx(mbb), ValNo);
251    DEBUG(dbgs() << " +" << NewLR);
252    interval.addRange(NewLR);
253
254    bool PHIJoin = LV->isPHIJoin(interval.reg);
255
256    if (PHIJoin) {
257      // A phi join register is killed at the end of the MBB and revived as a
258      // new valno in the killing blocks.
259      assert(vi.AliveBlocks.empty() && "Phi join can't pass through blocks");
260      DEBUG(dbgs() << " phi-join");
261      ValNo->setHasPHIKill(true);
262    } else {
263      // Iterate over all of the blocks that the variable is completely
264      // live in, adding [insrtIndex(begin), instrIndex(end)+4) to the
265      // live interval.
266      for (SparseBitVector<>::iterator I = vi.AliveBlocks.begin(),
267               E = vi.AliveBlocks.end(); I != E; ++I) {
268        MachineBasicBlock *aliveBlock = MF->getBlockNumbered(*I);
269        LiveRange LR(getMBBStartIdx(aliveBlock), getMBBEndIdx(aliveBlock),
270                     ValNo);
271        interval.addRange(LR);
272        DEBUG(dbgs() << " +" << LR);
273      }
274    }
275
276    // Finally, this virtual register is live from the start of any killing
277    // block to the 'use' slot of the killing instruction.
278    for (unsigned i = 0, e = vi.Kills.size(); i != e; ++i) {
279      MachineInstr *Kill = vi.Kills[i];
280      SlotIndex Start = getMBBStartIdx(Kill->getParent());
281      SlotIndex killIdx = getInstructionIndex(Kill).getRegSlot();
282
283      // Create interval with one of a NEW value number.  Note that this value
284      // number isn't actually defined by an instruction, weird huh? :)
285      if (PHIJoin) {
286        assert(getInstructionFromIndex(Start) == 0 &&
287               "PHI def index points at actual instruction.");
288        ValNo = interval.getNextValue(Start, VNInfoAllocator);
289        ValNo->setIsPHIDef(true);
290      }
291      LiveRange LR(Start, killIdx, ValNo);
292      interval.addRange(LR);
293      DEBUG(dbgs() << " +" << LR);
294    }
295
296  } else {
297    if (MultipleDefsBySameMI(*mi, MOIdx))
298      // Multiple defs of the same virtual register by the same instruction.
299      // e.g. %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ...
300      // This is likely due to elimination of REG_SEQUENCE instructions. Return
301      // here since there is nothing to do.
302      return;
303
304    // If this is the second time we see a virtual register definition, it
305    // must be due to phi elimination or two addr elimination.  If this is
306    // the result of two address elimination, then the vreg is one of the
307    // def-and-use register operand.
308
309    // It may also be partial redef like this:
310    // 80  %reg1041:6<def> = VSHRNv4i16 %reg1034<kill>, 12, pred:14, pred:%reg0
311    // 120 %reg1041:5<def> = VSHRNv4i16 %reg1039<kill>, 12, pred:14, pred:%reg0
312    bool PartReDef = isPartialRedef(MIIdx, MO, interval);
313    if (PartReDef || mi->isRegTiedToUseOperand(MOIdx)) {
314      // If this is a two-address definition, then we have already processed
315      // the live range.  The only problem is that we didn't realize there
316      // are actually two values in the live interval.  Because of this we
317      // need to take the LiveRegion that defines this register and split it
318      // into two values.
319      SlotIndex RedefIndex = MIIdx.getRegSlot(MO.isEarlyClobber());
320
321      const LiveRange *OldLR =
322        interval.getLiveRangeContaining(RedefIndex.getRegSlot(true));
323      VNInfo *OldValNo = OldLR->valno;
324      SlotIndex DefIndex = OldValNo->def.getRegSlot();
325
326      // Delete the previous value, which should be short and continuous,
327      // because the 2-addr copy must be in the same MBB as the redef.
328      interval.removeRange(DefIndex, RedefIndex);
329
330      // The new value number (#1) is defined by the instruction we claimed
331      // defined value #0.
332      VNInfo *ValNo = interval.createValueCopy(OldValNo, VNInfoAllocator);
333
334      // Value#0 is now defined by the 2-addr instruction.
335      OldValNo->def = RedefIndex;
336
337      // Add the new live interval which replaces the range for the input copy.
338      LiveRange LR(DefIndex, RedefIndex, ValNo);
339      DEBUG(dbgs() << " replace range with " << LR);
340      interval.addRange(LR);
341
342      // If this redefinition is dead, we need to add a dummy unit live
343      // range covering the def slot.
344      if (MO.isDead())
345        interval.addRange(LiveRange(RedefIndex, RedefIndex.getDeadSlot(),
346                                    OldValNo));
347
348      DEBUG(dbgs() << " RESULT: " << interval);
349    } else if (LV->isPHIJoin(interval.reg)) {
350      // In the case of PHI elimination, each variable definition is only
351      // live until the end of the block.  We've already taken care of the
352      // rest of the live range.
353
354      SlotIndex defIndex = MIIdx.getRegSlot();
355      if (MO.isEarlyClobber())
356        defIndex = MIIdx.getRegSlot(true);
357
358      VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator);
359
360      SlotIndex killIndex = getMBBEndIdx(mbb);
361      LiveRange LR(defIndex, killIndex, ValNo);
362      interval.addRange(LR);
363      ValNo->setHasPHIKill(true);
364      DEBUG(dbgs() << " phi-join +" << LR);
365    } else {
366      llvm_unreachable("Multiply defined register");
367    }
368  }
369
370  DEBUG(dbgs() << '\n');
371}
372
373static bool isRegLiveIntoSuccessor(const MachineBasicBlock *MBB, unsigned Reg) {
374  for (MachineBasicBlock::const_succ_iterator SI = MBB->succ_begin(),
375                                              SE = MBB->succ_end();
376       SI != SE; ++SI) {
377    const MachineBasicBlock* succ = *SI;
378    if (succ->isLiveIn(Reg))
379      return true;
380  }
381  return false;
382}
383
384void LiveIntervals::handlePhysicalRegisterDef(MachineBasicBlock *MBB,
385                                              MachineBasicBlock::iterator mi,
386                                              SlotIndex MIIdx,
387                                              MachineOperand& MO,
388                                              LiveInterval &interval) {
389  DEBUG(dbgs() << "\t\tregister: " << PrintReg(interval.reg, TRI));
390
391  SlotIndex baseIndex = MIIdx;
392  SlotIndex start = baseIndex.getRegSlot(MO.isEarlyClobber());
393  SlotIndex end = start;
394
395  // If it is not used after definition, it is considered dead at
396  // the instruction defining it. Hence its interval is:
397  // [defSlot(def), defSlot(def)+1)
398  // For earlyclobbers, the defSlot was pushed back one; the extra
399  // advance below compensates.
400  if (MO.isDead()) {
401    DEBUG(dbgs() << " dead");
402    end = start.getDeadSlot();
403    goto exit;
404  }
405
406  // If it is not dead on definition, it must be killed by a
407  // subsequent instruction. Hence its interval is:
408  // [defSlot(def), useSlot(kill)+1)
409  baseIndex = baseIndex.getNextIndex();
410  while (++mi != MBB->end()) {
411
412    if (mi->isDebugValue())
413      continue;
414    if (getInstructionFromIndex(baseIndex) == 0)
415      baseIndex = Indexes->getNextNonNullIndex(baseIndex);
416
417    if (mi->killsRegister(interval.reg, TRI)) {
418      DEBUG(dbgs() << " killed");
419      end = baseIndex.getRegSlot();
420      goto exit;
421    } else {
422      int DefIdx = mi->findRegisterDefOperandIdx(interval.reg,false,false,TRI);
423      if (DefIdx != -1) {
424        if (mi->isRegTiedToUseOperand(DefIdx)) {
425          // Two-address instruction.
426          end = baseIndex.getRegSlot(mi->getOperand(DefIdx).isEarlyClobber());
427        } else {
428          // Another instruction redefines the register before it is ever read.
429          // Then the register is essentially dead at the instruction that
430          // defines it. Hence its interval is:
431          // [defSlot(def), defSlot(def)+1)
432          DEBUG(dbgs() << " dead");
433          end = start.getDeadSlot();
434        }
435        goto exit;
436      }
437    }
438
439    baseIndex = baseIndex.getNextIndex();
440  }
441
442  // If we get here the register *should* be live out.
443  assert(!isAllocatable(interval.reg) && "Physregs shouldn't be live out!");
444
445  // FIXME: We need saner rules for reserved regs.
446  if (isReserved(interval.reg)) {
447    end = start.getDeadSlot();
448  } else {
449    // Unreserved, unallocable registers like EFLAGS can be live across basic
450    // block boundaries.
451    assert(isRegLiveIntoSuccessor(MBB, interval.reg) &&
452           "Unreserved reg not live-out?");
453    end = getMBBEndIdx(MBB);
454  }
455exit:
456  assert(start < end && "did not find end of interval?");
457
458  // Already exists? Extend old live interval.
459  VNInfo *ValNo = interval.getVNInfoAt(start);
460  bool Extend = ValNo != 0;
461  if (!Extend)
462    ValNo = interval.getNextValue(start, VNInfoAllocator);
463  LiveRange LR(start, end, ValNo);
464  interval.addRange(LR);
465  DEBUG(dbgs() << " +" << LR << '\n');
466}
467
468void LiveIntervals::handleRegisterDef(MachineBasicBlock *MBB,
469                                      MachineBasicBlock::iterator MI,
470                                      SlotIndex MIIdx,
471                                      MachineOperand& MO,
472                                      unsigned MOIdx) {
473  if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
474    handleVirtualRegisterDef(MBB, MI, MIIdx, MO, MOIdx,
475                             getOrCreateInterval(MO.getReg()));
476  else
477    handlePhysicalRegisterDef(MBB, MI, MIIdx, MO,
478                              getOrCreateInterval(MO.getReg()));
479}
480
481void LiveIntervals::handleLiveInRegister(MachineBasicBlock *MBB,
482                                         SlotIndex MIIdx,
483                                         LiveInterval &interval) {
484  assert(TargetRegisterInfo::isPhysicalRegister(interval.reg) &&
485         "Only physical registers can be live in.");
486  assert((!isAllocatable(interval.reg) || MBB->getParent()->begin() ||
487          MBB->isLandingPad()) &&
488          "Allocatable live-ins only valid for entry blocks and landing pads.");
489
490  DEBUG(dbgs() << "\t\tlivein register: " << PrintReg(interval.reg, TRI));
491
492  // Look for kills, if it reaches a def before it's killed, then it shouldn't
493  // be considered a livein.
494  MachineBasicBlock::iterator mi = MBB->begin();
495  MachineBasicBlock::iterator E = MBB->end();
496  // Skip over DBG_VALUE at the start of the MBB.
497  if (mi != E && mi->isDebugValue()) {
498    while (++mi != E && mi->isDebugValue())
499      ;
500    if (mi == E)
501      // MBB is empty except for DBG_VALUE's.
502      return;
503  }
504
505  SlotIndex baseIndex = MIIdx;
506  SlotIndex start = baseIndex;
507  if (getInstructionFromIndex(baseIndex) == 0)
508    baseIndex = Indexes->getNextNonNullIndex(baseIndex);
509
510  SlotIndex end = baseIndex;
511  bool SeenDefUse = false;
512
513  while (mi != E) {
514    if (mi->killsRegister(interval.reg, TRI)) {
515      DEBUG(dbgs() << " killed");
516      end = baseIndex.getRegSlot();
517      SeenDefUse = true;
518      break;
519    } else if (mi->modifiesRegister(interval.reg, TRI)) {
520      // Another instruction redefines the register before it is ever read.
521      // Then the register is essentially dead at the instruction that defines
522      // it. Hence its interval is:
523      // [defSlot(def), defSlot(def)+1)
524      DEBUG(dbgs() << " dead");
525      end = start.getDeadSlot();
526      SeenDefUse = true;
527      break;
528    }
529
530    while (++mi != E && mi->isDebugValue())
531      // Skip over DBG_VALUE.
532      ;
533    if (mi != E)
534      baseIndex = Indexes->getNextNonNullIndex(baseIndex);
535  }
536
537  // Live-in register might not be used at all.
538  if (!SeenDefUse) {
539    if (isAllocatable(interval.reg) ||
540        !isRegLiveIntoSuccessor(MBB, interval.reg)) {
541      // Allocatable registers are never live through.
542      // Non-allocatable registers that aren't live into any successors also
543      // aren't live through.
544      DEBUG(dbgs() << " dead");
545      return;
546    } else {
547      // If we get here the register is non-allocatable and live into some
548      // successor. We'll conservatively assume it's live-through.
549      DEBUG(dbgs() << " live through");
550      end = getMBBEndIdx(MBB);
551    }
552  }
553
554  SlotIndex defIdx = getMBBStartIdx(MBB);
555  assert(getInstructionFromIndex(defIdx) == 0 &&
556         "PHI def index points at actual instruction.");
557  VNInfo *vni = interval.getNextValue(defIdx, VNInfoAllocator);
558  vni->setIsPHIDef(true);
559  LiveRange LR(start, end, vni);
560
561  interval.addRange(LR);
562  DEBUG(dbgs() << " +" << LR << '\n');
563}
564
565/// computeIntervals - computes the live intervals for virtual
566/// registers. for some ordering of the machine instructions [1,N] a
567/// live interval is an interval [i, j) where 1 <= i <= j < N for
568/// which a variable is live
569void LiveIntervals::computeIntervals() {
570  DEBUG(dbgs() << "********** COMPUTING LIVE INTERVALS **********\n"
571               << "********** Function: "
572               << ((Value*)MF->getFunction())->getName() << '\n');
573
574  RegMaskBlocks.resize(MF->getNumBlockIDs());
575
576  SmallVector<unsigned, 8> UndefUses;
577  for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end();
578       MBBI != E; ++MBBI) {
579    MachineBasicBlock *MBB = MBBI;
580    RegMaskBlocks[MBB->getNumber()].first = RegMaskSlots.size();
581
582    if (MBB->empty())
583      continue;
584
585    // Track the index of the current machine instr.
586    SlotIndex MIIndex = getMBBStartIdx(MBB);
587    DEBUG(dbgs() << "BB#" << MBB->getNumber()
588          << ":\t\t# derived from " << MBB->getName() << "\n");
589
590    // Create intervals for live-ins to this BB first.
591    for (MachineBasicBlock::livein_iterator LI = MBB->livein_begin(),
592           LE = MBB->livein_end(); LI != LE; ++LI) {
593      handleLiveInRegister(MBB, MIIndex, getOrCreateInterval(*LI));
594    }
595
596    // Skip over empty initial indices.
597    if (getInstructionFromIndex(MIIndex) == 0)
598      MIIndex = Indexes->getNextNonNullIndex(MIIndex);
599
600    for (MachineBasicBlock::iterator MI = MBB->begin(), miEnd = MBB->end();
601         MI != miEnd; ++MI) {
602      DEBUG(dbgs() << MIIndex << "\t" << *MI);
603      if (MI->isDebugValue())
604        continue;
605      assert(Indexes->getInstructionFromIndex(MIIndex) == MI &&
606             "Lost SlotIndex synchronization");
607
608      // Handle defs.
609      for (int i = MI->getNumOperands() - 1; i >= 0; --i) {
610        MachineOperand &MO = MI->getOperand(i);
611
612        // Collect register masks.
613        if (MO.isRegMask()) {
614          RegMaskSlots.push_back(MIIndex.getRegSlot());
615          RegMaskBits.push_back(MO.getRegMask());
616          continue;
617        }
618
619        if (!MO.isReg() || !MO.getReg())
620          continue;
621
622        // handle register defs - build intervals
623        if (MO.isDef())
624          handleRegisterDef(MBB, MI, MIIndex, MO, i);
625        else if (MO.isUndef())
626          UndefUses.push_back(MO.getReg());
627      }
628
629      // Move to the next instr slot.
630      MIIndex = Indexes->getNextNonNullIndex(MIIndex);
631    }
632
633    // Compute the number of register mask instructions in this block.
634    std::pair<unsigned, unsigned> &RMB = RegMaskBlocks[MBB->getNumber()];
635    RMB.second = RegMaskSlots.size() - RMB.first;;
636  }
637
638  // Create empty intervals for registers defined by implicit_def's (except
639  // for those implicit_def that define values which are liveout of their
640  // blocks.
641  for (unsigned i = 0, e = UndefUses.size(); i != e; ++i) {
642    unsigned UndefReg = UndefUses[i];
643    (void)getOrCreateInterval(UndefReg);
644  }
645}
646
647LiveInterval* LiveIntervals::createInterval(unsigned reg) {
648  float Weight = TargetRegisterInfo::isPhysicalRegister(reg) ? HUGE_VALF : 0.0F;
649  return new LiveInterval(reg, Weight);
650}
651
652
653//===----------------------------------------------------------------------===//
654//                           Register Unit Liveness
655//===----------------------------------------------------------------------===//
656//
657// Fixed interference typically comes from ABI boundaries: Function arguments
658// and return values are passed in fixed registers, and so are exception
659// pointers entering landing pads. Certain instructions require values to be
660// present in specific registers. That is also represented through fixed
661// interference.
662//
663
664/// computeRegUnitInterval - Compute the live interval of a register unit, based
665/// on the uses and defs of aliasing registers.  The interval should be empty,
666/// or contain only dead phi-defs from ABI blocks.
667void LiveIntervals::computeRegUnitInterval(LiveInterval *LI) {
668  unsigned Unit = LI->reg;
669
670  assert(LRCalc && "LRCalc not initialized.");
671  LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
672
673  // The physregs aliasing Unit are the roots and their super-registers.
674  // Create all values as dead defs before extending to uses. Note that roots
675  // may share super-registers. That's OK because createDeadDefs() is
676  // idempotent. It is very rare for a register unit to have multiple roots, so
677  // uniquing super-registers is probably not worthwhile.
678  for (MCRegUnitRootIterator Roots(Unit, TRI); Roots.isValid(); ++Roots) {
679    unsigned Root = *Roots;
680    if (!MRI->reg_empty(Root))
681      LRCalc->createDeadDefs(LI, Root);
682    for (MCSuperRegIterator Supers(Root, TRI); Supers.isValid(); ++Supers) {
683      if (!MRI->reg_empty(*Supers))
684        LRCalc->createDeadDefs(LI, *Supers);
685    }
686  }
687
688  // Now extend LI to reach all uses.
689  // Ignore uses of reserved registers. We only track defs of those.
690  for (MCRegUnitRootIterator Roots(Unit, TRI); Roots.isValid(); ++Roots) {
691    unsigned Root = *Roots;
692    if (!isReserved(Root) && !MRI->reg_empty(Root))
693      LRCalc->extendToUses(LI, Root);
694    for (MCSuperRegIterator Supers(Root, TRI); Supers.isValid(); ++Supers) {
695      unsigned Reg = *Supers;
696      if (!isReserved(Reg) && !MRI->reg_empty(Reg))
697        LRCalc->extendToUses(LI, Reg);
698    }
699  }
700}
701
702
703/// computeLiveInRegUnits - Precompute the live ranges of any register units
704/// that are live-in to an ABI block somewhere. Register values can appear
705/// without a corresponding def when entering the entry block or a landing pad.
706///
707void LiveIntervals::computeLiveInRegUnits() {
708  RegUnitIntervals.resize(TRI->getNumRegUnits());
709  DEBUG(dbgs() << "Computing live-in reg-units in ABI blocks.\n");
710
711  // Keep track of the intervals allocated.
712  SmallVector<LiveInterval*, 8> NewIntvs;
713
714  // Check all basic blocks for live-ins.
715  for (MachineFunction::const_iterator MFI = MF->begin(), MFE = MF->end();
716       MFI != MFE; ++MFI) {
717    const MachineBasicBlock *MBB = MFI;
718
719    // We only care about ABI blocks: Entry + landing pads.
720    if ((MFI != MF->begin() && !MBB->isLandingPad()) || MBB->livein_empty())
721      continue;
722
723    // Create phi-defs at Begin for all live-in registers.
724    SlotIndex Begin = Indexes->getMBBStartIdx(MBB);
725    DEBUG(dbgs() << Begin << "\tBB#" << MBB->getNumber());
726    for (MachineBasicBlock::livein_iterator LII = MBB->livein_begin(),
727         LIE = MBB->livein_end(); LII != LIE; ++LII) {
728      for (MCRegUnitIterator Units(*LII, TRI); Units.isValid(); ++Units) {
729        unsigned Unit = *Units;
730        LiveInterval *Intv = RegUnitIntervals[Unit];
731        if (!Intv) {
732          Intv = RegUnitIntervals[Unit] = new LiveInterval(Unit, HUGE_VALF);
733          NewIntvs.push_back(Intv);
734        }
735        VNInfo *VNI = Intv->createDeadDef(Begin, getVNInfoAllocator());
736        (void)VNI;
737        DEBUG(dbgs() << ' ' << PrintRegUnit(Unit, TRI) << '#' << VNI->id);
738      }
739    }
740    DEBUG(dbgs() << '\n');
741  }
742  DEBUG(dbgs() << "Created " << NewIntvs.size() << " new intervals.\n");
743
744  // Compute the 'normal' part of the intervals.
745  for (unsigned i = 0, e = NewIntvs.size(); i != e; ++i)
746    computeRegUnitInterval(NewIntvs[i]);
747}
748
749
750/// shrinkToUses - After removing some uses of a register, shrink its live
751/// range to just the remaining uses. This method does not compute reaching
752/// defs for new uses, and it doesn't remove dead defs.
753bool LiveIntervals::shrinkToUses(LiveInterval *li,
754                                 SmallVectorImpl<MachineInstr*> *dead) {
755  DEBUG(dbgs() << "Shrink: " << *li << '\n');
756  assert(TargetRegisterInfo::isVirtualRegister(li->reg)
757         && "Can only shrink virtual registers");
758  // Find all the values used, including PHI kills.
759  SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList;
760
761  // Blocks that have already been added to WorkList as live-out.
762  SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
763
764  // Visit all instructions reading li->reg.
765  for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(li->reg);
766       MachineInstr *UseMI = I.skipInstruction();) {
767    if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg))
768      continue;
769    SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
770    LiveRangeQuery LRQ(*li, Idx);
771    VNInfo *VNI = LRQ.valueIn();
772    if (!VNI) {
773      // This shouldn't happen: readsVirtualRegister returns true, but there is
774      // no live value. It is likely caused by a target getting <undef> flags
775      // wrong.
776      DEBUG(dbgs() << Idx << '\t' << *UseMI
777                   << "Warning: Instr claims to read non-existent value in "
778                    << *li << '\n');
779      continue;
780    }
781    // Special case: An early-clobber tied operand reads and writes the
782    // register one slot early.
783    if (VNInfo *DefVNI = LRQ.valueDefined())
784      Idx = DefVNI->def;
785
786    WorkList.push_back(std::make_pair(Idx, VNI));
787  }
788
789  // Create a new live interval with only minimal live segments per def.
790  LiveInterval NewLI(li->reg, 0);
791  for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
792       I != E; ++I) {
793    VNInfo *VNI = *I;
794    if (VNI->isUnused())
795      continue;
796    NewLI.addRange(LiveRange(VNI->def, VNI->def.getDeadSlot(), VNI));
797  }
798
799  // Keep track of the PHIs that are in use.
800  SmallPtrSet<VNInfo*, 8> UsedPHIs;
801
802  // Extend intervals to reach all uses in WorkList.
803  while (!WorkList.empty()) {
804    SlotIndex Idx = WorkList.back().first;
805    VNInfo *VNI = WorkList.back().second;
806    WorkList.pop_back();
807    const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot());
808    SlotIndex BlockStart = getMBBStartIdx(MBB);
809
810    // Extend the live range for VNI to be live at Idx.
811    if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) {
812      (void)ExtVNI;
813      assert(ExtVNI == VNI && "Unexpected existing value number");
814      // Is this a PHIDef we haven't seen before?
815      if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI))
816        continue;
817      // The PHI is live, make sure the predecessors are live-out.
818      for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
819           PE = MBB->pred_end(); PI != PE; ++PI) {
820        if (!LiveOut.insert(*PI))
821          continue;
822        SlotIndex Stop = getMBBEndIdx(*PI);
823        // A predecessor is not required to have a live-out value for a PHI.
824        if (VNInfo *PVNI = li->getVNInfoBefore(Stop))
825          WorkList.push_back(std::make_pair(Stop, PVNI));
826      }
827      continue;
828    }
829
830    // VNI is live-in to MBB.
831    DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
832    NewLI.addRange(LiveRange(BlockStart, Idx, VNI));
833
834    // Make sure VNI is live-out from the predecessors.
835    for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
836         PE = MBB->pred_end(); PI != PE; ++PI) {
837      if (!LiveOut.insert(*PI))
838        continue;
839      SlotIndex Stop = getMBBEndIdx(*PI);
840      assert(li->getVNInfoBefore(Stop) == VNI &&
841             "Wrong value out of predecessor");
842      WorkList.push_back(std::make_pair(Stop, VNI));
843    }
844  }
845
846  // Handle dead values.
847  bool CanSeparate = false;
848  for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
849       I != E; ++I) {
850    VNInfo *VNI = *I;
851    if (VNI->isUnused())
852      continue;
853    LiveInterval::iterator LII = NewLI.FindLiveRangeContaining(VNI->def);
854    assert(LII != NewLI.end() && "Missing live range for PHI");
855    if (LII->end != VNI->def.getDeadSlot())
856      continue;
857    if (VNI->isPHIDef()) {
858      // This is a dead PHI. Remove it.
859      VNI->setIsUnused(true);
860      NewLI.removeRange(*LII);
861      DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n");
862      CanSeparate = true;
863    } else {
864      // This is a dead def. Make sure the instruction knows.
865      MachineInstr *MI = getInstructionFromIndex(VNI->def);
866      assert(MI && "No instruction defining live value");
867      MI->addRegisterDead(li->reg, TRI);
868      if (dead && MI->allDefsAreDead()) {
869        DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI);
870        dead->push_back(MI);
871      }
872    }
873  }
874
875  // Move the trimmed ranges back.
876  li->ranges.swap(NewLI.ranges);
877  DEBUG(dbgs() << "Shrunk: " << *li << '\n');
878  return CanSeparate;
879}
880
881
882//===----------------------------------------------------------------------===//
883// Register allocator hooks.
884//
885
886void LiveIntervals::addKillFlags() {
887  for (iterator I = begin(), E = end(); I != E; ++I) {
888    unsigned Reg = I->first;
889    if (TargetRegisterInfo::isPhysicalRegister(Reg))
890      continue;
891    if (MRI->reg_nodbg_empty(Reg))
892      continue;
893    LiveInterval *LI = I->second;
894
895    // Every instruction that kills Reg corresponds to a live range end point.
896    for (LiveInterval::iterator RI = LI->begin(), RE = LI->end(); RI != RE;
897         ++RI) {
898      // A block index indicates an MBB edge.
899      if (RI->end.isBlock())
900        continue;
901      MachineInstr *MI = getInstructionFromIndex(RI->end);
902      if (!MI)
903        continue;
904      MI->addRegisterKilled(Reg, NULL);
905    }
906  }
907}
908
909MachineBasicBlock*
910LiveIntervals::intervalIsInOneMBB(const LiveInterval &LI) const {
911  // A local live range must be fully contained inside the block, meaning it is
912  // defined and killed at instructions, not at block boundaries. It is not
913  // live in or or out of any block.
914  //
915  // It is technically possible to have a PHI-defined live range identical to a
916  // single block, but we are going to return false in that case.
917
918  SlotIndex Start = LI.beginIndex();
919  if (Start.isBlock())
920    return NULL;
921
922  SlotIndex Stop = LI.endIndex();
923  if (Stop.isBlock())
924    return NULL;
925
926  // getMBBFromIndex doesn't need to search the MBB table when both indexes
927  // belong to proper instructions.
928  MachineBasicBlock *MBB1 = Indexes->getMBBFromIndex(Start);
929  MachineBasicBlock *MBB2 = Indexes->getMBBFromIndex(Stop);
930  return MBB1 == MBB2 ? MBB1 : NULL;
931}
932
933float
934LiveIntervals::getSpillWeight(bool isDef, bool isUse, unsigned loopDepth) {
935  // Limit the loop depth ridiculousness.
936  if (loopDepth > 200)
937    loopDepth = 200;
938
939  // The loop depth is used to roughly estimate the number of times the
940  // instruction is executed. Something like 10^d is simple, but will quickly
941  // overflow a float. This expression behaves like 10^d for small d, but is
942  // more tempered for large d. At d=200 we get 6.7e33 which leaves a bit of
943  // headroom before overflow.
944  // By the way, powf() might be unavailable here. For consistency,
945  // We may take pow(double,double).
946  float lc = std::pow(1 + (100.0 / (loopDepth + 10)), (double)loopDepth);
947
948  return (isDef + isUse) * lc;
949}
950
951LiveRange LiveIntervals::addLiveRangeToEndOfBlock(unsigned reg,
952                                                  MachineInstr* startInst) {
953  LiveInterval& Interval = getOrCreateInterval(reg);
954  VNInfo* VN = Interval.getNextValue(
955    SlotIndex(getInstructionIndex(startInst).getRegSlot()),
956    getVNInfoAllocator());
957  VN->setHasPHIKill(true);
958  LiveRange LR(
959     SlotIndex(getInstructionIndex(startInst).getRegSlot()),
960     getMBBEndIdx(startInst->getParent()), VN);
961  Interval.addRange(LR);
962
963  return LR;
964}
965
966
967//===----------------------------------------------------------------------===//
968//                          Register mask functions
969//===----------------------------------------------------------------------===//
970
971bool LiveIntervals::checkRegMaskInterference(LiveInterval &LI,
972                                             BitVector &UsableRegs) {
973  if (LI.empty())
974    return false;
975  LiveInterval::iterator LiveI = LI.begin(), LiveE = LI.end();
976
977  // Use a smaller arrays for local live ranges.
978  ArrayRef<SlotIndex> Slots;
979  ArrayRef<const uint32_t*> Bits;
980  if (MachineBasicBlock *MBB = intervalIsInOneMBB(LI)) {
981    Slots = getRegMaskSlotsInBlock(MBB->getNumber());
982    Bits = getRegMaskBitsInBlock(MBB->getNumber());
983  } else {
984    Slots = getRegMaskSlots();
985    Bits = getRegMaskBits();
986  }
987
988  // We are going to enumerate all the register mask slots contained in LI.
989  // Start with a binary search of RegMaskSlots to find a starting point.
990  ArrayRef<SlotIndex>::iterator SlotI =
991    std::lower_bound(Slots.begin(), Slots.end(), LiveI->start);
992  ArrayRef<SlotIndex>::iterator SlotE = Slots.end();
993
994  // No slots in range, LI begins after the last call.
995  if (SlotI == SlotE)
996    return false;
997
998  bool Found = false;
999  for (;;) {
1000    assert(*SlotI >= LiveI->start);
1001    // Loop over all slots overlapping this segment.
1002    while (*SlotI < LiveI->end) {
1003      // *SlotI overlaps LI. Collect mask bits.
1004      if (!Found) {
1005        // This is the first overlap. Initialize UsableRegs to all ones.
1006        UsableRegs.clear();
1007        UsableRegs.resize(TRI->getNumRegs(), true);
1008        Found = true;
1009      }
1010      // Remove usable registers clobbered by this mask.
1011      UsableRegs.clearBitsNotInMask(Bits[SlotI-Slots.begin()]);
1012      if (++SlotI == SlotE)
1013        return Found;
1014    }
1015    // *SlotI is beyond the current LI segment.
1016    LiveI = LI.advanceTo(LiveI, *SlotI);
1017    if (LiveI == LiveE)
1018      return Found;
1019    // Advance SlotI until it overlaps.
1020    while (*SlotI < LiveI->start)
1021      if (++SlotI == SlotE)
1022        return Found;
1023  }
1024}
1025
1026//===----------------------------------------------------------------------===//
1027//                         IntervalUpdate class.
1028//===----------------------------------------------------------------------===//
1029
1030// HMEditor is a toolkit used by handleMove to trim or extend live intervals.
1031class LiveIntervals::HMEditor {
1032private:
1033  LiveIntervals& LIS;
1034  const MachineRegisterInfo& MRI;
1035  const TargetRegisterInfo& TRI;
1036  SlotIndex NewIdx;
1037
1038  typedef std::pair<LiveInterval*, LiveRange*> IntRangePair;
1039  typedef DenseSet<IntRangePair> RangeSet;
1040
1041  struct RegRanges {
1042    LiveRange* Use;
1043    LiveRange* EC;
1044    LiveRange* Dead;
1045    LiveRange* Def;
1046    RegRanges() : Use(0), EC(0), Dead(0), Def(0) {}
1047  };
1048  typedef DenseMap<unsigned, RegRanges> BundleRanges;
1049
1050public:
1051  HMEditor(LiveIntervals& LIS, const MachineRegisterInfo& MRI,
1052           const TargetRegisterInfo& TRI, SlotIndex NewIdx)
1053    : LIS(LIS), MRI(MRI), TRI(TRI), NewIdx(NewIdx) {}
1054
1055  // Update intervals for all operands of MI from OldIdx to NewIdx.
1056  // This assumes that MI used to be at OldIdx, and now resides at
1057  // NewIdx.
1058  void moveAllRangesFrom(MachineInstr* MI, SlotIndex OldIdx) {
1059    assert(NewIdx != OldIdx && "No-op move? That's a bit strange.");
1060
1061    // Collect the operands.
1062    RangeSet Entering, Internal, Exiting;
1063    bool hasRegMaskOp = false;
1064    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
1065
1066    // To keep the LiveRanges valid within an interval, move the ranges closest
1067    // to the destination first. This prevents ranges from overlapping, to that
1068    // APIs like removeRange still work.
1069    if (NewIdx < OldIdx) {
1070      moveAllEnteringFrom(OldIdx, Entering);
1071      moveAllInternalFrom(OldIdx, Internal);
1072      moveAllExitingFrom(OldIdx, Exiting);
1073    }
1074    else {
1075      moveAllExitingFrom(OldIdx, Exiting);
1076      moveAllInternalFrom(OldIdx, Internal);
1077      moveAllEnteringFrom(OldIdx, Entering);
1078    }
1079
1080    if (hasRegMaskOp)
1081      updateRegMaskSlots(OldIdx);
1082
1083#ifndef NDEBUG
1084    LIValidator validator;
1085    validator = std::for_each(Entering.begin(), Entering.end(), validator);
1086    validator = std::for_each(Internal.begin(), Internal.end(), validator);
1087    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
1088    assert(validator.rangesOk() && "moveAllOperandsFrom broke liveness.");
1089#endif
1090
1091  }
1092
1093  // Update intervals for all operands of MI to refer to BundleStart's
1094  // SlotIndex.
1095  void moveAllRangesInto(MachineInstr* MI, MachineInstr* BundleStart) {
1096    if (MI == BundleStart)
1097      return; // Bundling instr with itself - nothing to do.
1098
1099    SlotIndex OldIdx = LIS.getSlotIndexes()->getInstructionIndex(MI);
1100    assert(LIS.getSlotIndexes()->getInstructionFromIndex(OldIdx) == MI &&
1101           "SlotIndex <-> Instruction mapping broken for MI");
1102
1103    // Collect all ranges already in the bundle.
1104    MachineBasicBlock::instr_iterator BII(BundleStart);
1105    RangeSet Entering, Internal, Exiting;
1106    bool hasRegMaskOp = false;
1107    collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
1108    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
1109    for (++BII; &*BII == MI || BII->isInsideBundle(); ++BII) {
1110      if (&*BII == MI)
1111        continue;
1112      collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
1113      assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
1114    }
1115
1116    BundleRanges BR = createBundleRanges(Entering, Internal, Exiting);
1117
1118    Entering.clear();
1119    Internal.clear();
1120    Exiting.clear();
1121    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
1122    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
1123
1124    DEBUG(dbgs() << "Entering: " << Entering.size() << "\n");
1125    DEBUG(dbgs() << "Internal: " << Internal.size() << "\n");
1126    DEBUG(dbgs() << "Exiting: " << Exiting.size() << "\n");
1127
1128    moveAllEnteringFromInto(OldIdx, Entering, BR);
1129    moveAllInternalFromInto(OldIdx, Internal, BR);
1130    moveAllExitingFromInto(OldIdx, Exiting, BR);
1131
1132
1133#ifndef NDEBUG
1134    LIValidator validator;
1135    validator = std::for_each(Entering.begin(), Entering.end(), validator);
1136    validator = std::for_each(Internal.begin(), Internal.end(), validator);
1137    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
1138    assert(validator.rangesOk() && "moveAllOperandsInto broke liveness.");
1139#endif
1140  }
1141
1142private:
1143
1144#ifndef NDEBUG
1145  class LIValidator {
1146  private:
1147    DenseSet<const LiveInterval*> Checked, Bogus;
1148  public:
1149    void operator()(const IntRangePair& P) {
1150      const LiveInterval* LI = P.first;
1151      if (Checked.count(LI))
1152        return;
1153      Checked.insert(LI);
1154      if (LI->empty())
1155        return;
1156      SlotIndex LastEnd = LI->begin()->start;
1157      for (LiveInterval::const_iterator LRI = LI->begin(), LRE = LI->end();
1158           LRI != LRE; ++LRI) {
1159        const LiveRange& LR = *LRI;
1160        if (LastEnd > LR.start || LR.start >= LR.end)
1161          Bogus.insert(LI);
1162        LastEnd = LR.end;
1163      }
1164    }
1165
1166    bool rangesOk() const {
1167      return Bogus.empty();
1168    }
1169  };
1170#endif
1171
1172  // Collect IntRangePairs for all operands of MI that may need fixing.
1173  // Treat's MI's index as OldIdx (regardless of what it is in SlotIndexes'
1174  // maps).
1175  void collectRanges(MachineInstr* MI, RangeSet& Entering, RangeSet& Internal,
1176                     RangeSet& Exiting, bool& hasRegMaskOp, SlotIndex OldIdx) {
1177    hasRegMaskOp = false;
1178    for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
1179                                    MOE = MI->operands_end();
1180         MOI != MOE; ++MOI) {
1181      const MachineOperand& MO = *MOI;
1182
1183      if (MO.isRegMask()) {
1184        hasRegMaskOp = true;
1185        continue;
1186      }
1187
1188      if (!MO.isReg() || MO.getReg() == 0)
1189        continue;
1190
1191      unsigned Reg = MO.getReg();
1192
1193      // TODO: Currently we're skipping uses that are reserved or have no
1194      // interval, but we're not updating their kills. This should be
1195      // fixed.
1196      if (TargetRegisterInfo::isPhysicalRegister(Reg) && LIS.isReserved(Reg))
1197        continue;
1198
1199      if (TargetRegisterInfo::isPhysicalRegister(Reg) && LIS.trackingRegUnits())
1200          for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
1201            collectRanges(MO, &LIS.getRegUnit(*Units),
1202                          Entering, Internal, Exiting, OldIdx);
1203      else if (LIS.hasInterval(Reg))
1204        collectRanges(MO, &LIS.getInterval(Reg),
1205                      Entering, Internal, Exiting, OldIdx);
1206    }
1207  }
1208
1209  void collectRanges(const MachineOperand &MO, LiveInterval *LI,
1210                     RangeSet &Entering, RangeSet &Internal, RangeSet &Exiting,
1211                     SlotIndex OldIdx) {
1212    if (MO.readsReg()) {
1213      LiveRange* LR = LI->getLiveRangeContaining(OldIdx);
1214      if (LR != 0)
1215        Entering.insert(std::make_pair(LI, LR));
1216    }
1217    if (MO.isDef()) {
1218      LiveRange* LR = LI->getLiveRangeContaining(OldIdx.getRegSlot());
1219      assert(LR != 0 && "No live range for def?");
1220      if (LR->end > OldIdx.getDeadSlot())
1221        Exiting.insert(std::make_pair(LI, LR));
1222      else
1223        Internal.insert(std::make_pair(LI, LR));
1224    }
1225  }
1226
1227  BundleRanges createBundleRanges(RangeSet& Entering,
1228                                  RangeSet& Internal,
1229                                  RangeSet& Exiting) {
1230    BundleRanges BR;
1231
1232    for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1233         EI != EE; ++EI) {
1234      LiveInterval* LI = EI->first;
1235      LiveRange* LR = EI->second;
1236      BR[LI->reg].Use = LR;
1237    }
1238
1239    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1240         II != IE; ++II) {
1241      LiveInterval* LI = II->first;
1242      LiveRange* LR = II->second;
1243      if (LR->end.isDead()) {
1244        BR[LI->reg].Dead = LR;
1245      } else {
1246        BR[LI->reg].EC = LR;
1247      }
1248    }
1249
1250    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1251         EI != EE; ++EI) {
1252      LiveInterval* LI = EI->first;
1253      LiveRange* LR = EI->second;
1254      BR[LI->reg].Def = LR;
1255    }
1256
1257    return BR;
1258  }
1259
1260  void moveKillFlags(unsigned reg, SlotIndex OldIdx, SlotIndex newKillIdx) {
1261    MachineInstr* OldKillMI = LIS.getInstructionFromIndex(OldIdx);
1262    if (!OldKillMI->killsRegister(reg))
1263      return; // Bail out if we don't have kill flags on the old register.
1264    MachineInstr* NewKillMI = LIS.getInstructionFromIndex(newKillIdx);
1265    assert(OldKillMI->killsRegister(reg) && "Old 'kill' instr isn't a kill.");
1266    assert(!NewKillMI->killsRegister(reg) &&
1267           "New kill instr is already a kill.");
1268    OldKillMI->clearRegisterKills(reg, &TRI);
1269    NewKillMI->addRegisterKilled(reg, &TRI);
1270  }
1271
1272  void updateRegMaskSlots(SlotIndex OldIdx) {
1273    SmallVectorImpl<SlotIndex>::iterator RI =
1274      std::lower_bound(LIS.RegMaskSlots.begin(), LIS.RegMaskSlots.end(),
1275                       OldIdx);
1276    assert(*RI == OldIdx && "No RegMask at OldIdx.");
1277    *RI = NewIdx;
1278    assert(*prior(RI) < *RI && *RI < *next(RI) &&
1279           "RegSlots out of order. Did you move one call across another?");
1280  }
1281
1282  // Return the last use of reg between NewIdx and OldIdx.
1283  SlotIndex findLastUseBefore(unsigned Reg, SlotIndex OldIdx) {
1284    SlotIndex LastUse = NewIdx;
1285    for (MachineRegisterInfo::use_nodbg_iterator
1286           UI = MRI.use_nodbg_begin(Reg),
1287           UE = MRI.use_nodbg_end();
1288         UI != UE; UI.skipInstruction()) {
1289      const MachineInstr* MI = &*UI;
1290      SlotIndex InstSlot = LIS.getSlotIndexes()->getInstructionIndex(MI);
1291      if (InstSlot > LastUse && InstSlot < OldIdx)
1292        LastUse = InstSlot;
1293    }
1294    return LastUse;
1295  }
1296
1297  void moveEnteringUpFrom(SlotIndex OldIdx, IntRangePair& P) {
1298    LiveInterval* LI = P.first;
1299    LiveRange* LR = P.second;
1300    bool LiveThrough = LR->end > OldIdx.getRegSlot();
1301    if (LiveThrough)
1302      return;
1303    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
1304    if (LastUse != NewIdx)
1305      moveKillFlags(LI->reg, NewIdx, LastUse);
1306    LR->end = LastUse.getRegSlot();
1307  }
1308
1309  void moveEnteringDownFrom(SlotIndex OldIdx, IntRangePair& P) {
1310    LiveInterval* LI = P.first;
1311    LiveRange* LR = P.second;
1312    // Extend the LiveRange if NewIdx is past the end.
1313    if (NewIdx > LR->end) {
1314      // Move kill flags if OldIdx was not originally the end
1315      // (otherwise LR->end points to an invalid slot).
1316      if (LR->end.getRegSlot() != OldIdx.getRegSlot()) {
1317        assert(LR->end > OldIdx && "LiveRange does not cover original slot");
1318        moveKillFlags(LI->reg, LR->end, NewIdx);
1319      }
1320      LR->end = NewIdx.getRegSlot();
1321    }
1322  }
1323
1324  void moveAllEnteringFrom(SlotIndex OldIdx, RangeSet& Entering) {
1325    bool GoingUp = NewIdx < OldIdx;
1326
1327    if (GoingUp) {
1328      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1329           EI != EE; ++EI)
1330        moveEnteringUpFrom(OldIdx, *EI);
1331    } else {
1332      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1333           EI != EE; ++EI)
1334        moveEnteringDownFrom(OldIdx, *EI);
1335    }
1336  }
1337
1338  void moveInternalFrom(SlotIndex OldIdx, IntRangePair& P) {
1339    LiveInterval* LI = P.first;
1340    LiveRange* LR = P.second;
1341    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
1342           LR->end <= OldIdx.getDeadSlot() &&
1343           "Range should be internal to OldIdx.");
1344    LiveRange Tmp(*LR);
1345    Tmp.start = NewIdx.getRegSlot(LR->start.isEarlyClobber());
1346    Tmp.valno->def = Tmp.start;
1347    Tmp.end = LR->end.isDead() ? NewIdx.getDeadSlot() : NewIdx.getRegSlot();
1348    LI->removeRange(*LR);
1349    LI->addRange(Tmp);
1350  }
1351
1352  void moveAllInternalFrom(SlotIndex OldIdx, RangeSet& Internal) {
1353    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1354         II != IE; ++II)
1355      moveInternalFrom(OldIdx, *II);
1356  }
1357
1358  void moveExitingFrom(SlotIndex OldIdx, IntRangePair& P) {
1359    LiveRange* LR = P.second;
1360    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
1361           "Range should start in OldIdx.");
1362    assert(LR->end > OldIdx.getDeadSlot() && "Range should exit OldIdx.");
1363    SlotIndex NewStart = NewIdx.getRegSlot(LR->start.isEarlyClobber());
1364    LR->start = NewStart;
1365    LR->valno->def = NewStart;
1366  }
1367
1368  void moveAllExitingFrom(SlotIndex OldIdx, RangeSet& Exiting) {
1369    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1370         EI != EE; ++EI)
1371      moveExitingFrom(OldIdx, *EI);
1372  }
1373
1374  void moveEnteringUpFromInto(SlotIndex OldIdx, IntRangePair& P,
1375                              BundleRanges& BR) {
1376    LiveInterval* LI = P.first;
1377    LiveRange* LR = P.second;
1378    bool LiveThrough = LR->end > OldIdx.getRegSlot();
1379    if (LiveThrough) {
1380      assert((LR->start < NewIdx || BR[LI->reg].Def == LR) &&
1381             "Def in bundle should be def range.");
1382      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
1383             "If bundle has use for this reg it should be LR.");
1384      BR[LI->reg].Use = LR;
1385      return;
1386    }
1387
1388    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
1389    moveKillFlags(LI->reg, OldIdx, LastUse);
1390
1391    if (LR->start < NewIdx) {
1392      // Becoming a new entering range.
1393      assert(BR[LI->reg].Dead == 0 && BR[LI->reg].Def == 0 &&
1394             "Bundle shouldn't be re-defining reg mid-range.");
1395      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
1396             "Bundle shouldn't have different use range for same reg.");
1397      LR->end = LastUse.getRegSlot();
1398      BR[LI->reg].Use = LR;
1399    } else {
1400      // Becoming a new Dead-def.
1401      assert(LR->start == NewIdx.getRegSlot(LR->start.isEarlyClobber()) &&
1402             "Live range starting at unexpected slot.");
1403      assert(BR[LI->reg].Def == LR && "Reg should have def range.");
1404      assert(BR[LI->reg].Dead == 0 &&
1405               "Can't have def and dead def of same reg in a bundle.");
1406      LR->end = LastUse.getDeadSlot();
1407      BR[LI->reg].Dead = BR[LI->reg].Def;
1408      BR[LI->reg].Def = 0;
1409    }
1410  }
1411
1412  void moveEnteringDownFromInto(SlotIndex OldIdx, IntRangePair& P,
1413                                BundleRanges& BR) {
1414    LiveInterval* LI = P.first;
1415    LiveRange* LR = P.second;
1416    if (NewIdx > LR->end) {
1417      // Range extended to bundle. Add to bundle uses.
1418      // Note: Currently adds kill flags to bundle start.
1419      assert(BR[LI->reg].Use == 0 &&
1420             "Bundle already has use range for reg.");
1421      moveKillFlags(LI->reg, LR->end, NewIdx);
1422      LR->end = NewIdx.getRegSlot();
1423      BR[LI->reg].Use = LR;
1424    } else {
1425      assert(BR[LI->reg].Use != 0 &&
1426             "Bundle should already have a use range for reg.");
1427    }
1428  }
1429
1430  void moveAllEnteringFromInto(SlotIndex OldIdx, RangeSet& Entering,
1431                               BundleRanges& BR) {
1432    bool GoingUp = NewIdx < OldIdx;
1433
1434    if (GoingUp) {
1435      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1436           EI != EE; ++EI)
1437        moveEnteringUpFromInto(OldIdx, *EI, BR);
1438    } else {
1439      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
1440           EI != EE; ++EI)
1441        moveEnteringDownFromInto(OldIdx, *EI, BR);
1442    }
1443  }
1444
1445  void moveInternalFromInto(SlotIndex OldIdx, IntRangePair& P,
1446                            BundleRanges& BR) {
1447    // TODO: Sane rules for moving ranges into bundles.
1448  }
1449
1450  void moveAllInternalFromInto(SlotIndex OldIdx, RangeSet& Internal,
1451                               BundleRanges& BR) {
1452    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
1453         II != IE; ++II)
1454      moveInternalFromInto(OldIdx, *II, BR);
1455  }
1456
1457  void moveExitingFromInto(SlotIndex OldIdx, IntRangePair& P,
1458                           BundleRanges& BR) {
1459    LiveInterval* LI = P.first;
1460    LiveRange* LR = P.second;
1461
1462    assert(LR->start.isRegister() &&
1463           "Don't know how to merge exiting ECs into bundles yet.");
1464
1465    if (LR->end > NewIdx.getDeadSlot()) {
1466      // This range is becoming an exiting range on the bundle.
1467      // If there was an old dead-def of this reg, delete it.
1468      if (BR[LI->reg].Dead != 0) {
1469        LI->removeRange(*BR[LI->reg].Dead);
1470        BR[LI->reg].Dead = 0;
1471      }
1472      assert(BR[LI->reg].Def == 0 &&
1473             "Can't have two defs for the same variable exiting a bundle.");
1474      LR->start = NewIdx.getRegSlot();
1475      LR->valno->def = LR->start;
1476      BR[LI->reg].Def = LR;
1477    } else {
1478      // This range is becoming internal to the bundle.
1479      assert(LR->end == NewIdx.getRegSlot() &&
1480             "Can't bundle def whose kill is before the bundle");
1481      if (BR[LI->reg].Dead || BR[LI->reg].Def) {
1482        // Already have a def for this. Just delete range.
1483        LI->removeRange(*LR);
1484      } else {
1485        // Make range dead, record.
1486        LR->end = NewIdx.getDeadSlot();
1487        BR[LI->reg].Dead = LR;
1488        assert(BR[LI->reg].Use == LR &&
1489               "Range becoming dead should currently be use.");
1490      }
1491      // In both cases the range is no longer a use on the bundle.
1492      BR[LI->reg].Use = 0;
1493    }
1494  }
1495
1496  void moveAllExitingFromInto(SlotIndex OldIdx, RangeSet& Exiting,
1497                              BundleRanges& BR) {
1498    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
1499         EI != EE; ++EI)
1500      moveExitingFromInto(OldIdx, *EI, BR);
1501  }
1502
1503};
1504
1505void LiveIntervals::handleMove(MachineInstr* MI) {
1506  SlotIndex OldIndex = Indexes->getInstructionIndex(MI);
1507  Indexes->removeMachineInstrFromMaps(MI);
1508  SlotIndex NewIndex = MI->isInsideBundle() ?
1509                        Indexes->getInstructionIndex(MI) :
1510                        Indexes->insertMachineInstrInMaps(MI);
1511  assert(getMBBStartIdx(MI->getParent()) <= OldIndex &&
1512         OldIndex < getMBBEndIdx(MI->getParent()) &&
1513         "Cannot handle moves across basic block boundaries.");
1514  assert(!MI->isBundled() && "Can't handle bundled instructions yet.");
1515
1516  HMEditor HME(*this, *MRI, *TRI, NewIndex);
1517  HME.moveAllRangesFrom(MI, OldIndex);
1518}
1519
1520void LiveIntervals::handleMoveIntoBundle(MachineInstr* MI,
1521                                         MachineInstr* BundleStart) {
1522  SlotIndex NewIndex = Indexes->getInstructionIndex(BundleStart);
1523  HMEditor HME(*this, *MRI, *TRI, NewIndex);
1524  HME.moveAllRangesInto(MI, BundleStart);
1525}
1526