ScheduleDAGFast.cpp revision 36b56886974eae4f9c5ebc96befd3e7bfe5de338 
1//===----- ScheduleDAGFast.cpp - Fast poor list scheduler -----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements a fast scheduler.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "pre-RA-sched"
15#include "llvm/CodeGen/SchedulerRegistry.h"
16#include "InstrEmitter.h"
17#include "ScheduleDAGSDNodes.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/CodeGen/SelectionDAGISel.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/InlineAsm.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/Support/raw_ostream.h"
27#include "llvm/Target/TargetInstrInfo.h"
28#include "llvm/Target/TargetRegisterInfo.h"
29using namespace llvm;
30
31STATISTIC(NumUnfolds,    "Number of nodes unfolded");
32STATISTIC(NumDups,       "Number of duplicated nodes");
33STATISTIC(NumPRCopies,   "Number of physical copies");
34
35static RegisterScheduler
36  fastDAGScheduler("fast", "Fast suboptimal list scheduling",
37                   createFastDAGScheduler);
38static RegisterScheduler
39  linearizeDAGScheduler("linearize", "Linearize DAG, no scheduling",
40                        createDAGLinearizer);
41
42
43namespace {
44  /// FastPriorityQueue - A degenerate priority queue that considers
45  /// all nodes to have the same priority.
46  ///
47  struct FastPriorityQueue {
48    SmallVector<SUnit *, 16> Queue;
49
50    bool empty() const { return Queue.empty(); }
51
52    void push(SUnit *U) {
53      Queue.push_back(U);
54    }
55
56    SUnit *pop() {
57      if (empty()) return NULL;
58      SUnit *V = Queue.back();
59      Queue.pop_back();
60      return V;
61    }
62  };
63
64//===----------------------------------------------------------------------===//
65/// ScheduleDAGFast - The actual "fast" list scheduler implementation.
66///
67class ScheduleDAGFast : public ScheduleDAGSDNodes {
68private:
69  /// AvailableQueue - The priority queue to use for the available SUnits.
70  FastPriorityQueue AvailableQueue;
71
72  /// LiveRegDefs - A set of physical registers and their definition
73  /// that are "live". These nodes must be scheduled before any other nodes that
74  /// modifies the registers can be scheduled.
75  unsigned NumLiveRegs;
76  std::vector<SUnit*> LiveRegDefs;
77  std::vector<unsigned> LiveRegCycles;
78
79public:
80  ScheduleDAGFast(MachineFunction &mf)
81    : ScheduleDAGSDNodes(mf) {}
82
83  void Schedule() override;
84
85  /// AddPred - adds a predecessor edge to SUnit SU.
86  /// This returns true if this is a new predecessor.
87  void AddPred(SUnit *SU, const SDep &D) {
88    SU->addPred(D);
89  }
90
91  /// RemovePred - removes a predecessor edge from SUnit SU.
92  /// This returns true if an edge was removed.
93  void RemovePred(SUnit *SU, const SDep &D) {
94    SU->removePred(D);
95  }
96
97private:
98  void ReleasePred(SUnit *SU, SDep *PredEdge);
99  void ReleasePredecessors(SUnit *SU, unsigned CurCycle);
100  void ScheduleNodeBottomUp(SUnit*, unsigned);
101  SUnit *CopyAndMoveSuccessors(SUnit*);
102  void InsertCopiesAndMoveSuccs(SUnit*, unsigned,
103                                const TargetRegisterClass*,
104                                const TargetRegisterClass*,
105                                SmallVectorImpl<SUnit*>&);
106  bool DelayForLiveRegsBottomUp(SUnit*, SmallVectorImpl<unsigned>&);
107  void ListScheduleBottomUp();
108
109  /// forceUnitLatencies - The fast scheduler doesn't care about real latencies.
110  bool forceUnitLatencies() const override { return true; }
111};
112}  // end anonymous namespace
113
114
115/// Schedule - Schedule the DAG using list scheduling.
116void ScheduleDAGFast::Schedule() {
117  DEBUG(dbgs() << "********** List Scheduling **********\n");
118
119  NumLiveRegs = 0;
120  LiveRegDefs.resize(TRI->getNumRegs(), NULL);
121  LiveRegCycles.resize(TRI->getNumRegs(), 0);
122
123  // Build the scheduling graph.
124  BuildSchedGraph(NULL);
125
126  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
127          SUnits[su].dumpAll(this));
128
129  // Execute the actual scheduling loop.
130  ListScheduleBottomUp();
131}
132
133//===----------------------------------------------------------------------===//
134//  Bottom-Up Scheduling
135//===----------------------------------------------------------------------===//
136
137/// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to
138/// the AvailableQueue if the count reaches zero. Also update its cycle bound.
139void ScheduleDAGFast::ReleasePred(SUnit *SU, SDep *PredEdge) {
140  SUnit *PredSU = PredEdge->getSUnit();
141
142#ifndef NDEBUG
143  if (PredSU->NumSuccsLeft == 0) {
144    dbgs() << "*** Scheduling failed! ***\n";
145    PredSU->dump(this);
146    dbgs() << " has been released too many times!\n";
147    llvm_unreachable(0);
148  }
149#endif
150  --PredSU->NumSuccsLeft;
151
152  // If all the node's successors are scheduled, this node is ready
153  // to be scheduled. Ignore the special EntrySU node.
154  if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) {
155    PredSU->isAvailable = true;
156    AvailableQueue.push(PredSU);
157  }
158}
159
160void ScheduleDAGFast::ReleasePredecessors(SUnit *SU, unsigned CurCycle) {
161  // Bottom up: release predecessors
162  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
163       I != E; ++I) {
164    ReleasePred(SU, &*I);
165    if (I->isAssignedRegDep()) {
166      // This is a physical register dependency and it's impossible or
167      // expensive to copy the register. Make sure nothing that can
168      // clobber the register is scheduled between the predecessor and
169      // this node.
170      if (!LiveRegDefs[I->getReg()]) {
171        ++NumLiveRegs;
172        LiveRegDefs[I->getReg()] = I->getSUnit();
173        LiveRegCycles[I->getReg()] = CurCycle;
174      }
175    }
176  }
177}
178
179/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
180/// count of its predecessors. If a predecessor pending count is zero, add it to
181/// the Available queue.
182void ScheduleDAGFast::ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle) {
183  DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: ");
184  DEBUG(SU->dump(this));
185
186  assert(CurCycle >= SU->getHeight() && "Node scheduled below its height!");
187  SU->setHeightToAtLeast(CurCycle);
188  Sequence.push_back(SU);
189
190  ReleasePredecessors(SU, CurCycle);
191
192  // Release all the implicit physical register defs that are live.
193  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
194       I != E; ++I) {
195    if (I->isAssignedRegDep()) {
196      if (LiveRegCycles[I->getReg()] == I->getSUnit()->getHeight()) {
197        assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
198        assert(LiveRegDefs[I->getReg()] == SU &&
199               "Physical register dependency violated?");
200        --NumLiveRegs;
201        LiveRegDefs[I->getReg()] = NULL;
202        LiveRegCycles[I->getReg()] = 0;
203      }
204    }
205  }
206
207  SU->isScheduled = true;
208}
209
210/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled
211/// successors to the newly created node.
212SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
213  if (SU->getNode()->getGluedNode())
214    return NULL;
215
216  SDNode *N = SU->getNode();
217  if (!N)
218    return NULL;
219
220  SUnit *NewSU;
221  bool TryUnfold = false;
222  for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
223    EVT VT = N->getValueType(i);
224    if (VT == MVT::Glue)
225      return NULL;
226    else if (VT == MVT::Other)
227      TryUnfold = true;
228  }
229  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
230    const SDValue &Op = N->getOperand(i);
231    EVT VT = Op.getNode()->getValueType(Op.getResNo());
232    if (VT == MVT::Glue)
233      return NULL;
234  }
235
236  if (TryUnfold) {
237    SmallVector<SDNode*, 2> NewNodes;
238    if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes))
239      return NULL;
240
241    DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n");
242    assert(NewNodes.size() == 2 && "Expected a load folding node!");
243
244    N = NewNodes[1];
245    SDNode *LoadNode = NewNodes[0];
246    unsigned NumVals = N->getNumValues();
247    unsigned OldNumVals = SU->getNode()->getNumValues();
248    for (unsigned i = 0; i != NumVals; ++i)
249      DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
250    DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1),
251                                   SDValue(LoadNode, 1));
252
253    SUnit *NewSU = newSUnit(N);
254    assert(N->getNodeId() == -1 && "Node already inserted!");
255    N->setNodeId(NewSU->NodeNum);
256
257    const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
258    for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
259      if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
260        NewSU->isTwoAddress = true;
261        break;
262      }
263    }
264    if (MCID.isCommutable())
265      NewSU->isCommutable = true;
266
267    // LoadNode may already exist. This can happen when there is another
268    // load from the same location and producing the same type of value
269    // but it has different alignment or volatileness.
270    bool isNewLoad = true;
271    SUnit *LoadSU;
272    if (LoadNode->getNodeId() != -1) {
273      LoadSU = &SUnits[LoadNode->getNodeId()];
274      isNewLoad = false;
275    } else {
276      LoadSU = newSUnit(LoadNode);
277      LoadNode->setNodeId(LoadSU->NodeNum);
278    }
279
280    SDep ChainPred;
281    SmallVector<SDep, 4> ChainSuccs;
282    SmallVector<SDep, 4> LoadPreds;
283    SmallVector<SDep, 4> NodePreds;
284    SmallVector<SDep, 4> NodeSuccs;
285    for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
286         I != E; ++I) {
287      if (I->isCtrl())
288        ChainPred = *I;
289      else if (I->getSUnit()->getNode() &&
290               I->getSUnit()->getNode()->isOperandOf(LoadNode))
291        LoadPreds.push_back(*I);
292      else
293        NodePreds.push_back(*I);
294    }
295    for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
296         I != E; ++I) {
297      if (I->isCtrl())
298        ChainSuccs.push_back(*I);
299      else
300        NodeSuccs.push_back(*I);
301    }
302
303    if (ChainPred.getSUnit()) {
304      RemovePred(SU, ChainPred);
305      if (isNewLoad)
306        AddPred(LoadSU, ChainPred);
307    }
308    for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) {
309      const SDep &Pred = LoadPreds[i];
310      RemovePred(SU, Pred);
311      if (isNewLoad) {
312        AddPred(LoadSU, Pred);
313      }
314    }
315    for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) {
316      const SDep &Pred = NodePreds[i];
317      RemovePred(SU, Pred);
318      AddPred(NewSU, Pred);
319    }
320    for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) {
321      SDep D = NodeSuccs[i];
322      SUnit *SuccDep = D.getSUnit();
323      D.setSUnit(SU);
324      RemovePred(SuccDep, D);
325      D.setSUnit(NewSU);
326      AddPred(SuccDep, D);
327    }
328    for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) {
329      SDep D = ChainSuccs[i];
330      SUnit *SuccDep = D.getSUnit();
331      D.setSUnit(SU);
332      RemovePred(SuccDep, D);
333      if (isNewLoad) {
334        D.setSUnit(LoadSU);
335        AddPred(SuccDep, D);
336      }
337    }
338    if (isNewLoad) {
339      SDep D(LoadSU, SDep::Barrier);
340      D.setLatency(LoadSU->Latency);
341      AddPred(NewSU, D);
342    }
343
344    ++NumUnfolds;
345
346    if (NewSU->NumSuccsLeft == 0) {
347      NewSU->isAvailable = true;
348      return NewSU;
349    }
350    SU = NewSU;
351  }
352
353  DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n");
354  NewSU = Clone(SU);
355
356  // New SUnit has the exact same predecessors.
357  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
358       I != E; ++I)
359    if (!I->isArtificial())
360      AddPred(NewSU, *I);
361
362  // Only copy scheduled successors. Cut them from old node's successor
363  // list and move them over.
364  SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
365  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
366       I != E; ++I) {
367    if (I->isArtificial())
368      continue;
369    SUnit *SuccSU = I->getSUnit();
370    if (SuccSU->isScheduled) {
371      SDep D = *I;
372      D.setSUnit(NewSU);
373      AddPred(SuccSU, D);
374      D.setSUnit(SU);
375      DelDeps.push_back(std::make_pair(SuccSU, D));
376    }
377  }
378  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
379    RemovePred(DelDeps[i].first, DelDeps[i].second);
380
381  ++NumDups;
382  return NewSU;
383}
384
385/// InsertCopiesAndMoveSuccs - Insert register copies and move all
386/// scheduled successors of the given SUnit to the last copy.
387void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
388                                              const TargetRegisterClass *DestRC,
389                                              const TargetRegisterClass *SrcRC,
390                                              SmallVectorImpl<SUnit*> &Copies) {
391  SUnit *CopyFromSU = newSUnit(static_cast<SDNode *>(NULL));
392  CopyFromSU->CopySrcRC = SrcRC;
393  CopyFromSU->CopyDstRC = DestRC;
394
395  SUnit *CopyToSU = newSUnit(static_cast<SDNode *>(NULL));
396  CopyToSU->CopySrcRC = DestRC;
397  CopyToSU->CopyDstRC = SrcRC;
398
399  // Only copy scheduled successors. Cut them from old node's successor
400  // list and move them over.
401  SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
402  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
403       I != E; ++I) {
404    if (I->isArtificial())
405      continue;
406    SUnit *SuccSU = I->getSUnit();
407    if (SuccSU->isScheduled) {
408      SDep D = *I;
409      D.setSUnit(CopyToSU);
410      AddPred(SuccSU, D);
411      DelDeps.push_back(std::make_pair(SuccSU, *I));
412    }
413  }
414  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
415    RemovePred(DelDeps[i].first, DelDeps[i].second);
416  }
417  SDep FromDep(SU, SDep::Data, Reg);
418  FromDep.setLatency(SU->Latency);
419  AddPred(CopyFromSU, FromDep);
420  SDep ToDep(CopyFromSU, SDep::Data, 0);
421  ToDep.setLatency(CopyFromSU->Latency);
422  AddPred(CopyToSU, ToDep);
423
424  Copies.push_back(CopyFromSU);
425  Copies.push_back(CopyToSU);
426
427  ++NumPRCopies;
428}
429
430/// getPhysicalRegisterVT - Returns the ValueType of the physical register
431/// definition of the specified node.
432/// FIXME: Move to SelectionDAG?
433static EVT getPhysicalRegisterVT(SDNode *N, unsigned Reg,
434                                 const TargetInstrInfo *TII) {
435  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
436  assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!");
437  unsigned NumRes = MCID.getNumDefs();
438  for (const uint16_t *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) {
439    if (Reg == *ImpDef)
440      break;
441    ++NumRes;
442  }
443  return N->getValueType(NumRes);
444}
445
446/// CheckForLiveRegDef - Return true and update live register vector if the
447/// specified register def of the specified SUnit clobbers any "live" registers.
448static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg,
449                               std::vector<SUnit*> &LiveRegDefs,
450                               SmallSet<unsigned, 4> &RegAdded,
451                               SmallVectorImpl<unsigned> &LRegs,
452                               const TargetRegisterInfo *TRI) {
453  bool Added = false;
454  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
455    if (LiveRegDefs[*AI] && LiveRegDefs[*AI] != SU) {
456      if (RegAdded.insert(*AI)) {
457        LRegs.push_back(*AI);
458        Added = true;
459      }
460    }
461  }
462  return Added;
463}
464
465/// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay
466/// scheduling of the given node to satisfy live physical register dependencies.
467/// If the specific node is the last one that's available to schedule, do
468/// whatever is necessary (i.e. backtracking or cloning) to make it possible.
469bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU,
470                                              SmallVectorImpl<unsigned> &LRegs){
471  if (NumLiveRegs == 0)
472    return false;
473
474  SmallSet<unsigned, 4> RegAdded;
475  // If this node would clobber any "live" register, then it's not ready.
476  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
477       I != E; ++I) {
478    if (I->isAssignedRegDep()) {
479      CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs,
480                         RegAdded, LRegs, TRI);
481    }
482  }
483
484  for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
485    if (Node->getOpcode() == ISD::INLINEASM) {
486      // Inline asm can clobber physical defs.
487      unsigned NumOps = Node->getNumOperands();
488      if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
489        --NumOps;  // Ignore the glue operand.
490
491      for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
492        unsigned Flags =
493          cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
494        unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
495
496        ++i; // Skip the ID value.
497        if (InlineAsm::isRegDefKind(Flags) ||
498            InlineAsm::isRegDefEarlyClobberKind(Flags) ||
499            InlineAsm::isClobberKind(Flags)) {
500          // Check for def of register or earlyclobber register.
501          for (; NumVals; --NumVals, ++i) {
502            unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
503            if (TargetRegisterInfo::isPhysicalRegister(Reg))
504              CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI);
505          }
506        } else
507          i += NumVals;
508      }
509      continue;
510    }
511    if (!Node->isMachineOpcode())
512      continue;
513    const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode());
514    if (!MCID.ImplicitDefs)
515      continue;
516    for (const uint16_t *Reg = MCID.getImplicitDefs(); *Reg; ++Reg) {
517      CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI);
518    }
519  }
520  return !LRegs.empty();
521}
522
523
524/// ListScheduleBottomUp - The main loop of list scheduling for bottom-up
525/// schedulers.
526void ScheduleDAGFast::ListScheduleBottomUp() {
527  unsigned CurCycle = 0;
528
529  // Release any predecessors of the special Exit node.
530  ReleasePredecessors(&ExitSU, CurCycle);
531
532  // Add root to Available queue.
533  if (!SUnits.empty()) {
534    SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
535    assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!");
536    RootSU->isAvailable = true;
537    AvailableQueue.push(RootSU);
538  }
539
540  // While Available queue is not empty, grab the node with the highest
541  // priority. If it is not ready put it back.  Schedule the node.
542  SmallVector<SUnit*, 4> NotReady;
543  DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMap;
544  Sequence.reserve(SUnits.size());
545  while (!AvailableQueue.empty()) {
546    bool Delayed = false;
547    LRegsMap.clear();
548    SUnit *CurSU = AvailableQueue.pop();
549    while (CurSU) {
550      SmallVector<unsigned, 4> LRegs;
551      if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
552        break;
553      Delayed = true;
554      LRegsMap.insert(std::make_pair(CurSU, LRegs));
555
556      CurSU->isPending = true;  // This SU is not in AvailableQueue right now.
557      NotReady.push_back(CurSU);
558      CurSU = AvailableQueue.pop();
559    }
560
561    // All candidates are delayed due to live physical reg dependencies.
562    // Try code duplication or inserting cross class copies
563    // to resolve it.
564    if (Delayed && !CurSU) {
565      if (!CurSU) {
566        // Try duplicating the nodes that produces these
567        // "expensive to copy" values to break the dependency. In case even
568        // that doesn't work, insert cross class copies.
569        SUnit *TrySU = NotReady[0];
570        SmallVectorImpl<unsigned> &LRegs = LRegsMap[TrySU];
571        assert(LRegs.size() == 1 && "Can't handle this yet!");
572        unsigned Reg = LRegs[0];
573        SUnit *LRDef = LiveRegDefs[Reg];
574        EVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII);
575        const TargetRegisterClass *RC =
576          TRI->getMinimalPhysRegClass(Reg, VT);
577        const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC);
578
579        // If cross copy register class is the same as RC, then it must be
580        // possible copy the value directly. Do not try duplicate the def.
581        // If cross copy register class is not the same as RC, then it's
582        // possible to copy the value but it require cross register class copies
583        // and it is expensive.
584        // If cross copy register class is null, then it's not possible to copy
585        // the value at all.
586        SUnit *NewDef = 0;
587        if (DestRC != RC) {
588          NewDef = CopyAndMoveSuccessors(LRDef);
589          if (!DestRC && !NewDef)
590            report_fatal_error("Can't handle live physical "
591                               "register dependency!");
592        }
593        if (!NewDef) {
594          // Issue copies, these can be expensive cross register class copies.
595          SmallVector<SUnit*, 2> Copies;
596          InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
597          DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
598                       << " to SU #" << Copies.front()->NodeNum << "\n");
599          AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
600          NewDef = Copies.back();
601        }
602
603        DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
604                     << " to SU #" << TrySU->NodeNum << "\n");
605        LiveRegDefs[Reg] = NewDef;
606        AddPred(NewDef, SDep(TrySU, SDep::Artificial));
607        TrySU->isAvailable = false;
608        CurSU = NewDef;
609      }
610
611      if (!CurSU) {
612        llvm_unreachable("Unable to resolve live physical register dependencies!");
613      }
614    }
615
616    // Add the nodes that aren't ready back onto the available list.
617    for (unsigned i = 0, e = NotReady.size(); i != e; ++i) {
618      NotReady[i]->isPending = false;
619      // May no longer be available due to backtracking.
620      if (NotReady[i]->isAvailable)
621        AvailableQueue.push(NotReady[i]);
622    }
623    NotReady.clear();
624
625    if (CurSU)
626      ScheduleNodeBottomUp(CurSU, CurCycle);
627    ++CurCycle;
628  }
629
630  // Reverse the order since it is bottom up.
631  std::reverse(Sequence.begin(), Sequence.end());
632
633#ifndef NDEBUG
634  VerifyScheduledSequence(/*isBottomUp=*/true);
635#endif
636}
637
638
639namespace {
640//===----------------------------------------------------------------------===//
641// ScheduleDAGLinearize - No scheduling scheduler, it simply linearize the
642// DAG in topological order.
643// IMPORTANT: this may not work for targets with phyreg dependency.
644//
645class ScheduleDAGLinearize : public ScheduleDAGSDNodes {
646public:
647  ScheduleDAGLinearize(MachineFunction &mf) : ScheduleDAGSDNodes(mf) {}
648
649  void Schedule() override;
650
651  MachineBasicBlock *
652    EmitSchedule(MachineBasicBlock::iterator &InsertPos) override;
653
654private:
655  std::vector<SDNode*> Sequence;
656  DenseMap<SDNode*, SDNode*> GluedMap;  // Cache glue to its user
657
658  void ScheduleNode(SDNode *N);
659};
660} // end anonymous namespace
661
662void ScheduleDAGLinearize::ScheduleNode(SDNode *N) {
663  if (N->getNodeId() != 0)
664    llvm_unreachable(0);
665
666  if (!N->isMachineOpcode() &&
667      (N->getOpcode() == ISD::EntryToken || isPassiveNode(N)))
668    // These nodes do not need to be translated into MIs.
669    return;
670
671  DEBUG(dbgs() << "\n*** Scheduling: ");
672  DEBUG(N->dump(DAG));
673  Sequence.push_back(N);
674
675  unsigned NumOps = N->getNumOperands();
676  if (unsigned NumLeft = NumOps) {
677    SDNode *GluedOpN = 0;
678    do {
679      const SDValue &Op = N->getOperand(NumLeft-1);
680      SDNode *OpN = Op.getNode();
681
682      if (NumLeft == NumOps && Op.getValueType() == MVT::Glue) {
683        // Schedule glue operand right above N.
684        GluedOpN = OpN;
685        assert(OpN->getNodeId() != 0 && "Glue operand not ready?");
686        OpN->setNodeId(0);
687        ScheduleNode(OpN);
688        continue;
689      }
690
691      if (OpN == GluedOpN)
692        // Glue operand is already scheduled.
693        continue;
694
695      DenseMap<SDNode*, SDNode*>::iterator DI = GluedMap.find(OpN);
696      if (DI != GluedMap.end() && DI->second != N)
697        // Users of glues are counted against the glued users.
698        OpN = DI->second;
699
700      unsigned Degree = OpN->getNodeId();
701      assert(Degree > 0 && "Predecessor over-released!");
702      OpN->setNodeId(--Degree);
703      if (Degree == 0)
704        ScheduleNode(OpN);
705    } while (--NumLeft);
706  }
707}
708
709/// findGluedUser - Find the representative use of a glue value by walking
710/// the use chain.
711static SDNode *findGluedUser(SDNode *N) {
712  while (SDNode *Glued = N->getGluedUser())
713    N = Glued;
714  return N;
715}
716
717void ScheduleDAGLinearize::Schedule() {
718  DEBUG(dbgs() << "********** DAG Linearization **********\n");
719
720  SmallVector<SDNode*, 8> Glues;
721  unsigned DAGSize = 0;
722  for (SelectionDAG::allnodes_iterator I = DAG->allnodes_begin(),
723         E = DAG->allnodes_end(); I != E; ++I) {
724    SDNode *N = I;
725
726    // Use node id to record degree.
727    unsigned Degree = N->use_size();
728    N->setNodeId(Degree);
729    unsigned NumVals = N->getNumValues();
730    if (NumVals && N->getValueType(NumVals-1) == MVT::Glue &&
731        N->hasAnyUseOfValue(NumVals-1)) {
732      SDNode *User = findGluedUser(N);
733      if (User) {
734        Glues.push_back(N);
735        GluedMap.insert(std::make_pair(N, User));
736      }
737    }
738
739    if (N->isMachineOpcode() ||
740        (N->getOpcode() != ISD::EntryToken && !isPassiveNode(N)))
741      ++DAGSize;
742  }
743
744  for (unsigned i = 0, e = Glues.size(); i != e; ++i) {
745    SDNode *Glue = Glues[i];
746    SDNode *GUser = GluedMap[Glue];
747    unsigned Degree = Glue->getNodeId();
748    unsigned UDegree = GUser->getNodeId();
749
750    // Glue user must be scheduled together with the glue operand. So other
751    // users of the glue operand must be treated as its users.
752    SDNode *ImmGUser = Glue->getGluedUser();
753    for (SDNode::use_iterator ui = Glue->use_begin(), ue = Glue->use_end();
754         ui != ue; ++ui)
755      if (*ui == ImmGUser)
756        --Degree;
757    GUser->setNodeId(UDegree + Degree);
758    Glue->setNodeId(1);
759  }
760
761  Sequence.reserve(DAGSize);
762  ScheduleNode(DAG->getRoot().getNode());
763}
764
765MachineBasicBlock*
766ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
767  InstrEmitter Emitter(BB, InsertPos);
768  DenseMap<SDValue, unsigned> VRBaseMap;
769
770  DEBUG({
771      dbgs() << "\n*** Final schedule ***\n";
772    });
773
774  // FIXME: Handle dbg_values.
775  unsigned NumNodes = Sequence.size();
776  for (unsigned i = 0; i != NumNodes; ++i) {
777    SDNode *N = Sequence[NumNodes-i-1];
778    DEBUG(N->dump(DAG));
779    Emitter.EmitNode(N, false, false, VRBaseMap);
780  }
781
782  DEBUG(dbgs() << '\n');
783
784  InsertPos = Emitter.getInsertPos();
785  return Emitter.getBlock();
786}
787
788//===----------------------------------------------------------------------===//
789//                         Public Constructor Functions
790//===----------------------------------------------------------------------===//
791
792llvm::ScheduleDAGSDNodes *
793llvm::createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
794  return new ScheduleDAGFast(*IS->MF);
795}
796
797llvm::ScheduleDAGSDNodes *
798llvm::createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level) {
799  return new ScheduleDAGLinearize(*IS->MF);
800}
801