1//===-- R600MachineScheduler.cpp - R600 Scheduler Interface -*- C++ -*-----===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// \brief R600 Machine Scheduler interface
12//
13//===----------------------------------------------------------------------===//
14
15#include "R600MachineScheduler.h"
16#include "AMDGPUSubtarget.h"
17#include "llvm/CodeGen/MachineRegisterInfo.h"
18#include "llvm/Pass.h"
19#include "llvm/IR/LegacyPassManager.h"
20#include "llvm/Support/raw_ostream.h"
21
22using namespace llvm;
23
24#define DEBUG_TYPE "misched"
25
26void R600SchedStrategy::initialize(ScheduleDAGMI *dag) {
27  assert(dag->hasVRegLiveness() && "R600SchedStrategy needs vreg liveness");
28  DAG = static_cast<ScheduleDAGMILive*>(dag);
29  const AMDGPUSubtarget &ST = DAG->MF.getSubtarget<AMDGPUSubtarget>();
30  TII = static_cast<const R600InstrInfo*>(DAG->TII);
31  TRI = static_cast<const R600RegisterInfo*>(DAG->TRI);
32  VLIW5 = !ST.hasCaymanISA();
33  MRI = &DAG->MRI;
34  CurInstKind = IDOther;
35  CurEmitted = 0;
36  OccupedSlotsMask = 31;
37  InstKindLimit[IDAlu] = TII->getMaxAlusPerClause();
38  InstKindLimit[IDOther] = 32;
39  InstKindLimit[IDFetch] = ST.getTexVTXClauseSize();
40  AluInstCount = 0;
41  FetchInstCount = 0;
42}
43
44void R600SchedStrategy::MoveUnits(std::vector<SUnit *> &QSrc,
45                                  std::vector<SUnit *> &QDst)
46{
47  QDst.insert(QDst.end(), QSrc.begin(), QSrc.end());
48  QSrc.clear();
49}
50
51static
52unsigned getWFCountLimitedByGPR(unsigned GPRCount) {
53  assert (GPRCount && "GPRCount cannot be 0");
54  return 248 / GPRCount;
55}
56
57SUnit* R600SchedStrategy::pickNode(bool &IsTopNode) {
58  SUnit *SU = nullptr;
59  NextInstKind = IDOther;
60
61  IsTopNode = false;
62
63  // check if we might want to switch current clause type
64  bool AllowSwitchToAlu = (CurEmitted >= InstKindLimit[CurInstKind]) ||
65      (Available[CurInstKind].empty());
66  bool AllowSwitchFromAlu = (CurEmitted >= InstKindLimit[CurInstKind]) &&
67      (!Available[IDFetch].empty() || !Available[IDOther].empty());
68
69  if (CurInstKind == IDAlu && !Available[IDFetch].empty()) {
70    // We use the heuristic provided by AMD Accelerated Parallel Processing
71    // OpenCL Programming Guide :
72    // The approx. number of WF that allows TEX inst to hide ALU inst is :
73    // 500 (cycles for TEX) / (AluFetchRatio * 8 (cycles for ALU))
74    float ALUFetchRationEstimate =
75        (AluInstCount + AvailablesAluCount() + Pending[IDAlu].size()) /
76        (FetchInstCount + Available[IDFetch].size());
77    if (ALUFetchRationEstimate == 0) {
78      AllowSwitchFromAlu = true;
79    } else {
80      unsigned NeededWF = 62.5f / ALUFetchRationEstimate;
81      DEBUG( dbgs() << NeededWF << " approx. Wavefronts Required\n" );
82      // We assume the local GPR requirements to be "dominated" by the requirement
83      // of the TEX clause (which consumes 128 bits regs) ; ALU inst before and
84      // after TEX are indeed likely to consume or generate values from/for the
85      // TEX clause.
86      // Available[IDFetch].size() * 2 : GPRs required in the Fetch clause
87      // We assume that fetch instructions are either TnXYZW = TEX TnXYZW (need
88      // one GPR) or TmXYZW = TnXYZW (need 2 GPR).
89      // (TODO : use RegisterPressure)
90      // If we are going too use too many GPR, we flush Fetch instruction to lower
91      // register pressure on 128 bits regs.
92      unsigned NearRegisterRequirement = 2 * Available[IDFetch].size();
93      if (NeededWF > getWFCountLimitedByGPR(NearRegisterRequirement))
94        AllowSwitchFromAlu = true;
95    }
96  }
97
98  if (!SU && ((AllowSwitchToAlu && CurInstKind != IDAlu) ||
99      (!AllowSwitchFromAlu && CurInstKind == IDAlu))) {
100    // try to pick ALU
101    SU = pickAlu();
102    if (!SU && !PhysicalRegCopy.empty()) {
103      SU = PhysicalRegCopy.front();
104      PhysicalRegCopy.erase(PhysicalRegCopy.begin());
105    }
106    if (SU) {
107      if (CurEmitted >= InstKindLimit[IDAlu])
108        CurEmitted = 0;
109      NextInstKind = IDAlu;
110    }
111  }
112
113  if (!SU) {
114    // try to pick FETCH
115    SU = pickOther(IDFetch);
116    if (SU)
117      NextInstKind = IDFetch;
118  }
119
120  // try to pick other
121  if (!SU) {
122    SU = pickOther(IDOther);
123    if (SU)
124      NextInstKind = IDOther;
125  }
126
127  DEBUG(
128      if (SU) {
129        dbgs() << " ** Pick node **\n";
130        SU->dump(DAG);
131      } else {
132        dbgs() << "NO NODE \n";
133        for (unsigned i = 0; i < DAG->SUnits.size(); i++) {
134          const SUnit &S = DAG->SUnits[i];
135          if (!S.isScheduled)
136            S.dump(DAG);
137        }
138      }
139  );
140
141  return SU;
142}
143
144void R600SchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
145  if (NextInstKind != CurInstKind) {
146    DEBUG(dbgs() << "Instruction Type Switch\n");
147    if (NextInstKind != IDAlu)
148      OccupedSlotsMask |= 31;
149    CurEmitted = 0;
150    CurInstKind = NextInstKind;
151  }
152
153  if (CurInstKind == IDAlu) {
154    AluInstCount ++;
155    switch (getAluKind(SU)) {
156    case AluT_XYZW:
157      CurEmitted += 4;
158      break;
159    case AluDiscarded:
160      break;
161    default: {
162      ++CurEmitted;
163      for (MachineInstr::mop_iterator It = SU->getInstr()->operands_begin(),
164          E = SU->getInstr()->operands_end(); It != E; ++It) {
165        MachineOperand &MO = *It;
166        if (MO.isReg() && MO.getReg() == AMDGPU::ALU_LITERAL_X)
167          ++CurEmitted;
168      }
169    }
170    }
171  } else {
172    ++CurEmitted;
173  }
174
175
176  DEBUG(dbgs() << CurEmitted << " Instructions Emitted in this clause\n");
177
178  if (CurInstKind != IDFetch) {
179    MoveUnits(Pending[IDFetch], Available[IDFetch]);
180  } else
181    FetchInstCount++;
182}
183
184static bool
185isPhysicalRegCopy(MachineInstr *MI) {
186  if (MI->getOpcode() != AMDGPU::COPY)
187    return false;
188
189  return !TargetRegisterInfo::isVirtualRegister(MI->getOperand(1).getReg());
190}
191
192void R600SchedStrategy::releaseTopNode(SUnit *SU) {
193  DEBUG(dbgs() << "Top Releasing ";SU->dump(DAG););
194}
195
196void R600SchedStrategy::releaseBottomNode(SUnit *SU) {
197  DEBUG(dbgs() << "Bottom Releasing ";SU->dump(DAG););
198  if (isPhysicalRegCopy(SU->getInstr())) {
199    PhysicalRegCopy.push_back(SU);
200    return;
201  }
202
203  int IK = getInstKind(SU);
204
205  // There is no export clause, we can schedule one as soon as its ready
206  if (IK == IDOther)
207    Available[IDOther].push_back(SU);
208  else
209    Pending[IK].push_back(SU);
210
211}
212
213bool R600SchedStrategy::regBelongsToClass(unsigned Reg,
214                                          const TargetRegisterClass *RC) const {
215  if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
216    return RC->contains(Reg);
217  } else {
218    return MRI->getRegClass(Reg) == RC;
219  }
220}
221
222R600SchedStrategy::AluKind R600SchedStrategy::getAluKind(SUnit *SU) const {
223  MachineInstr *MI = SU->getInstr();
224
225  if (TII->isTransOnly(MI))
226    return AluTrans;
227
228    switch (MI->getOpcode()) {
229    case AMDGPU::PRED_X:
230      return AluPredX;
231    case AMDGPU::INTERP_PAIR_XY:
232    case AMDGPU::INTERP_PAIR_ZW:
233    case AMDGPU::INTERP_VEC_LOAD:
234    case AMDGPU::DOT_4:
235      return AluT_XYZW;
236    case AMDGPU::COPY:
237      if (MI->getOperand(1).isUndef()) {
238        // MI will become a KILL, don't considers it in scheduling
239        return AluDiscarded;
240      }
241    default:
242      break;
243    }
244
245    // Does the instruction take a whole IG ?
246    // XXX: Is it possible to add a helper function in R600InstrInfo that can
247    // be used here and in R600PacketizerList::isSoloInstruction() ?
248    if(TII->isVector(*MI) ||
249        TII->isCubeOp(MI->getOpcode()) ||
250        TII->isReductionOp(MI->getOpcode()) ||
251        MI->getOpcode() == AMDGPU::GROUP_BARRIER) {
252      return AluT_XYZW;
253    }
254
255    if (TII->isLDSInstr(MI->getOpcode())) {
256      return AluT_X;
257    }
258
259    // Is the result already assigned to a channel ?
260    unsigned DestSubReg = MI->getOperand(0).getSubReg();
261    switch (DestSubReg) {
262    case AMDGPU::sub0:
263      return AluT_X;
264    case AMDGPU::sub1:
265      return AluT_Y;
266    case AMDGPU::sub2:
267      return AluT_Z;
268    case AMDGPU::sub3:
269      return AluT_W;
270    default:
271      break;
272    }
273
274    // Is the result already member of a X/Y/Z/W class ?
275    unsigned DestReg = MI->getOperand(0).getReg();
276    if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_XRegClass) ||
277        regBelongsToClass(DestReg, &AMDGPU::R600_AddrRegClass))
278      return AluT_X;
279    if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_YRegClass))
280      return AluT_Y;
281    if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_ZRegClass))
282      return AluT_Z;
283    if (regBelongsToClass(DestReg, &AMDGPU::R600_TReg32_WRegClass))
284      return AluT_W;
285    if (regBelongsToClass(DestReg, &AMDGPU::R600_Reg128RegClass))
286      return AluT_XYZW;
287
288    // LDS src registers cannot be used in the Trans slot.
289    if (TII->readsLDSSrcReg(MI))
290      return AluT_XYZW;
291
292    return AluAny;
293
294}
295
296int R600SchedStrategy::getInstKind(SUnit* SU) {
297  int Opcode = SU->getInstr()->getOpcode();
298
299  if (TII->usesTextureCache(Opcode) || TII->usesVertexCache(Opcode))
300    return IDFetch;
301
302  if (TII->isALUInstr(Opcode)) {
303    return IDAlu;
304  }
305
306  switch (Opcode) {
307  case AMDGPU::PRED_X:
308  case AMDGPU::COPY:
309  case AMDGPU::CONST_COPY:
310  case AMDGPU::INTERP_PAIR_XY:
311  case AMDGPU::INTERP_PAIR_ZW:
312  case AMDGPU::INTERP_VEC_LOAD:
313  case AMDGPU::DOT_4:
314    return IDAlu;
315  default:
316    return IDOther;
317  }
318}
319
320SUnit *R600SchedStrategy::PopInst(std::vector<SUnit *> &Q, bool AnyALU) {
321  if (Q.empty())
322    return nullptr;
323  for (std::vector<SUnit *>::reverse_iterator It = Q.rbegin(), E = Q.rend();
324      It != E; ++It) {
325    SUnit *SU = *It;
326    InstructionsGroupCandidate.push_back(SU->getInstr());
327    if (TII->fitsConstReadLimitations(InstructionsGroupCandidate)
328        && (!AnyALU || !TII->isVectorOnly(SU->getInstr()))
329    ) {
330      InstructionsGroupCandidate.pop_back();
331      Q.erase((It + 1).base());
332      return SU;
333    } else {
334      InstructionsGroupCandidate.pop_back();
335    }
336  }
337  return nullptr;
338}
339
340void R600SchedStrategy::LoadAlu() {
341  std::vector<SUnit *> &QSrc = Pending[IDAlu];
342  for (unsigned i = 0, e = QSrc.size(); i < e; ++i) {
343    AluKind AK = getAluKind(QSrc[i]);
344    AvailableAlus[AK].push_back(QSrc[i]);
345  }
346  QSrc.clear();
347}
348
349void R600SchedStrategy::PrepareNextSlot() {
350  DEBUG(dbgs() << "New Slot\n");
351  assert (OccupedSlotsMask && "Slot wasn't filled");
352  OccupedSlotsMask = 0;
353//  if (HwGen == AMDGPUSubtarget::NORTHERN_ISLANDS)
354//    OccupedSlotsMask |= 16;
355  InstructionsGroupCandidate.clear();
356  LoadAlu();
357}
358
359void R600SchedStrategy::AssignSlot(MachineInstr* MI, unsigned Slot) {
360  int DstIndex = TII->getOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst);
361  if (DstIndex == -1) {
362    return;
363  }
364  unsigned DestReg = MI->getOperand(DstIndex).getReg();
365  // PressureRegister crashes if an operand is def and used in the same inst
366  // and we try to constraint its regclass
367  for (MachineInstr::mop_iterator It = MI->operands_begin(),
368      E = MI->operands_end(); It != E; ++It) {
369    MachineOperand &MO = *It;
370    if (MO.isReg() && !MO.isDef() &&
371        MO.getReg() == DestReg)
372      return;
373  }
374  // Constrains the regclass of DestReg to assign it to Slot
375  switch (Slot) {
376  case 0:
377    MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_XRegClass);
378    break;
379  case 1:
380    MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_YRegClass);
381    break;
382  case 2:
383    MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_ZRegClass);
384    break;
385  case 3:
386    MRI->constrainRegClass(DestReg, &AMDGPU::R600_TReg32_WRegClass);
387    break;
388  }
389}
390
391SUnit *R600SchedStrategy::AttemptFillSlot(unsigned Slot, bool AnyAlu) {
392  static const AluKind IndexToID[] = {AluT_X, AluT_Y, AluT_Z, AluT_W};
393  SUnit *SlotedSU = PopInst(AvailableAlus[IndexToID[Slot]], AnyAlu);
394  if (SlotedSU)
395    return SlotedSU;
396  SUnit *UnslotedSU = PopInst(AvailableAlus[AluAny], AnyAlu);
397  if (UnslotedSU)
398    AssignSlot(UnslotedSU->getInstr(), Slot);
399  return UnslotedSU;
400}
401
402unsigned R600SchedStrategy::AvailablesAluCount() const {
403  return AvailableAlus[AluAny].size() + AvailableAlus[AluT_XYZW].size() +
404      AvailableAlus[AluT_X].size() + AvailableAlus[AluT_Y].size() +
405      AvailableAlus[AluT_Z].size() + AvailableAlus[AluT_W].size() +
406      AvailableAlus[AluTrans].size() + AvailableAlus[AluDiscarded].size() +
407      AvailableAlus[AluPredX].size();
408}
409
410SUnit* R600SchedStrategy::pickAlu() {
411  while (AvailablesAluCount() || !Pending[IDAlu].empty()) {
412    if (!OccupedSlotsMask) {
413      // Bottom up scheduling : predX must comes first
414      if (!AvailableAlus[AluPredX].empty()) {
415        OccupedSlotsMask |= 31;
416        return PopInst(AvailableAlus[AluPredX], false);
417      }
418      // Flush physical reg copies (RA will discard them)
419      if (!AvailableAlus[AluDiscarded].empty()) {
420        OccupedSlotsMask |= 31;
421        return PopInst(AvailableAlus[AluDiscarded], false);
422      }
423      // If there is a T_XYZW alu available, use it
424      if (!AvailableAlus[AluT_XYZW].empty()) {
425        OccupedSlotsMask |= 15;
426        return PopInst(AvailableAlus[AluT_XYZW], false);
427      }
428    }
429    bool TransSlotOccuped = OccupedSlotsMask & 16;
430    if (!TransSlotOccuped && VLIW5) {
431      if (!AvailableAlus[AluTrans].empty()) {
432        OccupedSlotsMask |= 16;
433        return PopInst(AvailableAlus[AluTrans], false);
434      }
435      SUnit *SU = AttemptFillSlot(3, true);
436      if (SU) {
437        OccupedSlotsMask |= 16;
438        return SU;
439      }
440    }
441    for (int Chan = 3; Chan > -1; --Chan) {
442      bool isOccupied = OccupedSlotsMask & (1 << Chan);
443      if (!isOccupied) {
444        SUnit *SU = AttemptFillSlot(Chan, false);
445        if (SU) {
446          OccupedSlotsMask |= (1 << Chan);
447          InstructionsGroupCandidate.push_back(SU->getInstr());
448          return SU;
449        }
450      }
451    }
452    PrepareNextSlot();
453  }
454  return nullptr;
455}
456
457SUnit* R600SchedStrategy::pickOther(int QID) {
458  SUnit *SU = nullptr;
459  std::vector<SUnit *> &AQ = Available[QID];
460
461  if (AQ.empty()) {
462    MoveUnits(Pending[QID], AQ);
463  }
464  if (!AQ.empty()) {
465    SU = AQ.back();
466    AQ.resize(AQ.size() - 1);
467  }
468  return SU;
469}
470