1//===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains the Base ARM implementation of the TargetInstrInfo class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMBaseRegisterInfo.h"
17#include "ARMConstantPoolValue.h"
18#include "ARMFeatures.h"
19#include "ARMHazardRecognizer.h"
20#include "ARMMachineFunctionInfo.h"
21#include "MCTargetDesc/ARMAddressingModes.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/CodeGen/LiveVariables.h"
24#include "llvm/CodeGen/MachineConstantPool.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/SelectionDAGNodes.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/GlobalValue.h"
34#include "llvm/MC/MCAsmInfo.h"
35#include "llvm/MC/MCExpr.h"
36#include "llvm/Support/BranchProbability.h"
37#include "llvm/Support/CommandLine.h"
38#include "llvm/Support/Debug.h"
39#include "llvm/Support/ErrorHandling.h"
40#include "llvm/Support/raw_ostream.h"
41
42using namespace llvm;
43
44#define DEBUG_TYPE "arm-instrinfo"
45
46#define GET_INSTRINFO_CTOR_DTOR
47#include "ARMGenInstrInfo.inc"
48
49static cl::opt<bool>
50EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden,
51               cl::desc("Enable ARM 2-addr to 3-addr conv"));
52
53static cl::opt<bool>
54WidenVMOVS("widen-vmovs", cl::Hidden, cl::init(true),
55           cl::desc("Widen ARM vmovs to vmovd when possible"));
56
57static cl::opt<unsigned>
58SwiftPartialUpdateClearance("swift-partial-update-clearance",
59     cl::Hidden, cl::init(12),
60     cl::desc("Clearance before partial register updates"));
61
62/// ARM_MLxEntry - Record information about MLA / MLS instructions.
63struct ARM_MLxEntry {
64  uint16_t MLxOpc;     // MLA / MLS opcode
65  uint16_t MulOpc;     // Expanded multiplication opcode
66  uint16_t AddSubOpc;  // Expanded add / sub opcode
67  bool NegAcc;         // True if the acc is negated before the add / sub.
68  bool HasLane;        // True if instruction has an extra "lane" operand.
69};
70
71static const ARM_MLxEntry ARM_MLxTable[] = {
72  // MLxOpc,          MulOpc,           AddSubOpc,       NegAcc, HasLane
73  // fp scalar ops
74  { ARM::VMLAS,       ARM::VMULS,       ARM::VADDS,      false,  false },
75  { ARM::VMLSS,       ARM::VMULS,       ARM::VSUBS,      false,  false },
76  { ARM::VMLAD,       ARM::VMULD,       ARM::VADDD,      false,  false },
77  { ARM::VMLSD,       ARM::VMULD,       ARM::VSUBD,      false,  false },
78  { ARM::VNMLAS,      ARM::VNMULS,      ARM::VSUBS,      true,   false },
79  { ARM::VNMLSS,      ARM::VMULS,       ARM::VSUBS,      true,   false },
80  { ARM::VNMLAD,      ARM::VNMULD,      ARM::VSUBD,      true,   false },
81  { ARM::VNMLSD,      ARM::VMULD,       ARM::VSUBD,      true,   false },
82
83  // fp SIMD ops
84  { ARM::VMLAfd,      ARM::VMULfd,      ARM::VADDfd,     false,  false },
85  { ARM::VMLSfd,      ARM::VMULfd,      ARM::VSUBfd,     false,  false },
86  { ARM::VMLAfq,      ARM::VMULfq,      ARM::VADDfq,     false,  false },
87  { ARM::VMLSfq,      ARM::VMULfq,      ARM::VSUBfq,     false,  false },
88  { ARM::VMLAslfd,    ARM::VMULslfd,    ARM::VADDfd,     false,  true  },
89  { ARM::VMLSslfd,    ARM::VMULslfd,    ARM::VSUBfd,     false,  true  },
90  { ARM::VMLAslfq,    ARM::VMULslfq,    ARM::VADDfq,     false,  true  },
91  { ARM::VMLSslfq,    ARM::VMULslfq,    ARM::VSUBfq,     false,  true  },
92};
93
94ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget& STI)
95  : ARMGenInstrInfo(ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP),
96    Subtarget(STI) {
97  for (unsigned i = 0, e = array_lengthof(ARM_MLxTable); i != e; ++i) {
98    if (!MLxEntryMap.insert(std::make_pair(ARM_MLxTable[i].MLxOpc, i)).second)
99      assert(false && "Duplicated entries?");
100    MLxHazardOpcodes.insert(ARM_MLxTable[i].AddSubOpc);
101    MLxHazardOpcodes.insert(ARM_MLxTable[i].MulOpc);
102  }
103}
104
105// Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
106// currently defaults to no prepass hazard recognizer.
107ScheduleHazardRecognizer *
108ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
109                                               const ScheduleDAG *DAG) const {
110  if (usePreRAHazardRecognizer()) {
111    const InstrItineraryData *II =
112        static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData();
113    return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
114  }
115  return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
116}
117
118ScheduleHazardRecognizer *ARMBaseInstrInfo::
119CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
120                                   const ScheduleDAG *DAG) const {
121  if (Subtarget.isThumb2() || Subtarget.hasVFP2())
122    return (ScheduleHazardRecognizer *)new ARMHazardRecognizer(II, DAG);
123  return TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG);
124}
125
126MachineInstr *
127ARMBaseInstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
128                                        MachineBasicBlock::iterator &MBBI,
129                                        LiveVariables *LV) const {
130  // FIXME: Thumb2 support.
131
132  if (!EnableARM3Addr)
133    return nullptr;
134
135  MachineInstr *MI = MBBI;
136  MachineFunction &MF = *MI->getParent()->getParent();
137  uint64_t TSFlags = MI->getDesc().TSFlags;
138  bool isPre = false;
139  switch ((TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift) {
140  default: return nullptr;
141  case ARMII::IndexModePre:
142    isPre = true;
143    break;
144  case ARMII::IndexModePost:
145    break;
146  }
147
148  // Try splitting an indexed load/store to an un-indexed one plus an add/sub
149  // operation.
150  unsigned MemOpc = getUnindexedOpcode(MI->getOpcode());
151  if (MemOpc == 0)
152    return nullptr;
153
154  MachineInstr *UpdateMI = nullptr;
155  MachineInstr *MemMI = nullptr;
156  unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
157  const MCInstrDesc &MCID = MI->getDesc();
158  unsigned NumOps = MCID.getNumOperands();
159  bool isLoad = !MI->mayStore();
160  const MachineOperand &WB = isLoad ? MI->getOperand(1) : MI->getOperand(0);
161  const MachineOperand &Base = MI->getOperand(2);
162  const MachineOperand &Offset = MI->getOperand(NumOps-3);
163  unsigned WBReg = WB.getReg();
164  unsigned BaseReg = Base.getReg();
165  unsigned OffReg = Offset.getReg();
166  unsigned OffImm = MI->getOperand(NumOps-2).getImm();
167  ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NumOps-1).getImm();
168  switch (AddrMode) {
169  default: llvm_unreachable("Unknown indexed op!");
170  case ARMII::AddrMode2: {
171    bool isSub = ARM_AM::getAM2Op(OffImm) == ARM_AM::sub;
172    unsigned Amt = ARM_AM::getAM2Offset(OffImm);
173    if (OffReg == 0) {
174      if (ARM_AM::getSOImmVal(Amt) == -1)
175        // Can't encode it in a so_imm operand. This transformation will
176        // add more than 1 instruction. Abandon!
177        return nullptr;
178      UpdateMI = BuildMI(MF, MI->getDebugLoc(),
179                         get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
180        .addReg(BaseReg).addImm(Amt)
181        .addImm(Pred).addReg(0).addReg(0);
182    } else if (Amt != 0) {
183      ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm);
184      unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt);
185      UpdateMI = BuildMI(MF, MI->getDebugLoc(),
186                         get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg)
187        .addReg(BaseReg).addReg(OffReg).addReg(0).addImm(SOOpc)
188        .addImm(Pred).addReg(0).addReg(0);
189    } else
190      UpdateMI = BuildMI(MF, MI->getDebugLoc(),
191                         get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
192        .addReg(BaseReg).addReg(OffReg)
193        .addImm(Pred).addReg(0).addReg(0);
194    break;
195  }
196  case ARMII::AddrMode3 : {
197    bool isSub = ARM_AM::getAM3Op(OffImm) == ARM_AM::sub;
198    unsigned Amt = ARM_AM::getAM3Offset(OffImm);
199    if (OffReg == 0)
200      // Immediate is 8-bits. It's guaranteed to fit in a so_imm operand.
201      UpdateMI = BuildMI(MF, MI->getDebugLoc(),
202                         get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
203        .addReg(BaseReg).addImm(Amt)
204        .addImm(Pred).addReg(0).addReg(0);
205    else
206      UpdateMI = BuildMI(MF, MI->getDebugLoc(),
207                         get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
208        .addReg(BaseReg).addReg(OffReg)
209        .addImm(Pred).addReg(0).addReg(0);
210    break;
211  }
212  }
213
214  std::vector<MachineInstr*> NewMIs;
215  if (isPre) {
216    if (isLoad)
217      MemMI = BuildMI(MF, MI->getDebugLoc(),
218                      get(MemOpc), MI->getOperand(0).getReg())
219        .addReg(WBReg).addImm(0).addImm(Pred);
220    else
221      MemMI = BuildMI(MF, MI->getDebugLoc(),
222                      get(MemOpc)).addReg(MI->getOperand(1).getReg())
223        .addReg(WBReg).addReg(0).addImm(0).addImm(Pred);
224    NewMIs.push_back(MemMI);
225    NewMIs.push_back(UpdateMI);
226  } else {
227    if (isLoad)
228      MemMI = BuildMI(MF, MI->getDebugLoc(),
229                      get(MemOpc), MI->getOperand(0).getReg())
230        .addReg(BaseReg).addImm(0).addImm(Pred);
231    else
232      MemMI = BuildMI(MF, MI->getDebugLoc(),
233                      get(MemOpc)).addReg(MI->getOperand(1).getReg())
234        .addReg(BaseReg).addReg(0).addImm(0).addImm(Pred);
235    if (WB.isDead())
236      UpdateMI->getOperand(0).setIsDead();
237    NewMIs.push_back(UpdateMI);
238    NewMIs.push_back(MemMI);
239  }
240
241  // Transfer LiveVariables states, kill / dead info.
242  if (LV) {
243    for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
244      MachineOperand &MO = MI->getOperand(i);
245      if (MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
246        unsigned Reg = MO.getReg();
247
248        LiveVariables::VarInfo &VI = LV->getVarInfo(Reg);
249        if (MO.isDef()) {
250          MachineInstr *NewMI = (Reg == WBReg) ? UpdateMI : MemMI;
251          if (MO.isDead())
252            LV->addVirtualRegisterDead(Reg, NewMI);
253        }
254        if (MO.isUse() && MO.isKill()) {
255          for (unsigned j = 0; j < 2; ++j) {
256            // Look at the two new MI's in reverse order.
257            MachineInstr *NewMI = NewMIs[j];
258            if (!NewMI->readsRegister(Reg))
259              continue;
260            LV->addVirtualRegisterKilled(Reg, NewMI);
261            if (VI.removeKill(MI))
262              VI.Kills.push_back(NewMI);
263            break;
264          }
265        }
266      }
267    }
268  }
269
270  MFI->insert(MBBI, NewMIs[1]);
271  MFI->insert(MBBI, NewMIs[0]);
272  return NewMIs[0];
273}
274
275// Branch analysis.
276bool
277ARMBaseInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB,
278                                MachineBasicBlock *&FBB,
279                                SmallVectorImpl<MachineOperand> &Cond,
280                                bool AllowModify) const {
281  TBB = nullptr;
282  FBB = nullptr;
283
284  MachineBasicBlock::iterator I = MBB.end();
285  if (I == MBB.begin())
286    return false; // Empty blocks are easy.
287  --I;
288
289  // Walk backwards from the end of the basic block until the branch is
290  // analyzed or we give up.
291  while (isPredicated(I) || I->isTerminator() || I->isDebugValue()) {
292
293    // Flag to be raised on unanalyzeable instructions. This is useful in cases
294    // where we want to clean up on the end of the basic block before we bail
295    // out.
296    bool CantAnalyze = false;
297
298    // Skip over DEBUG values and predicated nonterminators.
299    while (I->isDebugValue() || !I->isTerminator()) {
300      if (I == MBB.begin())
301        return false;
302      --I;
303    }
304
305    if (isIndirectBranchOpcode(I->getOpcode()) ||
306        isJumpTableBranchOpcode(I->getOpcode())) {
307      // Indirect branches and jump tables can't be analyzed, but we still want
308      // to clean up any instructions at the tail of the basic block.
309      CantAnalyze = true;
310    } else if (isUncondBranchOpcode(I->getOpcode())) {
311      TBB = I->getOperand(0).getMBB();
312    } else if (isCondBranchOpcode(I->getOpcode())) {
313      // Bail out if we encounter multiple conditional branches.
314      if (!Cond.empty())
315        return true;
316
317      assert(!FBB && "FBB should have been null.");
318      FBB = TBB;
319      TBB = I->getOperand(0).getMBB();
320      Cond.push_back(I->getOperand(1));
321      Cond.push_back(I->getOperand(2));
322    } else if (I->isReturn()) {
323      // Returns can't be analyzed, but we should run cleanup.
324      CantAnalyze = !isPredicated(I);
325    } else {
326      // We encountered other unrecognized terminator. Bail out immediately.
327      return true;
328    }
329
330    // Cleanup code - to be run for unpredicated unconditional branches and
331    //                returns.
332    if (!isPredicated(I) &&
333          (isUncondBranchOpcode(I->getOpcode()) ||
334           isIndirectBranchOpcode(I->getOpcode()) ||
335           isJumpTableBranchOpcode(I->getOpcode()) ||
336           I->isReturn())) {
337      // Forget any previous condition branch information - it no longer applies.
338      Cond.clear();
339      FBB = nullptr;
340
341      // If we can modify the function, delete everything below this
342      // unconditional branch.
343      if (AllowModify) {
344        MachineBasicBlock::iterator DI = std::next(I);
345        while (DI != MBB.end()) {
346          MachineInstr *InstToDelete = DI;
347          ++DI;
348          InstToDelete->eraseFromParent();
349        }
350      }
351    }
352
353    if (CantAnalyze)
354      return true;
355
356    if (I == MBB.begin())
357      return false;
358
359    --I;
360  }
361
362  // We made it past the terminators without bailing out - we must have
363  // analyzed this branch successfully.
364  return false;
365}
366
367
368unsigned ARMBaseInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
369  MachineBasicBlock::iterator I = MBB.end();
370  if (I == MBB.begin()) return 0;
371  --I;
372  while (I->isDebugValue()) {
373    if (I == MBB.begin())
374      return 0;
375    --I;
376  }
377  if (!isUncondBranchOpcode(I->getOpcode()) &&
378      !isCondBranchOpcode(I->getOpcode()))
379    return 0;
380
381  // Remove the branch.
382  I->eraseFromParent();
383
384  I = MBB.end();
385
386  if (I == MBB.begin()) return 1;
387  --I;
388  if (!isCondBranchOpcode(I->getOpcode()))
389    return 1;
390
391  // Remove the branch.
392  I->eraseFromParent();
393  return 2;
394}
395
396unsigned
397ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
398                               MachineBasicBlock *FBB,
399                               const SmallVectorImpl<MachineOperand> &Cond,
400                               DebugLoc DL) const {
401  ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
402  int BOpc   = !AFI->isThumbFunction()
403    ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
404  int BccOpc = !AFI->isThumbFunction()
405    ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
406  bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
407
408  // Shouldn't be a fall through.
409  assert(TBB && "InsertBranch must not be told to insert a fallthrough");
410  assert((Cond.size() == 2 || Cond.size() == 0) &&
411         "ARM branch conditions have two components!");
412
413  if (!FBB) {
414    if (Cond.empty()) { // Unconditional branch?
415      if (isThumb)
416        BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).addImm(ARMCC::AL).addReg(0);
417      else
418        BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
419    } else
420      BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB)
421        .addImm(Cond[0].getImm()).addReg(Cond[1].getReg());
422    return 1;
423  }
424
425  // Two-way conditional branch.
426  BuildMI(&MBB, DL, get(BccOpc)).addMBB(TBB)
427    .addImm(Cond[0].getImm()).addReg(Cond[1].getReg());
428  if (isThumb)
429    BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).addImm(ARMCC::AL).addReg(0);
430  else
431    BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
432  return 2;
433}
434
435bool ARMBaseInstrInfo::
436ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
437  ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm();
438  Cond[0].setImm(ARMCC::getOppositeCondition(CC));
439  return false;
440}
441
442bool ARMBaseInstrInfo::isPredicated(const MachineInstr *MI) const {
443  if (MI->isBundle()) {
444    MachineBasicBlock::const_instr_iterator I = MI;
445    MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end();
446    while (++I != E && I->isInsideBundle()) {
447      int PIdx = I->findFirstPredOperandIdx();
448      if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL)
449        return true;
450    }
451    return false;
452  }
453
454  int PIdx = MI->findFirstPredOperandIdx();
455  return PIdx != -1 && MI->getOperand(PIdx).getImm() != ARMCC::AL;
456}
457
458bool ARMBaseInstrInfo::
459PredicateInstruction(MachineInstr *MI,
460                     const SmallVectorImpl<MachineOperand> &Pred) const {
461  unsigned Opc = MI->getOpcode();
462  if (isUncondBranchOpcode(Opc)) {
463    MI->setDesc(get(getMatchingCondBranchOpcode(Opc)));
464    MachineInstrBuilder(*MI->getParent()->getParent(), MI)
465      .addImm(Pred[0].getImm())
466      .addReg(Pred[1].getReg());
467    return true;
468  }
469
470  int PIdx = MI->findFirstPredOperandIdx();
471  if (PIdx != -1) {
472    MachineOperand &PMO = MI->getOperand(PIdx);
473    PMO.setImm(Pred[0].getImm());
474    MI->getOperand(PIdx+1).setReg(Pred[1].getReg());
475    return true;
476  }
477  return false;
478}
479
480bool ARMBaseInstrInfo::
481SubsumesPredicate(const SmallVectorImpl<MachineOperand> &Pred1,
482                  const SmallVectorImpl<MachineOperand> &Pred2) const {
483  if (Pred1.size() > 2 || Pred2.size() > 2)
484    return false;
485
486  ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm();
487  ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm();
488  if (CC1 == CC2)
489    return true;
490
491  switch (CC1) {
492  default:
493    return false;
494  case ARMCC::AL:
495    return true;
496  case ARMCC::HS:
497    return CC2 == ARMCC::HI;
498  case ARMCC::LS:
499    return CC2 == ARMCC::LO || CC2 == ARMCC::EQ;
500  case ARMCC::GE:
501    return CC2 == ARMCC::GT;
502  case ARMCC::LE:
503    return CC2 == ARMCC::LT;
504  }
505}
506
507bool ARMBaseInstrInfo::DefinesPredicate(MachineInstr *MI,
508                                    std::vector<MachineOperand> &Pred) const {
509  bool Found = false;
510  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
511    const MachineOperand &MO = MI->getOperand(i);
512    if ((MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) ||
513        (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)) {
514      Pred.push_back(MO);
515      Found = true;
516    }
517  }
518
519  return Found;
520}
521
522static bool isCPSRDefined(const MachineInstr *MI) {
523  for (const auto &MO : MI->operands())
524    if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef())
525      return true;
526  return false;
527}
528
529static bool isEligibleForITBlock(const MachineInstr *MI) {
530  switch (MI->getOpcode()) {
531  default: return true;
532  case ARM::tADC:   // ADC (register) T1
533  case ARM::tADDi3: // ADD (immediate) T1
534  case ARM::tADDi8: // ADD (immediate) T2
535  case ARM::tADDrr: // ADD (register) T1
536  case ARM::tAND:   // AND (register) T1
537  case ARM::tASRri: // ASR (immediate) T1
538  case ARM::tASRrr: // ASR (register) T1
539  case ARM::tBIC:   // BIC (register) T1
540  case ARM::tEOR:   // EOR (register) T1
541  case ARM::tLSLri: // LSL (immediate) T1
542  case ARM::tLSLrr: // LSL (register) T1
543  case ARM::tLSRri: // LSR (immediate) T1
544  case ARM::tLSRrr: // LSR (register) T1
545  case ARM::tMUL:   // MUL T1
546  case ARM::tMVN:   // MVN (register) T1
547  case ARM::tORR:   // ORR (register) T1
548  case ARM::tROR:   // ROR (register) T1
549  case ARM::tRSB:   // RSB (immediate) T1
550  case ARM::tSBC:   // SBC (register) T1
551  case ARM::tSUBi3: // SUB (immediate) T1
552  case ARM::tSUBi8: // SUB (immediate) T2
553  case ARM::tSUBrr: // SUB (register) T1
554    return !isCPSRDefined(MI);
555  }
556}
557
558/// isPredicable - Return true if the specified instruction can be predicated.
559/// By default, this returns true for every instruction with a
560/// PredicateOperand.
561bool ARMBaseInstrInfo::isPredicable(MachineInstr *MI) const {
562  if (!MI->isPredicable())
563    return false;
564
565  if (!isEligibleForITBlock(MI))
566    return false;
567
568  ARMFunctionInfo *AFI =
569    MI->getParent()->getParent()->getInfo<ARMFunctionInfo>();
570
571  if (AFI->isThumb2Function()) {
572    if (getSubtarget().restrictIT())
573      return isV8EligibleForIT(MI);
574  } else { // non-Thumb
575    if ((MI->getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON)
576      return false;
577  }
578
579  return true;
580}
581
582namespace llvm {
583template <> bool IsCPSRDead<MachineInstr>(MachineInstr *MI) {
584  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
585    const MachineOperand &MO = MI->getOperand(i);
586    if (!MO.isReg() || MO.isUndef() || MO.isUse())
587      continue;
588    if (MO.getReg() != ARM::CPSR)
589      continue;
590    if (!MO.isDead())
591      return false;
592  }
593  // all definitions of CPSR are dead
594  return true;
595}
596}
597
598/// GetInstSize - Return the size of the specified MachineInstr.
599///
600unsigned ARMBaseInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
601  const MachineBasicBlock &MBB = *MI->getParent();
602  const MachineFunction *MF = MBB.getParent();
603  const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
604
605  const MCInstrDesc &MCID = MI->getDesc();
606  if (MCID.getSize())
607    return MCID.getSize();
608
609  // If this machine instr is an inline asm, measure it.
610  if (MI->getOpcode() == ARM::INLINEASM)
611    return getInlineAsmLength(MI->getOperand(0).getSymbolName(), *MAI);
612  unsigned Opc = MI->getOpcode();
613  switch (Opc) {
614  default:
615    // pseudo-instruction sizes are zero.
616    return 0;
617  case TargetOpcode::BUNDLE:
618    return getInstBundleLength(MI);
619  case ARM::MOVi16_ga_pcrel:
620  case ARM::MOVTi16_ga_pcrel:
621  case ARM::t2MOVi16_ga_pcrel:
622  case ARM::t2MOVTi16_ga_pcrel:
623    return 4;
624  case ARM::MOVi32imm:
625  case ARM::t2MOVi32imm:
626    return 8;
627  case ARM::CONSTPOOL_ENTRY:
628    // If this machine instr is a constant pool entry, its size is recorded as
629    // operand #2.
630    return MI->getOperand(2).getImm();
631  case ARM::Int_eh_sjlj_longjmp:
632    return 16;
633  case ARM::tInt_eh_sjlj_longjmp:
634    return 10;
635  case ARM::Int_eh_sjlj_setjmp:
636  case ARM::Int_eh_sjlj_setjmp_nofp:
637    return 20;
638  case ARM::tInt_eh_sjlj_setjmp:
639  case ARM::t2Int_eh_sjlj_setjmp:
640  case ARM::t2Int_eh_sjlj_setjmp_nofp:
641    return 12;
642  case ARM::BR_JTr:
643  case ARM::BR_JTm:
644  case ARM::BR_JTadd:
645  case ARM::tBR_JTr:
646  case ARM::t2BR_JT:
647  case ARM::t2TBB_JT:
648  case ARM::t2TBH_JT: {
649    // These are jumptable branches, i.e. a branch followed by an inlined
650    // jumptable. The size is 4 + 4 * number of entries. For TBB, each
651    // entry is one byte; TBH two byte each.
652    unsigned EntrySize = (Opc == ARM::t2TBB_JT)
653      ? 1 : ((Opc == ARM::t2TBH_JT) ? 2 : 4);
654    unsigned NumOps = MCID.getNumOperands();
655    MachineOperand JTOP =
656      MI->getOperand(NumOps - (MI->isPredicable() ? 3 : 2));
657    unsigned JTI = JTOP.getIndex();
658    const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
659    assert(MJTI != nullptr);
660    const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
661    assert(JTI < JT.size());
662    // Thumb instructions are 2 byte aligned, but JT entries are 4 byte
663    // 4 aligned. The assembler / linker may add 2 byte padding just before
664    // the JT entries.  The size does not include this padding; the
665    // constant islands pass does separate bookkeeping for it.
666    // FIXME: If we know the size of the function is less than (1 << 16) *2
667    // bytes, we can use 16-bit entries instead. Then there won't be an
668    // alignment issue.
669    unsigned InstSize = (Opc == ARM::tBR_JTr || Opc == ARM::t2BR_JT) ? 2 : 4;
670    unsigned NumEntries = JT[JTI].MBBs.size();
671    if (Opc == ARM::t2TBB_JT && (NumEntries & 1))
672      // Make sure the instruction that follows TBB is 2-byte aligned.
673      // FIXME: Constant island pass should insert an "ALIGN" instruction
674      // instead.
675      ++NumEntries;
676    return NumEntries * EntrySize + InstSize;
677  }
678  case ARM::SPACE:
679    return MI->getOperand(1).getImm();
680  }
681}
682
683unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr *MI) const {
684  unsigned Size = 0;
685  MachineBasicBlock::const_instr_iterator I = MI;
686  MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end();
687  while (++I != E && I->isInsideBundle()) {
688    assert(!I->isBundle() && "No nested bundle!");
689    Size += GetInstSizeInBytes(&*I);
690  }
691  return Size;
692}
693
694void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB,
695                                    MachineBasicBlock::iterator I,
696                                    unsigned DestReg, bool KillSrc,
697                                    const ARMSubtarget &Subtarget) const {
698  unsigned Opc = Subtarget.isThumb()
699                     ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR)
700                     : ARM::MRS;
701
702  MachineInstrBuilder MIB =
703      BuildMI(MBB, I, I->getDebugLoc(), get(Opc), DestReg);
704
705  // There is only 1 A/R class MRS instruction, and it always refers to
706  // APSR. However, there are lots of other possibilities on M-class cores.
707  if (Subtarget.isMClass())
708    MIB.addImm(0x800);
709
710  AddDefaultPred(MIB);
711
712  MIB.addReg(ARM::CPSR, RegState::Implicit | getKillRegState(KillSrc));
713}
714
715void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB,
716                                  MachineBasicBlock::iterator I,
717                                  unsigned SrcReg, bool KillSrc,
718                                  const ARMSubtarget &Subtarget) const {
719  unsigned Opc = Subtarget.isThumb()
720                     ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR)
721                     : ARM::MSR;
722
723  MachineInstrBuilder MIB = BuildMI(MBB, I, I->getDebugLoc(), get(Opc));
724
725  if (Subtarget.isMClass())
726    MIB.addImm(0x800);
727  else
728    MIB.addImm(8);
729
730  MIB.addReg(SrcReg, getKillRegState(KillSrc));
731
732  AddDefaultPred(MIB);
733
734  MIB.addReg(ARM::CPSR, RegState::Implicit | RegState::Define);
735}
736
737void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
738                                   MachineBasicBlock::iterator I, DebugLoc DL,
739                                   unsigned DestReg, unsigned SrcReg,
740                                   bool KillSrc) const {
741  bool GPRDest = ARM::GPRRegClass.contains(DestReg);
742  bool GPRSrc = ARM::GPRRegClass.contains(SrcReg);
743
744  if (GPRDest && GPRSrc) {
745    AddDefaultCC(AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::MOVr), DestReg)
746                                    .addReg(SrcReg, getKillRegState(KillSrc))));
747    return;
748  }
749
750  bool SPRDest = ARM::SPRRegClass.contains(DestReg);
751  bool SPRSrc = ARM::SPRRegClass.contains(SrcReg);
752
753  unsigned Opc = 0;
754  if (SPRDest && SPRSrc)
755    Opc = ARM::VMOVS;
756  else if (GPRDest && SPRSrc)
757    Opc = ARM::VMOVRS;
758  else if (SPRDest && GPRSrc)
759    Opc = ARM::VMOVSR;
760  else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && !Subtarget.isFPOnlySP())
761    Opc = ARM::VMOVD;
762  else if (ARM::QPRRegClass.contains(DestReg, SrcReg))
763    Opc = ARM::VORRq;
764
765  if (Opc) {
766    MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg);
767    MIB.addReg(SrcReg, getKillRegState(KillSrc));
768    if (Opc == ARM::VORRq)
769      MIB.addReg(SrcReg, getKillRegState(KillSrc));
770    AddDefaultPred(MIB);
771    return;
772  }
773
774  // Handle register classes that require multiple instructions.
775  unsigned BeginIdx = 0;
776  unsigned SubRegs = 0;
777  int Spacing = 1;
778
779  // Use VORRq when possible.
780  if (ARM::QQPRRegClass.contains(DestReg, SrcReg)) {
781    Opc = ARM::VORRq;
782    BeginIdx = ARM::qsub_0;
783    SubRegs = 2;
784  } else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) {
785    Opc = ARM::VORRq;
786    BeginIdx = ARM::qsub_0;
787    SubRegs = 4;
788  // Fall back to VMOVD.
789  } else if (ARM::DPairRegClass.contains(DestReg, SrcReg)) {
790    Opc = ARM::VMOVD;
791    BeginIdx = ARM::dsub_0;
792    SubRegs = 2;
793  } else if (ARM::DTripleRegClass.contains(DestReg, SrcReg)) {
794    Opc = ARM::VMOVD;
795    BeginIdx = ARM::dsub_0;
796    SubRegs = 3;
797  } else if (ARM::DQuadRegClass.contains(DestReg, SrcReg)) {
798    Opc = ARM::VMOVD;
799    BeginIdx = ARM::dsub_0;
800    SubRegs = 4;
801  } else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg)) {
802    Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr;
803    BeginIdx = ARM::gsub_0;
804    SubRegs = 2;
805  } else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg)) {
806    Opc = ARM::VMOVD;
807    BeginIdx = ARM::dsub_0;
808    SubRegs = 2;
809    Spacing = 2;
810  } else if (ARM::DTripleSpcRegClass.contains(DestReg, SrcReg)) {
811    Opc = ARM::VMOVD;
812    BeginIdx = ARM::dsub_0;
813    SubRegs = 3;
814    Spacing = 2;
815  } else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg)) {
816    Opc = ARM::VMOVD;
817    BeginIdx = ARM::dsub_0;
818    SubRegs = 4;
819    Spacing = 2;
820  } else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && Subtarget.isFPOnlySP()) {
821    Opc = ARM::VMOVS;
822    BeginIdx = ARM::ssub_0;
823    SubRegs = 2;
824  } else if (SrcReg == ARM::CPSR) {
825    copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget);
826    return;
827  } else if (DestReg == ARM::CPSR) {
828    copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget);
829    return;
830  }
831
832  assert(Opc && "Impossible reg-to-reg copy");
833
834  const TargetRegisterInfo *TRI = &getRegisterInfo();
835  MachineInstrBuilder Mov;
836
837  // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
838  if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) {
839    BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing);
840    Spacing = -Spacing;
841  }
842#ifndef NDEBUG
843  SmallSet<unsigned, 4> DstRegs;
844#endif
845  for (unsigned i = 0; i != SubRegs; ++i) {
846    unsigned Dst = TRI->getSubReg(DestReg, BeginIdx + i * Spacing);
847    unsigned Src = TRI->getSubReg(SrcReg, BeginIdx + i * Spacing);
848    assert(Dst && Src && "Bad sub-register");
849#ifndef NDEBUG
850    assert(!DstRegs.count(Src) && "destructive vector copy");
851    DstRegs.insert(Dst);
852#endif
853    Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst).addReg(Src);
854    // VORR takes two source operands.
855    if (Opc == ARM::VORRq)
856      Mov.addReg(Src);
857    Mov = AddDefaultPred(Mov);
858    // MOVr can set CC.
859    if (Opc == ARM::MOVr)
860      Mov = AddDefaultCC(Mov);
861  }
862  // Add implicit super-register defs and kills to the last instruction.
863  Mov->addRegisterDefined(DestReg, TRI);
864  if (KillSrc)
865    Mov->addRegisterKilled(SrcReg, TRI);
866}
867
868const MachineInstrBuilder &
869ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB, unsigned Reg,
870                          unsigned SubIdx, unsigned State,
871                          const TargetRegisterInfo *TRI) const {
872  if (!SubIdx)
873    return MIB.addReg(Reg, State);
874
875  if (TargetRegisterInfo::isPhysicalRegister(Reg))
876    return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
877  return MIB.addReg(Reg, State, SubIdx);
878}
879
880void ARMBaseInstrInfo::
881storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
882                    unsigned SrcReg, bool isKill, int FI,
883                    const TargetRegisterClass *RC,
884                    const TargetRegisterInfo *TRI) const {
885  DebugLoc DL;
886  if (I != MBB.end()) DL = I->getDebugLoc();
887  MachineFunction &MF = *MBB.getParent();
888  MachineFrameInfo &MFI = *MF.getFrameInfo();
889  unsigned Align = MFI.getObjectAlignment(FI);
890
891  MachineMemOperand *MMO =
892    MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(FI),
893                            MachineMemOperand::MOStore,
894                            MFI.getObjectSize(FI),
895                            Align);
896
897  switch (RC->getSize()) {
898    case 4:
899      if (ARM::GPRRegClass.hasSubClassEq(RC)) {
900        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::STRi12))
901                   .addReg(SrcReg, getKillRegState(isKill))
902                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
903      } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
904        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRS))
905                   .addReg(SrcReg, getKillRegState(isKill))
906                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
907      } else
908        llvm_unreachable("Unknown reg class!");
909      break;
910    case 8:
911      if (ARM::DPRRegClass.hasSubClassEq(RC)) {
912        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRD))
913                   .addReg(SrcReg, getKillRegState(isKill))
914                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
915      } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
916        if (Subtarget.hasV5TEOps()) {
917          MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::STRD));
918          AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
919          AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
920          MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO);
921
922          AddDefaultPred(MIB);
923        } else {
924          // Fallback to STM instruction, which has existed since the dawn of
925          // time.
926          MachineInstrBuilder MIB =
927            AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::STMIA))
928                             .addFrameIndex(FI).addMemOperand(MMO));
929          AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
930          AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
931        }
932      } else
933        llvm_unreachable("Unknown reg class!");
934      break;
935    case 16:
936      if (ARM::DPairRegClass.hasSubClassEq(RC)) {
937        // Use aligned spills if the stack can be realigned.
938        if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
939          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1q64))
940                     .addFrameIndex(FI).addImm(16)
941                     .addReg(SrcReg, getKillRegState(isKill))
942                     .addMemOperand(MMO));
943        } else {
944          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMQIA))
945                     .addReg(SrcReg, getKillRegState(isKill))
946                     .addFrameIndex(FI)
947                     .addMemOperand(MMO));
948        }
949      } else
950        llvm_unreachable("Unknown reg class!");
951      break;
952    case 24:
953      if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
954        // Use aligned spills if the stack can be realigned.
955        if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
956          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1d64TPseudo))
957                     .addFrameIndex(FI).addImm(16)
958                     .addReg(SrcReg, getKillRegState(isKill))
959                     .addMemOperand(MMO));
960        } else {
961          MachineInstrBuilder MIB =
962          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA))
963                       .addFrameIndex(FI))
964                       .addMemOperand(MMO);
965          MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
966          MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
967          AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
968        }
969      } else
970        llvm_unreachable("Unknown reg class!");
971      break;
972    case 32:
973      if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
974        if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
975          // FIXME: It's possible to only store part of the QQ register if the
976          // spilled def has a sub-register index.
977          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VST1d64QPseudo))
978                     .addFrameIndex(FI).addImm(16)
979                     .addReg(SrcReg, getKillRegState(isKill))
980                     .addMemOperand(MMO));
981        } else {
982          MachineInstrBuilder MIB =
983          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA))
984                       .addFrameIndex(FI))
985                       .addMemOperand(MMO);
986          MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
987          MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
988          MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
989                AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
990        }
991      } else
992        llvm_unreachable("Unknown reg class!");
993      break;
994    case 64:
995      if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
996        MachineInstrBuilder MIB =
997          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTMDIA))
998                         .addFrameIndex(FI))
999                         .addMemOperand(MMO);
1000        MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1001        MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1002        MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1003        MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1004        MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI);
1005        MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI);
1006        MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI);
1007              AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI);
1008      } else
1009        llvm_unreachable("Unknown reg class!");
1010      break;
1011    default:
1012      llvm_unreachable("Unknown reg class!");
1013  }
1014}
1015
1016unsigned
1017ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr *MI,
1018                                     int &FrameIndex) const {
1019  switch (MI->getOpcode()) {
1020  default: break;
1021  case ARM::STRrs:
1022  case ARM::t2STRs: // FIXME: don't use t2STRs to access frame.
1023    if (MI->getOperand(1).isFI() &&
1024        MI->getOperand(2).isReg() &&
1025        MI->getOperand(3).isImm() &&
1026        MI->getOperand(2).getReg() == 0 &&
1027        MI->getOperand(3).getImm() == 0) {
1028      FrameIndex = MI->getOperand(1).getIndex();
1029      return MI->getOperand(0).getReg();
1030    }
1031    break;
1032  case ARM::STRi12:
1033  case ARM::t2STRi12:
1034  case ARM::tSTRspi:
1035  case ARM::VSTRD:
1036  case ARM::VSTRS:
1037    if (MI->getOperand(1).isFI() &&
1038        MI->getOperand(2).isImm() &&
1039        MI->getOperand(2).getImm() == 0) {
1040      FrameIndex = MI->getOperand(1).getIndex();
1041      return MI->getOperand(0).getReg();
1042    }
1043    break;
1044  case ARM::VST1q64:
1045  case ARM::VST1d64TPseudo:
1046  case ARM::VST1d64QPseudo:
1047    if (MI->getOperand(0).isFI() &&
1048        MI->getOperand(2).getSubReg() == 0) {
1049      FrameIndex = MI->getOperand(0).getIndex();
1050      return MI->getOperand(2).getReg();
1051    }
1052    break;
1053  case ARM::VSTMQIA:
1054    if (MI->getOperand(1).isFI() &&
1055        MI->getOperand(0).getSubReg() == 0) {
1056      FrameIndex = MI->getOperand(1).getIndex();
1057      return MI->getOperand(0).getReg();
1058    }
1059    break;
1060  }
1061
1062  return 0;
1063}
1064
1065unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr *MI,
1066                                                    int &FrameIndex) const {
1067  const MachineMemOperand *Dummy;
1068  return MI->mayStore() && hasStoreToStackSlot(MI, Dummy, FrameIndex);
1069}
1070
1071void ARMBaseInstrInfo::
1072loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1073                     unsigned DestReg, int FI,
1074                     const TargetRegisterClass *RC,
1075                     const TargetRegisterInfo *TRI) const {
1076  DebugLoc DL;
1077  if (I != MBB.end()) DL = I->getDebugLoc();
1078  MachineFunction &MF = *MBB.getParent();
1079  MachineFrameInfo &MFI = *MF.getFrameInfo();
1080  unsigned Align = MFI.getObjectAlignment(FI);
1081  MachineMemOperand *MMO =
1082    MF.getMachineMemOperand(
1083                    MachinePointerInfo::getFixedStack(FI),
1084                            MachineMemOperand::MOLoad,
1085                            MFI.getObjectSize(FI),
1086                            Align);
1087
1088  switch (RC->getSize()) {
1089  case 4:
1090    if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1091      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg)
1092                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
1093
1094    } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1095      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg)
1096                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
1097    } else
1098      llvm_unreachable("Unknown reg class!");
1099    break;
1100  case 8:
1101    if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1102      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
1103                   .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
1104    } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1105      MachineInstrBuilder MIB;
1106
1107      if (Subtarget.hasV5TEOps()) {
1108        MIB = BuildMI(MBB, I, DL, get(ARM::LDRD));
1109        AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1110        AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1111        MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO);
1112
1113        AddDefaultPred(MIB);
1114      } else {
1115        // Fallback to LDM instruction, which has existed since the dawn of
1116        // time.
1117        MIB = AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::LDMIA))
1118                                 .addFrameIndex(FI).addMemOperand(MMO));
1119        MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1120        MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1121      }
1122
1123      if (TargetRegisterInfo::isPhysicalRegister(DestReg))
1124        MIB.addReg(DestReg, RegState::ImplicitDefine);
1125    } else
1126      llvm_unreachable("Unknown reg class!");
1127    break;
1128  case 16:
1129    if (ARM::DPairRegClass.hasSubClassEq(RC)) {
1130      if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
1131        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg)
1132                     .addFrameIndex(FI).addImm(16)
1133                     .addMemOperand(MMO));
1134      } else {
1135        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg)
1136                       .addFrameIndex(FI)
1137                       .addMemOperand(MMO));
1138      }
1139    } else
1140      llvm_unreachable("Unknown reg class!");
1141    break;
1142  case 24:
1143    if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1144      if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
1145        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg)
1146                     .addFrameIndex(FI).addImm(16)
1147                     .addMemOperand(MMO));
1148      } else {
1149        MachineInstrBuilder MIB =
1150          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1151                         .addFrameIndex(FI)
1152                         .addMemOperand(MMO));
1153        MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1154        MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1155        MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1156        if (TargetRegisterInfo::isPhysicalRegister(DestReg))
1157          MIB.addReg(DestReg, RegState::ImplicitDefine);
1158      }
1159    } else
1160      llvm_unreachable("Unknown reg class!");
1161    break;
1162   case 32:
1163    if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
1164      if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
1165        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
1166                     .addFrameIndex(FI).addImm(16)
1167                     .addMemOperand(MMO));
1168      } else {
1169        MachineInstrBuilder MIB =
1170        AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1171                       .addFrameIndex(FI))
1172                       .addMemOperand(MMO);
1173        MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1174        MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1175        MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1176        MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1177        if (TargetRegisterInfo::isPhysicalRegister(DestReg))
1178          MIB.addReg(DestReg, RegState::ImplicitDefine);
1179      }
1180    } else
1181      llvm_unreachable("Unknown reg class!");
1182    break;
1183  case 64:
1184    if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1185      MachineInstrBuilder MIB =
1186      AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1187                     .addFrameIndex(FI))
1188                     .addMemOperand(MMO);
1189      MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1190      MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1191      MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1192      MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1193      MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::DefineNoRead, TRI);
1194      MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::DefineNoRead, TRI);
1195      MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::DefineNoRead, TRI);
1196      MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::DefineNoRead, TRI);
1197      if (TargetRegisterInfo::isPhysicalRegister(DestReg))
1198        MIB.addReg(DestReg, RegState::ImplicitDefine);
1199    } else
1200      llvm_unreachable("Unknown reg class!");
1201    break;
1202  default:
1203    llvm_unreachable("Unknown regclass!");
1204  }
1205}
1206
1207unsigned
1208ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
1209                                      int &FrameIndex) const {
1210  switch (MI->getOpcode()) {
1211  default: break;
1212  case ARM::LDRrs:
1213  case ARM::t2LDRs:  // FIXME: don't use t2LDRs to access frame.
1214    if (MI->getOperand(1).isFI() &&
1215        MI->getOperand(2).isReg() &&
1216        MI->getOperand(3).isImm() &&
1217        MI->getOperand(2).getReg() == 0 &&
1218        MI->getOperand(3).getImm() == 0) {
1219      FrameIndex = MI->getOperand(1).getIndex();
1220      return MI->getOperand(0).getReg();
1221    }
1222    break;
1223  case ARM::LDRi12:
1224  case ARM::t2LDRi12:
1225  case ARM::tLDRspi:
1226  case ARM::VLDRD:
1227  case ARM::VLDRS:
1228    if (MI->getOperand(1).isFI() &&
1229        MI->getOperand(2).isImm() &&
1230        MI->getOperand(2).getImm() == 0) {
1231      FrameIndex = MI->getOperand(1).getIndex();
1232      return MI->getOperand(0).getReg();
1233    }
1234    break;
1235  case ARM::VLD1q64:
1236  case ARM::VLD1d64TPseudo:
1237  case ARM::VLD1d64QPseudo:
1238    if (MI->getOperand(1).isFI() &&
1239        MI->getOperand(0).getSubReg() == 0) {
1240      FrameIndex = MI->getOperand(1).getIndex();
1241      return MI->getOperand(0).getReg();
1242    }
1243    break;
1244  case ARM::VLDMQIA:
1245    if (MI->getOperand(1).isFI() &&
1246        MI->getOperand(0).getSubReg() == 0) {
1247      FrameIndex = MI->getOperand(1).getIndex();
1248      return MI->getOperand(0).getReg();
1249    }
1250    break;
1251  }
1252
1253  return 0;
1254}
1255
1256unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr *MI,
1257                                             int &FrameIndex) const {
1258  const MachineMemOperand *Dummy;
1259  return MI->mayLoad() && hasLoadFromStackSlot(MI, Dummy, FrameIndex);
1260}
1261
1262bool
1263ARMBaseInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
1264  MachineFunction &MF = *MI->getParent()->getParent();
1265  Reloc::Model RM = MF.getTarget().getRelocationModel();
1266
1267  if (MI->getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
1268    assert(getSubtarget().getTargetTriple().getObjectFormat() ==
1269           Triple::MachO &&
1270           "LOAD_STACK_GUARD currently supported only for MachO.");
1271    expandLoadStackGuard(MI, RM);
1272    MI->getParent()->erase(MI);
1273    return true;
1274  }
1275
1276  // This hook gets to expand COPY instructions before they become
1277  // copyPhysReg() calls.  Look for VMOVS instructions that can legally be
1278  // widened to VMOVD.  We prefer the VMOVD when possible because it may be
1279  // changed into a VORR that can go down the NEON pipeline.
1280  if (!WidenVMOVS || !MI->isCopy() || Subtarget.isCortexA15() ||
1281      Subtarget.isFPOnlySP())
1282    return false;
1283
1284  // Look for a copy between even S-registers.  That is where we keep floats
1285  // when using NEON v2f32 instructions for f32 arithmetic.
1286  unsigned DstRegS = MI->getOperand(0).getReg();
1287  unsigned SrcRegS = MI->getOperand(1).getReg();
1288  if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS))
1289    return false;
1290
1291  const TargetRegisterInfo *TRI = &getRegisterInfo();
1292  unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0,
1293                                              &ARM::DPRRegClass);
1294  unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0,
1295                                              &ARM::DPRRegClass);
1296  if (!DstRegD || !SrcRegD)
1297    return false;
1298
1299  // We want to widen this into a DstRegD = VMOVD SrcRegD copy.  This is only
1300  // legal if the COPY already defines the full DstRegD, and it isn't a
1301  // sub-register insertion.
1302  if (!MI->definesRegister(DstRegD, TRI) || MI->readsRegister(DstRegD, TRI))
1303    return false;
1304
1305  // A dead copy shouldn't show up here, but reject it just in case.
1306  if (MI->getOperand(0).isDead())
1307    return false;
1308
1309  // All clear, widen the COPY.
1310  DEBUG(dbgs() << "widening:    " << *MI);
1311  MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI);
1312
1313  // Get rid of the old <imp-def> of DstRegD.  Leave it if it defines a Q-reg
1314  // or some other super-register.
1315  int ImpDefIdx = MI->findRegisterDefOperandIdx(DstRegD);
1316  if (ImpDefIdx != -1)
1317    MI->RemoveOperand(ImpDefIdx);
1318
1319  // Change the opcode and operands.
1320  MI->setDesc(get(ARM::VMOVD));
1321  MI->getOperand(0).setReg(DstRegD);
1322  MI->getOperand(1).setReg(SrcRegD);
1323  AddDefaultPred(MIB);
1324
1325  // We are now reading SrcRegD instead of SrcRegS.  This may upset the
1326  // register scavenger and machine verifier, so we need to indicate that we
1327  // are reading an undefined value from SrcRegD, but a proper value from
1328  // SrcRegS.
1329  MI->getOperand(1).setIsUndef();
1330  MIB.addReg(SrcRegS, RegState::Implicit);
1331
1332  // SrcRegD may actually contain an unrelated value in the ssub_1
1333  // sub-register.  Don't kill it.  Only kill the ssub_0 sub-register.
1334  if (MI->getOperand(1).isKill()) {
1335    MI->getOperand(1).setIsKill(false);
1336    MI->addRegisterKilled(SrcRegS, TRI, true);
1337  }
1338
1339  DEBUG(dbgs() << "replaced by: " << *MI);
1340  return true;
1341}
1342
1343/// Create a copy of a const pool value. Update CPI to the new index and return
1344/// the label UID.
1345static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
1346  MachineConstantPool *MCP = MF.getConstantPool();
1347  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1348
1349  const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI];
1350  assert(MCPE.isMachineConstantPoolEntry() &&
1351         "Expecting a machine constantpool entry!");
1352  ARMConstantPoolValue *ACPV =
1353    static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
1354
1355  unsigned PCLabelId = AFI->createPICLabelUId();
1356  ARMConstantPoolValue *NewCPV = nullptr;
1357
1358  // FIXME: The below assumes PIC relocation model and that the function
1359  // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and
1360  // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR
1361  // instructions, so that's probably OK, but is PIC always correct when
1362  // we get here?
1363  if (ACPV->isGlobalValue())
1364    NewCPV = ARMConstantPoolConstant::
1365      Create(cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId,
1366             ARMCP::CPValue, 4);
1367  else if (ACPV->isExtSymbol())
1368    NewCPV = ARMConstantPoolSymbol::
1369      Create(MF.getFunction()->getContext(),
1370             cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4);
1371  else if (ACPV->isBlockAddress())
1372    NewCPV = ARMConstantPoolConstant::
1373      Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId,
1374             ARMCP::CPBlockAddress, 4);
1375  else if (ACPV->isLSDA())
1376    NewCPV = ARMConstantPoolConstant::Create(MF.getFunction(), PCLabelId,
1377                                             ARMCP::CPLSDA, 4);
1378  else if (ACPV->isMachineBasicBlock())
1379    NewCPV = ARMConstantPoolMBB::
1380      Create(MF.getFunction()->getContext(),
1381             cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4);
1382  else
1383    llvm_unreachable("Unexpected ARM constantpool value type!!");
1384  CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlignment());
1385  return PCLabelId;
1386}
1387
1388void ARMBaseInstrInfo::
1389reMaterialize(MachineBasicBlock &MBB,
1390              MachineBasicBlock::iterator I,
1391              unsigned DestReg, unsigned SubIdx,
1392              const MachineInstr *Orig,
1393              const TargetRegisterInfo &TRI) const {
1394  unsigned Opcode = Orig->getOpcode();
1395  switch (Opcode) {
1396  default: {
1397    MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig);
1398    MI->substituteRegister(Orig->getOperand(0).getReg(), DestReg, SubIdx, TRI);
1399    MBB.insert(I, MI);
1400    break;
1401  }
1402  case ARM::tLDRpci_pic:
1403  case ARM::t2LDRpci_pic: {
1404    MachineFunction &MF = *MBB.getParent();
1405    unsigned CPI = Orig->getOperand(1).getIndex();
1406    unsigned PCLabelId = duplicateCPV(MF, CPI);
1407    MachineInstrBuilder MIB = BuildMI(MBB, I, Orig->getDebugLoc(), get(Opcode),
1408                                      DestReg)
1409      .addConstantPoolIndex(CPI).addImm(PCLabelId);
1410    MIB->setMemRefs(Orig->memoperands_begin(), Orig->memoperands_end());
1411    break;
1412  }
1413  }
1414}
1415
1416MachineInstr *
1417ARMBaseInstrInfo::duplicate(MachineInstr *Orig, MachineFunction &MF) const {
1418  MachineInstr *MI = TargetInstrInfo::duplicate(Orig, MF);
1419  switch(Orig->getOpcode()) {
1420  case ARM::tLDRpci_pic:
1421  case ARM::t2LDRpci_pic: {
1422    unsigned CPI = Orig->getOperand(1).getIndex();
1423    unsigned PCLabelId = duplicateCPV(MF, CPI);
1424    Orig->getOperand(1).setIndex(CPI);
1425    Orig->getOperand(2).setImm(PCLabelId);
1426    break;
1427  }
1428  }
1429  return MI;
1430}
1431
1432bool ARMBaseInstrInfo::produceSameValue(const MachineInstr *MI0,
1433                                        const MachineInstr *MI1,
1434                                        const MachineRegisterInfo *MRI) const {
1435  int Opcode = MI0->getOpcode();
1436  if (Opcode == ARM::t2LDRpci ||
1437      Opcode == ARM::t2LDRpci_pic ||
1438      Opcode == ARM::tLDRpci ||
1439      Opcode == ARM::tLDRpci_pic ||
1440      Opcode == ARM::LDRLIT_ga_pcrel ||
1441      Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1442      Opcode == ARM::tLDRLIT_ga_pcrel ||
1443      Opcode == ARM::MOV_ga_pcrel ||
1444      Opcode == ARM::MOV_ga_pcrel_ldr ||
1445      Opcode == ARM::t2MOV_ga_pcrel) {
1446    if (MI1->getOpcode() != Opcode)
1447      return false;
1448    if (MI0->getNumOperands() != MI1->getNumOperands())
1449      return false;
1450
1451    const MachineOperand &MO0 = MI0->getOperand(1);
1452    const MachineOperand &MO1 = MI1->getOperand(1);
1453    if (MO0.getOffset() != MO1.getOffset())
1454      return false;
1455
1456    if (Opcode == ARM::LDRLIT_ga_pcrel ||
1457        Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1458        Opcode == ARM::tLDRLIT_ga_pcrel ||
1459        Opcode == ARM::MOV_ga_pcrel ||
1460        Opcode == ARM::MOV_ga_pcrel_ldr ||
1461        Opcode == ARM::t2MOV_ga_pcrel)
1462      // Ignore the PC labels.
1463      return MO0.getGlobal() == MO1.getGlobal();
1464
1465    const MachineFunction *MF = MI0->getParent()->getParent();
1466    const MachineConstantPool *MCP = MF->getConstantPool();
1467    int CPI0 = MO0.getIndex();
1468    int CPI1 = MO1.getIndex();
1469    const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0];
1470    const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1];
1471    bool isARMCP0 = MCPE0.isMachineConstantPoolEntry();
1472    bool isARMCP1 = MCPE1.isMachineConstantPoolEntry();
1473    if (isARMCP0 && isARMCP1) {
1474      ARMConstantPoolValue *ACPV0 =
1475        static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal);
1476      ARMConstantPoolValue *ACPV1 =
1477        static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal);
1478      return ACPV0->hasSameValue(ACPV1);
1479    } else if (!isARMCP0 && !isARMCP1) {
1480      return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal;
1481    }
1482    return false;
1483  } else if (Opcode == ARM::PICLDR) {
1484    if (MI1->getOpcode() != Opcode)
1485      return false;
1486    if (MI0->getNumOperands() != MI1->getNumOperands())
1487      return false;
1488
1489    unsigned Addr0 = MI0->getOperand(1).getReg();
1490    unsigned Addr1 = MI1->getOperand(1).getReg();
1491    if (Addr0 != Addr1) {
1492      if (!MRI ||
1493          !TargetRegisterInfo::isVirtualRegister(Addr0) ||
1494          !TargetRegisterInfo::isVirtualRegister(Addr1))
1495        return false;
1496
1497      // This assumes SSA form.
1498      MachineInstr *Def0 = MRI->getVRegDef(Addr0);
1499      MachineInstr *Def1 = MRI->getVRegDef(Addr1);
1500      // Check if the loaded value, e.g. a constantpool of a global address, are
1501      // the same.
1502      if (!produceSameValue(Def0, Def1, MRI))
1503        return false;
1504    }
1505
1506    for (unsigned i = 3, e = MI0->getNumOperands(); i != e; ++i) {
1507      // %vreg12<def> = PICLDR %vreg11, 0, pred:14, pred:%noreg
1508      const MachineOperand &MO0 = MI0->getOperand(i);
1509      const MachineOperand &MO1 = MI1->getOperand(i);
1510      if (!MO0.isIdenticalTo(MO1))
1511        return false;
1512    }
1513    return true;
1514  }
1515
1516  return MI0->isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs);
1517}
1518
1519/// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
1520/// determine if two loads are loading from the same base address. It should
1521/// only return true if the base pointers are the same and the only differences
1522/// between the two addresses is the offset. It also returns the offsets by
1523/// reference.
1524///
1525/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1526/// is permanently disabled.
1527bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
1528                                               int64_t &Offset1,
1529                                               int64_t &Offset2) const {
1530  // Don't worry about Thumb: just ARM and Thumb2.
1531  if (Subtarget.isThumb1Only()) return false;
1532
1533  if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
1534    return false;
1535
1536  switch (Load1->getMachineOpcode()) {
1537  default:
1538    return false;
1539  case ARM::LDRi12:
1540  case ARM::LDRBi12:
1541  case ARM::LDRD:
1542  case ARM::LDRH:
1543  case ARM::LDRSB:
1544  case ARM::LDRSH:
1545  case ARM::VLDRD:
1546  case ARM::VLDRS:
1547  case ARM::t2LDRi8:
1548  case ARM::t2LDRBi8:
1549  case ARM::t2LDRDi8:
1550  case ARM::t2LDRSHi8:
1551  case ARM::t2LDRi12:
1552  case ARM::t2LDRBi12:
1553  case ARM::t2LDRSHi12:
1554    break;
1555  }
1556
1557  switch (Load2->getMachineOpcode()) {
1558  default:
1559    return false;
1560  case ARM::LDRi12:
1561  case ARM::LDRBi12:
1562  case ARM::LDRD:
1563  case ARM::LDRH:
1564  case ARM::LDRSB:
1565  case ARM::LDRSH:
1566  case ARM::VLDRD:
1567  case ARM::VLDRS:
1568  case ARM::t2LDRi8:
1569  case ARM::t2LDRBi8:
1570  case ARM::t2LDRSHi8:
1571  case ARM::t2LDRi12:
1572  case ARM::t2LDRBi12:
1573  case ARM::t2LDRSHi12:
1574    break;
1575  }
1576
1577  // Check if base addresses and chain operands match.
1578  if (Load1->getOperand(0) != Load2->getOperand(0) ||
1579      Load1->getOperand(4) != Load2->getOperand(4))
1580    return false;
1581
1582  // Index should be Reg0.
1583  if (Load1->getOperand(3) != Load2->getOperand(3))
1584    return false;
1585
1586  // Determine the offsets.
1587  if (isa<ConstantSDNode>(Load1->getOperand(1)) &&
1588      isa<ConstantSDNode>(Load2->getOperand(1))) {
1589    Offset1 = cast<ConstantSDNode>(Load1->getOperand(1))->getSExtValue();
1590    Offset2 = cast<ConstantSDNode>(Load2->getOperand(1))->getSExtValue();
1591    return true;
1592  }
1593
1594  return false;
1595}
1596
1597/// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
1598/// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
1599/// be scheduled togther. On some targets if two loads are loading from
1600/// addresses in the same cache line, it's better if they are scheduled
1601/// together. This function takes two integers that represent the load offsets
1602/// from the common base address. It returns true if it decides it's desirable
1603/// to schedule the two loads together. "NumLoads" is the number of loads that
1604/// have already been scheduled after Load1.
1605///
1606/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1607/// is permanently disabled.
1608bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
1609                                               int64_t Offset1, int64_t Offset2,
1610                                               unsigned NumLoads) const {
1611  // Don't worry about Thumb: just ARM and Thumb2.
1612  if (Subtarget.isThumb1Only()) return false;
1613
1614  assert(Offset2 > Offset1);
1615
1616  if ((Offset2 - Offset1) / 8 > 64)
1617    return false;
1618
1619  // Check if the machine opcodes are different. If they are different
1620  // then we consider them to not be of the same base address,
1621  // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12.
1622  // In this case, they are considered to be the same because they are different
1623  // encoding forms of the same basic instruction.
1624  if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) &&
1625      !((Load1->getMachineOpcode() == ARM::t2LDRBi8 &&
1626         Load2->getMachineOpcode() == ARM::t2LDRBi12) ||
1627        (Load1->getMachineOpcode() == ARM::t2LDRBi12 &&
1628         Load2->getMachineOpcode() == ARM::t2LDRBi8)))
1629    return false;  // FIXME: overly conservative?
1630
1631  // Four loads in a row should be sufficient.
1632  if (NumLoads >= 3)
1633    return false;
1634
1635  return true;
1636}
1637
1638bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr *MI,
1639                                            const MachineBasicBlock *MBB,
1640                                            const MachineFunction &MF) const {
1641  // Debug info is never a scheduling boundary. It's necessary to be explicit
1642  // due to the special treatment of IT instructions below, otherwise a
1643  // dbg_value followed by an IT will result in the IT instruction being
1644  // considered a scheduling hazard, which is wrong. It should be the actual
1645  // instruction preceding the dbg_value instruction(s), just like it is
1646  // when debug info is not present.
1647  if (MI->isDebugValue())
1648    return false;
1649
1650  // Terminators and labels can't be scheduled around.
1651  if (MI->isTerminator() || MI->isPosition())
1652    return true;
1653
1654  // Treat the start of the IT block as a scheduling boundary, but schedule
1655  // t2IT along with all instructions following it.
1656  // FIXME: This is a big hammer. But the alternative is to add all potential
1657  // true and anti dependencies to IT block instructions as implicit operands
1658  // to the t2IT instruction. The added compile time and complexity does not
1659  // seem worth it.
1660  MachineBasicBlock::const_iterator I = MI;
1661  // Make sure to skip any dbg_value instructions
1662  while (++I != MBB->end() && I->isDebugValue())
1663    ;
1664  if (I != MBB->end() && I->getOpcode() == ARM::t2IT)
1665    return true;
1666
1667  // Don't attempt to schedule around any instruction that defines
1668  // a stack-oriented pointer, as it's unlikely to be profitable. This
1669  // saves compile time, because it doesn't require every single
1670  // stack slot reference to depend on the instruction that does the
1671  // modification.
1672  // Calls don't actually change the stack pointer, even if they have imp-defs.
1673  // No ARM calling conventions change the stack pointer. (X86 calling
1674  // conventions sometimes do).
1675  if (!MI->isCall() && MI->definesRegister(ARM::SP))
1676    return true;
1677
1678  return false;
1679}
1680
1681bool ARMBaseInstrInfo::
1682isProfitableToIfCvt(MachineBasicBlock &MBB,
1683                    unsigned NumCycles, unsigned ExtraPredCycles,
1684                    const BranchProbability &Probability) const {
1685  if (!NumCycles)
1686    return false;
1687
1688  // Attempt to estimate the relative costs of predication versus branching.
1689  unsigned UnpredCost = Probability.getNumerator() * NumCycles;
1690  UnpredCost /= Probability.getDenominator();
1691  UnpredCost += 1; // The branch itself
1692  UnpredCost += Subtarget.getMispredictionPenalty() / 10;
1693
1694  return (NumCycles + ExtraPredCycles) <= UnpredCost;
1695}
1696
1697bool ARMBaseInstrInfo::
1698isProfitableToIfCvt(MachineBasicBlock &TMBB,
1699                    unsigned TCycles, unsigned TExtra,
1700                    MachineBasicBlock &FMBB,
1701                    unsigned FCycles, unsigned FExtra,
1702                    const BranchProbability &Probability) const {
1703  if (!TCycles || !FCycles)
1704    return false;
1705
1706  // Attempt to estimate the relative costs of predication versus branching.
1707  unsigned TUnpredCost = Probability.getNumerator() * TCycles;
1708  TUnpredCost /= Probability.getDenominator();
1709
1710  uint32_t Comp = Probability.getDenominator() - Probability.getNumerator();
1711  unsigned FUnpredCost = Comp * FCycles;
1712  FUnpredCost /= Probability.getDenominator();
1713
1714  unsigned UnpredCost = TUnpredCost + FUnpredCost;
1715  UnpredCost += 1; // The branch itself
1716  UnpredCost += Subtarget.getMispredictionPenalty() / 10;
1717
1718  return (TCycles + FCycles + TExtra + FExtra) <= UnpredCost;
1719}
1720
1721bool
1722ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
1723                                            MachineBasicBlock &FMBB) const {
1724  // Reduce false anti-dependencies to let Swift's out-of-order execution
1725  // engine do its thing.
1726  return Subtarget.isSwift();
1727}
1728
1729/// getInstrPredicate - If instruction is predicated, returns its predicate
1730/// condition, otherwise returns AL. It also returns the condition code
1731/// register by reference.
1732ARMCC::CondCodes
1733llvm::getInstrPredicate(const MachineInstr *MI, unsigned &PredReg) {
1734  int PIdx = MI->findFirstPredOperandIdx();
1735  if (PIdx == -1) {
1736    PredReg = 0;
1737    return ARMCC::AL;
1738  }
1739
1740  PredReg = MI->getOperand(PIdx+1).getReg();
1741  return (ARMCC::CondCodes)MI->getOperand(PIdx).getImm();
1742}
1743
1744
1745int llvm::getMatchingCondBranchOpcode(int Opc) {
1746  if (Opc == ARM::B)
1747    return ARM::Bcc;
1748  if (Opc == ARM::tB)
1749    return ARM::tBcc;
1750  if (Opc == ARM::t2B)
1751    return ARM::t2Bcc;
1752
1753  llvm_unreachable("Unknown unconditional branch opcode!");
1754}
1755
1756/// commuteInstruction - Handle commutable instructions.
1757MachineInstr *
1758ARMBaseInstrInfo::commuteInstruction(MachineInstr *MI, bool NewMI) const {
1759  switch (MI->getOpcode()) {
1760  case ARM::MOVCCr:
1761  case ARM::t2MOVCCr: {
1762    // MOVCC can be commuted by inverting the condition.
1763    unsigned PredReg = 0;
1764    ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg);
1765    // MOVCC AL can't be inverted. Shouldn't happen.
1766    if (CC == ARMCC::AL || PredReg != ARM::CPSR)
1767      return nullptr;
1768    MI = TargetInstrInfo::commuteInstruction(MI, NewMI);
1769    if (!MI)
1770      return nullptr;
1771    // After swapping the MOVCC operands, also invert the condition.
1772    MI->getOperand(MI->findFirstPredOperandIdx())
1773      .setImm(ARMCC::getOppositeCondition(CC));
1774    return MI;
1775  }
1776  }
1777  return TargetInstrInfo::commuteInstruction(MI, NewMI);
1778}
1779
1780/// Identify instructions that can be folded into a MOVCC instruction, and
1781/// return the defining instruction.
1782static MachineInstr *canFoldIntoMOVCC(unsigned Reg,
1783                                      const MachineRegisterInfo &MRI,
1784                                      const TargetInstrInfo *TII) {
1785  if (!TargetRegisterInfo::isVirtualRegister(Reg))
1786    return nullptr;
1787  if (!MRI.hasOneNonDBGUse(Reg))
1788    return nullptr;
1789  MachineInstr *MI = MRI.getVRegDef(Reg);
1790  if (!MI)
1791    return nullptr;
1792  // MI is folded into the MOVCC by predicating it.
1793  if (!MI->isPredicable())
1794    return nullptr;
1795  // Check if MI has any non-dead defs or physreg uses. This also detects
1796  // predicated instructions which will be reading CPSR.
1797  for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) {
1798    const MachineOperand &MO = MI->getOperand(i);
1799    // Reject frame index operands, PEI can't handle the predicated pseudos.
1800    if (MO.isFI() || MO.isCPI() || MO.isJTI())
1801      return nullptr;
1802    if (!MO.isReg())
1803      continue;
1804    // MI can't have any tied operands, that would conflict with predication.
1805    if (MO.isTied())
1806      return nullptr;
1807    if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()))
1808      return nullptr;
1809    if (MO.isDef() && !MO.isDead())
1810      return nullptr;
1811  }
1812  bool DontMoveAcrossStores = true;
1813  if (!MI->isSafeToMove(TII, /* AliasAnalysis = */ nullptr,
1814                        DontMoveAcrossStores))
1815    return nullptr;
1816  return MI;
1817}
1818
1819bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr *MI,
1820                                     SmallVectorImpl<MachineOperand> &Cond,
1821                                     unsigned &TrueOp, unsigned &FalseOp,
1822                                     bool &Optimizable) const {
1823  assert((MI->getOpcode() == ARM::MOVCCr || MI->getOpcode() == ARM::t2MOVCCr) &&
1824         "Unknown select instruction");
1825  // MOVCC operands:
1826  // 0: Def.
1827  // 1: True use.
1828  // 2: False use.
1829  // 3: Condition code.
1830  // 4: CPSR use.
1831  TrueOp = 1;
1832  FalseOp = 2;
1833  Cond.push_back(MI->getOperand(3));
1834  Cond.push_back(MI->getOperand(4));
1835  // We can always fold a def.
1836  Optimizable = true;
1837  return false;
1838}
1839
1840MachineInstr *
1841ARMBaseInstrInfo::optimizeSelect(MachineInstr *MI,
1842                                 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
1843                                 bool PreferFalse) const {
1844  assert((MI->getOpcode() == ARM::MOVCCr || MI->getOpcode() == ARM::t2MOVCCr) &&
1845         "Unknown select instruction");
1846  MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo();
1847  MachineInstr *DefMI = canFoldIntoMOVCC(MI->getOperand(2).getReg(), MRI, this);
1848  bool Invert = !DefMI;
1849  if (!DefMI)
1850    DefMI = canFoldIntoMOVCC(MI->getOperand(1).getReg(), MRI, this);
1851  if (!DefMI)
1852    return nullptr;
1853
1854  // Find new register class to use.
1855  MachineOperand FalseReg = MI->getOperand(Invert ? 2 : 1);
1856  unsigned       DestReg  = MI->getOperand(0).getReg();
1857  const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
1858  if (!MRI.constrainRegClass(DestReg, PreviousClass))
1859    return nullptr;
1860
1861  // Create a new predicated version of DefMI.
1862  // Rfalse is the first use.
1863  MachineInstrBuilder NewMI = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1864                                      DefMI->getDesc(), DestReg);
1865
1866  // Copy all the DefMI operands, excluding its (null) predicate.
1867  const MCInstrDesc &DefDesc = DefMI->getDesc();
1868  for (unsigned i = 1, e = DefDesc.getNumOperands();
1869       i != e && !DefDesc.OpInfo[i].isPredicate(); ++i)
1870    NewMI.addOperand(DefMI->getOperand(i));
1871
1872  unsigned CondCode = MI->getOperand(3).getImm();
1873  if (Invert)
1874    NewMI.addImm(ARMCC::getOppositeCondition(ARMCC::CondCodes(CondCode)));
1875  else
1876    NewMI.addImm(CondCode);
1877  NewMI.addOperand(MI->getOperand(4));
1878
1879  // DefMI is not the -S version that sets CPSR, so add an optional %noreg.
1880  if (NewMI->hasOptionalDef())
1881    AddDefaultCC(NewMI);
1882
1883  // The output register value when the predicate is false is an implicit
1884  // register operand tied to the first def.
1885  // The tie makes the register allocator ensure the FalseReg is allocated the
1886  // same register as operand 0.
1887  FalseReg.setImplicit();
1888  NewMI.addOperand(FalseReg);
1889  NewMI->tieOperands(0, NewMI->getNumOperands() - 1);
1890
1891  // Update SeenMIs set: register newly created MI and erase removed DefMI.
1892  SeenMIs.insert(NewMI);
1893  SeenMIs.erase(DefMI);
1894
1895  // The caller will erase MI, but not DefMI.
1896  DefMI->eraseFromParent();
1897  return NewMI;
1898}
1899
1900/// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
1901/// instruction is encoded with an 'S' bit is determined by the optional CPSR
1902/// def operand.
1903///
1904/// This will go away once we can teach tblgen how to set the optional CPSR def
1905/// operand itself.
1906struct AddSubFlagsOpcodePair {
1907  uint16_t PseudoOpc;
1908  uint16_t MachineOpc;
1909};
1910
1911static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
1912  {ARM::ADDSri, ARM::ADDri},
1913  {ARM::ADDSrr, ARM::ADDrr},
1914  {ARM::ADDSrsi, ARM::ADDrsi},
1915  {ARM::ADDSrsr, ARM::ADDrsr},
1916
1917  {ARM::SUBSri, ARM::SUBri},
1918  {ARM::SUBSrr, ARM::SUBrr},
1919  {ARM::SUBSrsi, ARM::SUBrsi},
1920  {ARM::SUBSrsr, ARM::SUBrsr},
1921
1922  {ARM::RSBSri, ARM::RSBri},
1923  {ARM::RSBSrsi, ARM::RSBrsi},
1924  {ARM::RSBSrsr, ARM::RSBrsr},
1925
1926  {ARM::t2ADDSri, ARM::t2ADDri},
1927  {ARM::t2ADDSrr, ARM::t2ADDrr},
1928  {ARM::t2ADDSrs, ARM::t2ADDrs},
1929
1930  {ARM::t2SUBSri, ARM::t2SUBri},
1931  {ARM::t2SUBSrr, ARM::t2SUBrr},
1932  {ARM::t2SUBSrs, ARM::t2SUBrs},
1933
1934  {ARM::t2RSBSri, ARM::t2RSBri},
1935  {ARM::t2RSBSrs, ARM::t2RSBrs},
1936};
1937
1938unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
1939  for (unsigned i = 0, e = array_lengthof(AddSubFlagsOpcodeMap); i != e; ++i)
1940    if (OldOpc == AddSubFlagsOpcodeMap[i].PseudoOpc)
1941      return AddSubFlagsOpcodeMap[i].MachineOpc;
1942  return 0;
1943}
1944
1945void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
1946                               MachineBasicBlock::iterator &MBBI, DebugLoc dl,
1947                               unsigned DestReg, unsigned BaseReg, int NumBytes,
1948                               ARMCC::CondCodes Pred, unsigned PredReg,
1949                               const ARMBaseInstrInfo &TII, unsigned MIFlags) {
1950  if (NumBytes == 0 && DestReg != BaseReg) {
1951    BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), DestReg)
1952      .addReg(BaseReg, RegState::Kill)
1953      .addImm((unsigned)Pred).addReg(PredReg).addReg(0)
1954      .setMIFlags(MIFlags);
1955    return;
1956  }
1957
1958  bool isSub = NumBytes < 0;
1959  if (isSub) NumBytes = -NumBytes;
1960
1961  while (NumBytes) {
1962    unsigned RotAmt = ARM_AM::getSOImmValRotate(NumBytes);
1963    unsigned ThisVal = NumBytes & ARM_AM::rotr32(0xFF, RotAmt);
1964    assert(ThisVal && "Didn't extract field correctly");
1965
1966    // We will handle these bits from offset, clear them.
1967    NumBytes &= ~ThisVal;
1968
1969    assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?");
1970
1971    // Build the new ADD / SUB.
1972    unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri;
1973    BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg)
1974      .addReg(BaseReg, RegState::Kill).addImm(ThisVal)
1975      .addImm((unsigned)Pred).addReg(PredReg).addReg(0)
1976      .setMIFlags(MIFlags);
1977    BaseReg = DestReg;
1978  }
1979}
1980
1981static bool isAnySubRegLive(unsigned Reg, const TargetRegisterInfo *TRI,
1982                      MachineInstr *MI) {
1983  for (MCSubRegIterator Subreg(Reg, TRI, /* IncludeSelf */ true);
1984       Subreg.isValid(); ++Subreg)
1985    if (MI->getParent()->computeRegisterLiveness(TRI, *Subreg, MI) !=
1986        MachineBasicBlock::LQR_Dead)
1987      return true;
1988  return false;
1989}
1990bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
1991                                      MachineFunction &MF, MachineInstr *MI,
1992                                      unsigned NumBytes) {
1993  // This optimisation potentially adds lots of load and store
1994  // micro-operations, it's only really a great benefit to code-size.
1995  if (!MF.getFunction()->hasFnAttribute(Attribute::MinSize))
1996    return false;
1997
1998  // If only one register is pushed/popped, LLVM can use an LDR/STR
1999  // instead. We can't modify those so make sure we're dealing with an
2000  // instruction we understand.
2001  bool IsPop = isPopOpcode(MI->getOpcode());
2002  bool IsPush = isPushOpcode(MI->getOpcode());
2003  if (!IsPush && !IsPop)
2004    return false;
2005
2006  bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD ||
2007                      MI->getOpcode() == ARM::VLDMDIA_UPD;
2008  bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH ||
2009                     MI->getOpcode() == ARM::tPOP ||
2010                     MI->getOpcode() == ARM::tPOP_RET;
2011
2012  assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP &&
2013                          MI->getOperand(1).getReg() == ARM::SP)) &&
2014         "trying to fold sp update into non-sp-updating push/pop");
2015
2016  // The VFP push & pop act on D-registers, so we can only fold an adjustment
2017  // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try
2018  // if this is violated.
2019  if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0)
2020    return false;
2021
2022  // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
2023  // pred) so the list starts at 4. Thumb1 starts after the predicate.
2024  int RegListIdx = IsT1PushPop ? 2 : 4;
2025
2026  // Calculate the space we'll need in terms of registers.
2027  unsigned FirstReg = MI->getOperand(RegListIdx).getReg();
2028  unsigned RD0Reg, RegsNeeded;
2029  if (IsVFPPushPop) {
2030    RD0Reg = ARM::D0;
2031    RegsNeeded = NumBytes / 8;
2032  } else {
2033    RD0Reg = ARM::R0;
2034    RegsNeeded = NumBytes / 4;
2035  }
2036
2037  // We're going to have to strip all list operands off before
2038  // re-adding them since the order matters, so save the existing ones
2039  // for later.
2040  SmallVector<MachineOperand, 4> RegList;
2041  for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2042    RegList.push_back(MI->getOperand(i));
2043
2044  const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo();
2045  const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
2046
2047  // Now try to find enough space in the reglist to allocate NumBytes.
2048  for (unsigned CurReg = FirstReg - 1; CurReg >= RD0Reg && RegsNeeded;
2049       --CurReg) {
2050    if (!IsPop) {
2051      // Pushing any register is completely harmless, mark the
2052      // register involved as undef since we don't care about it in
2053      // the slightest.
2054      RegList.push_back(MachineOperand::CreateReg(CurReg, false, false,
2055                                                  false, false, true));
2056      --RegsNeeded;
2057      continue;
2058    }
2059
2060    // However, we can only pop an extra register if it's not live. For
2061    // registers live within the function we might clobber a return value
2062    // register; the other way a register can be live here is if it's
2063    // callee-saved.
2064    // TODO: Currently, computeRegisterLiveness() does not report "live" if a
2065    // sub reg is live. When computeRegisterLiveness() works for sub reg, it
2066    // can replace isAnySubRegLive().
2067    if (isCalleeSavedRegister(CurReg, CSRegs) ||
2068        isAnySubRegLive(CurReg, TRI, MI)) {
2069      // VFP pops don't allow holes in the register list, so any skip is fatal
2070      // for our transformation. GPR pops do, so we should just keep looking.
2071      if (IsVFPPushPop)
2072        return false;
2073      else
2074        continue;
2075    }
2076
2077    // Mark the unimportant registers as <def,dead> in the POP.
2078    RegList.push_back(MachineOperand::CreateReg(CurReg, true, false, false,
2079                                                true));
2080    --RegsNeeded;
2081  }
2082
2083  if (RegsNeeded > 0)
2084    return false;
2085
2086  // Finally we know we can profitably perform the optimisation so go
2087  // ahead: strip all existing registers off and add them back again
2088  // in the right order.
2089  for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2090    MI->RemoveOperand(i);
2091
2092  // Add the complete list back in.
2093  MachineInstrBuilder MIB(MF, &*MI);
2094  for (int i = RegList.size() - 1; i >= 0; --i)
2095    MIB.addOperand(RegList[i]);
2096
2097  return true;
2098}
2099
2100bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2101                                unsigned FrameReg, int &Offset,
2102                                const ARMBaseInstrInfo &TII) {
2103  unsigned Opcode = MI.getOpcode();
2104  const MCInstrDesc &Desc = MI.getDesc();
2105  unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
2106  bool isSub = false;
2107
2108  // Memory operands in inline assembly always use AddrMode2.
2109  if (Opcode == ARM::INLINEASM)
2110    AddrMode = ARMII::AddrMode2;
2111
2112  if (Opcode == ARM::ADDri) {
2113    Offset += MI.getOperand(FrameRegIdx+1).getImm();
2114    if (Offset == 0) {
2115      // Turn it into a move.
2116      MI.setDesc(TII.get(ARM::MOVr));
2117      MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2118      MI.RemoveOperand(FrameRegIdx+1);
2119      Offset = 0;
2120      return true;
2121    } else if (Offset < 0) {
2122      Offset = -Offset;
2123      isSub = true;
2124      MI.setDesc(TII.get(ARM::SUBri));
2125    }
2126
2127    // Common case: small offset, fits into instruction.
2128    if (ARM_AM::getSOImmVal(Offset) != -1) {
2129      // Replace the FrameIndex with sp / fp
2130      MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2131      MI.getOperand(FrameRegIdx+1).ChangeToImmediate(Offset);
2132      Offset = 0;
2133      return true;
2134    }
2135
2136    // Otherwise, pull as much of the immedidate into this ADDri/SUBri
2137    // as possible.
2138    unsigned RotAmt = ARM_AM::getSOImmValRotate(Offset);
2139    unsigned ThisImmVal = Offset & ARM_AM::rotr32(0xFF, RotAmt);
2140
2141    // We will handle these bits from offset, clear them.
2142    Offset &= ~ThisImmVal;
2143
2144    // Get the properly encoded SOImmVal field.
2145    assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 &&
2146           "Bit extraction didn't work?");
2147    MI.getOperand(FrameRegIdx+1).ChangeToImmediate(ThisImmVal);
2148 } else {
2149    unsigned ImmIdx = 0;
2150    int InstrOffs = 0;
2151    unsigned NumBits = 0;
2152    unsigned Scale = 1;
2153    switch (AddrMode) {
2154    case ARMII::AddrMode_i12: {
2155      ImmIdx = FrameRegIdx + 1;
2156      InstrOffs = MI.getOperand(ImmIdx).getImm();
2157      NumBits = 12;
2158      break;
2159    }
2160    case ARMII::AddrMode2: {
2161      ImmIdx = FrameRegIdx+2;
2162      InstrOffs = ARM_AM::getAM2Offset(MI.getOperand(ImmIdx).getImm());
2163      if (ARM_AM::getAM2Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2164        InstrOffs *= -1;
2165      NumBits = 12;
2166      break;
2167    }
2168    case ARMII::AddrMode3: {
2169      ImmIdx = FrameRegIdx+2;
2170      InstrOffs = ARM_AM::getAM3Offset(MI.getOperand(ImmIdx).getImm());
2171      if (ARM_AM::getAM3Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2172        InstrOffs *= -1;
2173      NumBits = 8;
2174      break;
2175    }
2176    case ARMII::AddrMode4:
2177    case ARMII::AddrMode6:
2178      // Can't fold any offset even if it's zero.
2179      return false;
2180    case ARMII::AddrMode5: {
2181      ImmIdx = FrameRegIdx+1;
2182      InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2183      if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2184        InstrOffs *= -1;
2185      NumBits = 8;
2186      Scale = 4;
2187      break;
2188    }
2189    default:
2190      llvm_unreachable("Unsupported addressing mode!");
2191    }
2192
2193    Offset += InstrOffs * Scale;
2194    assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!");
2195    if (Offset < 0) {
2196      Offset = -Offset;
2197      isSub = true;
2198    }
2199
2200    // Attempt to fold address comp. if opcode has offset bits
2201    if (NumBits > 0) {
2202      // Common case: small offset, fits into instruction.
2203      MachineOperand &ImmOp = MI.getOperand(ImmIdx);
2204      int ImmedOffset = Offset / Scale;
2205      unsigned Mask = (1 << NumBits) - 1;
2206      if ((unsigned)Offset <= Mask * Scale) {
2207        // Replace the FrameIndex with sp
2208        MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2209        // FIXME: When addrmode2 goes away, this will simplify (like the
2210        // T2 version), as the LDR.i12 versions don't need the encoding
2211        // tricks for the offset value.
2212        if (isSub) {
2213          if (AddrMode == ARMII::AddrMode_i12)
2214            ImmedOffset = -ImmedOffset;
2215          else
2216            ImmedOffset |= 1 << NumBits;
2217        }
2218        ImmOp.ChangeToImmediate(ImmedOffset);
2219        Offset = 0;
2220        return true;
2221      }
2222
2223      // Otherwise, it didn't fit. Pull in what we can to simplify the immed.
2224      ImmedOffset = ImmedOffset & Mask;
2225      if (isSub) {
2226        if (AddrMode == ARMII::AddrMode_i12)
2227          ImmedOffset = -ImmedOffset;
2228        else
2229          ImmedOffset |= 1 << NumBits;
2230      }
2231      ImmOp.ChangeToImmediate(ImmedOffset);
2232      Offset &= ~(Mask*Scale);
2233    }
2234  }
2235
2236  Offset = (isSub) ? -Offset : Offset;
2237  return Offset == 0;
2238}
2239
2240/// analyzeCompare - For a comparison instruction, return the source registers
2241/// in SrcReg and SrcReg2 if having two register operands, and the value it
2242/// compares against in CmpValue. Return true if the comparison instruction
2243/// can be analyzed.
2244bool ARMBaseInstrInfo::
2245analyzeCompare(const MachineInstr *MI, unsigned &SrcReg, unsigned &SrcReg2,
2246               int &CmpMask, int &CmpValue) const {
2247  switch (MI->getOpcode()) {
2248  default: break;
2249  case ARM::CMPri:
2250  case ARM::t2CMPri:
2251    SrcReg = MI->getOperand(0).getReg();
2252    SrcReg2 = 0;
2253    CmpMask = ~0;
2254    CmpValue = MI->getOperand(1).getImm();
2255    return true;
2256  case ARM::CMPrr:
2257  case ARM::t2CMPrr:
2258    SrcReg = MI->getOperand(0).getReg();
2259    SrcReg2 = MI->getOperand(1).getReg();
2260    CmpMask = ~0;
2261    CmpValue = 0;
2262    return true;
2263  case ARM::TSTri:
2264  case ARM::t2TSTri:
2265    SrcReg = MI->getOperand(0).getReg();
2266    SrcReg2 = 0;
2267    CmpMask = MI->getOperand(1).getImm();
2268    CmpValue = 0;
2269    return true;
2270  }
2271
2272  return false;
2273}
2274
2275/// isSuitableForMask - Identify a suitable 'and' instruction that
2276/// operates on the given source register and applies the same mask
2277/// as a 'tst' instruction. Provide a limited look-through for copies.
2278/// When successful, MI will hold the found instruction.
2279static bool isSuitableForMask(MachineInstr *&MI, unsigned SrcReg,
2280                              int CmpMask, bool CommonUse) {
2281  switch (MI->getOpcode()) {
2282    case ARM::ANDri:
2283    case ARM::t2ANDri:
2284      if (CmpMask != MI->getOperand(2).getImm())
2285        return false;
2286      if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg())
2287        return true;
2288      break;
2289    case ARM::COPY: {
2290      // Walk down one instruction which is potentially an 'and'.
2291      const MachineInstr &Copy = *MI;
2292      MachineBasicBlock::iterator AND(
2293        std::next(MachineBasicBlock::iterator(MI)));
2294      if (AND == MI->getParent()->end()) return false;
2295      MI = AND;
2296      return isSuitableForMask(MI, Copy.getOperand(0).getReg(),
2297                               CmpMask, true);
2298    }
2299  }
2300
2301  return false;
2302}
2303
2304/// getSwappedCondition - assume the flags are set by MI(a,b), return
2305/// the condition code if we modify the instructions such that flags are
2306/// set by MI(b,a).
2307inline static ARMCC::CondCodes getSwappedCondition(ARMCC::CondCodes CC) {
2308  switch (CC) {
2309  default: return ARMCC::AL;
2310  case ARMCC::EQ: return ARMCC::EQ;
2311  case ARMCC::NE: return ARMCC::NE;
2312  case ARMCC::HS: return ARMCC::LS;
2313  case ARMCC::LO: return ARMCC::HI;
2314  case ARMCC::HI: return ARMCC::LO;
2315  case ARMCC::LS: return ARMCC::HS;
2316  case ARMCC::GE: return ARMCC::LE;
2317  case ARMCC::LT: return ARMCC::GT;
2318  case ARMCC::GT: return ARMCC::LT;
2319  case ARMCC::LE: return ARMCC::GE;
2320  }
2321}
2322
2323/// isRedundantFlagInstr - check whether the first instruction, whose only
2324/// purpose is to update flags, can be made redundant.
2325/// CMPrr can be made redundant by SUBrr if the operands are the same.
2326/// CMPri can be made redundant by SUBri if the operands are the same.
2327/// This function can be extended later on.
2328inline static bool isRedundantFlagInstr(MachineInstr *CmpI, unsigned SrcReg,
2329                                        unsigned SrcReg2, int ImmValue,
2330                                        MachineInstr *OI) {
2331  if ((CmpI->getOpcode() == ARM::CMPrr ||
2332       CmpI->getOpcode() == ARM::t2CMPrr) &&
2333      (OI->getOpcode() == ARM::SUBrr ||
2334       OI->getOpcode() == ARM::t2SUBrr) &&
2335      ((OI->getOperand(1).getReg() == SrcReg &&
2336        OI->getOperand(2).getReg() == SrcReg2) ||
2337       (OI->getOperand(1).getReg() == SrcReg2 &&
2338        OI->getOperand(2).getReg() == SrcReg)))
2339    return true;
2340
2341  if ((CmpI->getOpcode() == ARM::CMPri ||
2342       CmpI->getOpcode() == ARM::t2CMPri) &&
2343      (OI->getOpcode() == ARM::SUBri ||
2344       OI->getOpcode() == ARM::t2SUBri) &&
2345      OI->getOperand(1).getReg() == SrcReg &&
2346      OI->getOperand(2).getImm() == ImmValue)
2347    return true;
2348  return false;
2349}
2350
2351/// optimizeCompareInstr - Convert the instruction supplying the argument to the
2352/// comparison into one that sets the zero bit in the flags register;
2353/// Remove a redundant Compare instruction if an earlier instruction can set the
2354/// flags in the same way as Compare.
2355/// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two
2356/// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
2357/// condition code of instructions which use the flags.
2358bool ARMBaseInstrInfo::
2359optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
2360                     int CmpMask, int CmpValue,
2361                     const MachineRegisterInfo *MRI) const {
2362  // Get the unique definition of SrcReg.
2363  MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2364  if (!MI) return false;
2365
2366  // Masked compares sometimes use the same register as the corresponding 'and'.
2367  if (CmpMask != ~0) {
2368    if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(MI)) {
2369      MI = nullptr;
2370      for (MachineRegisterInfo::use_instr_iterator
2371           UI = MRI->use_instr_begin(SrcReg), UE = MRI->use_instr_end();
2372           UI != UE; ++UI) {
2373        if (UI->getParent() != CmpInstr->getParent()) continue;
2374        MachineInstr *PotentialAND = &*UI;
2375        if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) ||
2376            isPredicated(PotentialAND))
2377          continue;
2378        MI = PotentialAND;
2379        break;
2380      }
2381      if (!MI) return false;
2382    }
2383  }
2384
2385  // Get ready to iterate backward from CmpInstr.
2386  MachineBasicBlock::iterator I = CmpInstr, E = MI,
2387                              B = CmpInstr->getParent()->begin();
2388
2389  // Early exit if CmpInstr is at the beginning of the BB.
2390  if (I == B) return false;
2391
2392  // There are two possible candidates which can be changed to set CPSR:
2393  // One is MI, the other is a SUB instruction.
2394  // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2395  // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue).
2396  MachineInstr *Sub = nullptr;
2397  if (SrcReg2 != 0)
2398    // MI is not a candidate for CMPrr.
2399    MI = nullptr;
2400  else if (MI->getParent() != CmpInstr->getParent() || CmpValue != 0) {
2401    // Conservatively refuse to convert an instruction which isn't in the same
2402    // BB as the comparison.
2403    // For CMPri w/ CmpValue != 0, a Sub may still be a candidate.
2404    // Thus we cannot return here.
2405    if (CmpInstr->getOpcode() == ARM::CMPri ||
2406       CmpInstr->getOpcode() == ARM::t2CMPri)
2407      MI = nullptr;
2408    else
2409      return false;
2410  }
2411
2412  // Check that CPSR isn't set between the comparison instruction and the one we
2413  // want to change. At the same time, search for Sub.
2414  const TargetRegisterInfo *TRI = &getRegisterInfo();
2415  --I;
2416  for (; I != E; --I) {
2417    const MachineInstr &Instr = *I;
2418
2419    if (Instr.modifiesRegister(ARM::CPSR, TRI) ||
2420        Instr.readsRegister(ARM::CPSR, TRI))
2421      // This instruction modifies or uses CPSR after the one we want to
2422      // change. We can't do this transformation.
2423      return false;
2424
2425    // Check whether CmpInstr can be made redundant by the current instruction.
2426    if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpValue, &*I)) {
2427      Sub = &*I;
2428      break;
2429    }
2430
2431    if (I == B)
2432      // The 'and' is below the comparison instruction.
2433      return false;
2434  }
2435
2436  // Return false if no candidates exist.
2437  if (!MI && !Sub)
2438    return false;
2439
2440  // The single candidate is called MI.
2441  if (!MI) MI = Sub;
2442
2443  // We can't use a predicated instruction - it doesn't always write the flags.
2444  if (isPredicated(MI))
2445    return false;
2446
2447  switch (MI->getOpcode()) {
2448  default: break;
2449  case ARM::RSBrr:
2450  case ARM::RSBri:
2451  case ARM::RSCrr:
2452  case ARM::RSCri:
2453  case ARM::ADDrr:
2454  case ARM::ADDri:
2455  case ARM::ADCrr:
2456  case ARM::ADCri:
2457  case ARM::SUBrr:
2458  case ARM::SUBri:
2459  case ARM::SBCrr:
2460  case ARM::SBCri:
2461  case ARM::t2RSBri:
2462  case ARM::t2ADDrr:
2463  case ARM::t2ADDri:
2464  case ARM::t2ADCrr:
2465  case ARM::t2ADCri:
2466  case ARM::t2SUBrr:
2467  case ARM::t2SUBri:
2468  case ARM::t2SBCrr:
2469  case ARM::t2SBCri:
2470  case ARM::ANDrr:
2471  case ARM::ANDri:
2472  case ARM::t2ANDrr:
2473  case ARM::t2ANDri:
2474  case ARM::ORRrr:
2475  case ARM::ORRri:
2476  case ARM::t2ORRrr:
2477  case ARM::t2ORRri:
2478  case ARM::EORrr:
2479  case ARM::EORri:
2480  case ARM::t2EORrr:
2481  case ARM::t2EORri: {
2482    // Scan forward for the use of CPSR
2483    // When checking against MI: if it's a conditional code that requires
2484    // checking of the V bit or C bit, then this is not safe to do.
2485    // It is safe to remove CmpInstr if CPSR is redefined or killed.
2486    // If we are done with the basic block, we need to check whether CPSR is
2487    // live-out.
2488    SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4>
2489        OperandsToUpdate;
2490    bool isSafe = false;
2491    I = CmpInstr;
2492    E = CmpInstr->getParent()->end();
2493    while (!isSafe && ++I != E) {
2494      const MachineInstr &Instr = *I;
2495      for (unsigned IO = 0, EO = Instr.getNumOperands();
2496           !isSafe && IO != EO; ++IO) {
2497        const MachineOperand &MO = Instr.getOperand(IO);
2498        if (MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) {
2499          isSafe = true;
2500          break;
2501        }
2502        if (!MO.isReg() || MO.getReg() != ARM::CPSR)
2503          continue;
2504        if (MO.isDef()) {
2505          isSafe = true;
2506          break;
2507        }
2508        // Condition code is after the operand before CPSR except for VSELs.
2509        ARMCC::CondCodes CC;
2510        bool IsInstrVSel = true;
2511        switch (Instr.getOpcode()) {
2512        default:
2513          IsInstrVSel = false;
2514          CC = (ARMCC::CondCodes)Instr.getOperand(IO - 1).getImm();
2515          break;
2516        case ARM::VSELEQD:
2517        case ARM::VSELEQS:
2518          CC = ARMCC::EQ;
2519          break;
2520        case ARM::VSELGTD:
2521        case ARM::VSELGTS:
2522          CC = ARMCC::GT;
2523          break;
2524        case ARM::VSELGED:
2525        case ARM::VSELGES:
2526          CC = ARMCC::GE;
2527          break;
2528        case ARM::VSELVSS:
2529        case ARM::VSELVSD:
2530          CC = ARMCC::VS;
2531          break;
2532        }
2533
2534        if (Sub) {
2535          ARMCC::CondCodes NewCC = getSwappedCondition(CC);
2536          if (NewCC == ARMCC::AL)
2537            return false;
2538          // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based
2539          // on CMP needs to be updated to be based on SUB.
2540          // Push the condition code operands to OperandsToUpdate.
2541          // If it is safe to remove CmpInstr, the condition code of these
2542          // operands will be modified.
2543          if (SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2544              Sub->getOperand(2).getReg() == SrcReg) {
2545            // VSel doesn't support condition code update.
2546            if (IsInstrVSel)
2547              return false;
2548            OperandsToUpdate.push_back(
2549                std::make_pair(&((*I).getOperand(IO - 1)), NewCC));
2550          }
2551        } else {
2552          // No Sub, so this is x = <op> y, z; cmp x, 0.
2553          switch (CC) {
2554          case ARMCC::EQ: // Z
2555          case ARMCC::NE: // Z
2556          case ARMCC::MI: // N
2557          case ARMCC::PL: // N
2558          case ARMCC::AL: // none
2559            // CPSR can be used multiple times, we should continue.
2560            break;
2561          case ARMCC::HS: // C
2562          case ARMCC::LO: // C
2563          case ARMCC::VS: // V
2564          case ARMCC::VC: // V
2565          case ARMCC::HI: // C Z
2566          case ARMCC::LS: // C Z
2567          case ARMCC::GE: // N V
2568          case ARMCC::LT: // N V
2569          case ARMCC::GT: // Z N V
2570          case ARMCC::LE: // Z N V
2571            // The instruction uses the V bit or C bit which is not safe.
2572            return false;
2573          }
2574        }
2575      }
2576    }
2577
2578    // If CPSR is not killed nor re-defined, we should check whether it is
2579    // live-out. If it is live-out, do not optimize.
2580    if (!isSafe) {
2581      MachineBasicBlock *MBB = CmpInstr->getParent();
2582      for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
2583               SE = MBB->succ_end(); SI != SE; ++SI)
2584        if ((*SI)->isLiveIn(ARM::CPSR))
2585          return false;
2586    }
2587
2588    // Toggle the optional operand to CPSR.
2589    MI->getOperand(5).setReg(ARM::CPSR);
2590    MI->getOperand(5).setIsDef(true);
2591    assert(!isPredicated(MI) && "Can't use flags from predicated instruction");
2592    CmpInstr->eraseFromParent();
2593
2594    // Modify the condition code of operands in OperandsToUpdate.
2595    // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2596    // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2597    for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++)
2598      OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second);
2599    return true;
2600  }
2601  }
2602
2603  return false;
2604}
2605
2606bool ARMBaseInstrInfo::FoldImmediate(MachineInstr *UseMI,
2607                                     MachineInstr *DefMI, unsigned Reg,
2608                                     MachineRegisterInfo *MRI) const {
2609  // Fold large immediates into add, sub, or, xor.
2610  unsigned DefOpc = DefMI->getOpcode();
2611  if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm)
2612    return false;
2613  if (!DefMI->getOperand(1).isImm())
2614    // Could be t2MOVi32imm <ga:xx>
2615    return false;
2616
2617  if (!MRI->hasOneNonDBGUse(Reg))
2618    return false;
2619
2620  const MCInstrDesc &DefMCID = DefMI->getDesc();
2621  if (DefMCID.hasOptionalDef()) {
2622    unsigned NumOps = DefMCID.getNumOperands();
2623    const MachineOperand &MO = DefMI->getOperand(NumOps-1);
2624    if (MO.getReg() == ARM::CPSR && !MO.isDead())
2625      // If DefMI defines CPSR and it is not dead, it's obviously not safe
2626      // to delete DefMI.
2627      return false;
2628  }
2629
2630  const MCInstrDesc &UseMCID = UseMI->getDesc();
2631  if (UseMCID.hasOptionalDef()) {
2632    unsigned NumOps = UseMCID.getNumOperands();
2633    if (UseMI->getOperand(NumOps-1).getReg() == ARM::CPSR)
2634      // If the instruction sets the flag, do not attempt this optimization
2635      // since it may change the semantics of the code.
2636      return false;
2637  }
2638
2639  unsigned UseOpc = UseMI->getOpcode();
2640  unsigned NewUseOpc = 0;
2641  uint32_t ImmVal = (uint32_t)DefMI->getOperand(1).getImm();
2642  uint32_t SOImmValV1 = 0, SOImmValV2 = 0;
2643  bool Commute = false;
2644  switch (UseOpc) {
2645  default: return false;
2646  case ARM::SUBrr:
2647  case ARM::ADDrr:
2648  case ARM::ORRrr:
2649  case ARM::EORrr:
2650  case ARM::t2SUBrr:
2651  case ARM::t2ADDrr:
2652  case ARM::t2ORRrr:
2653  case ARM::t2EORrr: {
2654    Commute = UseMI->getOperand(2).getReg() != Reg;
2655    switch (UseOpc) {
2656    default: break;
2657    case ARM::SUBrr: {
2658      if (Commute)
2659        return false;
2660      ImmVal = -ImmVal;
2661      NewUseOpc = ARM::SUBri;
2662      // Fallthrough
2663    }
2664    case ARM::ADDrr:
2665    case ARM::ORRrr:
2666    case ARM::EORrr: {
2667      if (!ARM_AM::isSOImmTwoPartVal(ImmVal))
2668        return false;
2669      SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
2670      SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
2671      switch (UseOpc) {
2672      default: break;
2673      case ARM::ADDrr: NewUseOpc = ARM::ADDri; break;
2674      case ARM::ORRrr: NewUseOpc = ARM::ORRri; break;
2675      case ARM::EORrr: NewUseOpc = ARM::EORri; break;
2676      }
2677      break;
2678    }
2679    case ARM::t2SUBrr: {
2680      if (Commute)
2681        return false;
2682      ImmVal = -ImmVal;
2683      NewUseOpc = ARM::t2SUBri;
2684      // Fallthrough
2685    }
2686    case ARM::t2ADDrr:
2687    case ARM::t2ORRrr:
2688    case ARM::t2EORrr: {
2689      if (!ARM_AM::isT2SOImmTwoPartVal(ImmVal))
2690        return false;
2691      SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
2692      SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
2693      switch (UseOpc) {
2694      default: break;
2695      case ARM::t2ADDrr: NewUseOpc = ARM::t2ADDri; break;
2696      case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break;
2697      case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break;
2698      }
2699      break;
2700    }
2701    }
2702  }
2703  }
2704
2705  unsigned OpIdx = Commute ? 2 : 1;
2706  unsigned Reg1 = UseMI->getOperand(OpIdx).getReg();
2707  bool isKill = UseMI->getOperand(OpIdx).isKill();
2708  unsigned NewReg = MRI->createVirtualRegister(MRI->getRegClass(Reg));
2709  AddDefaultCC(AddDefaultPred(BuildMI(*UseMI->getParent(),
2710                                      UseMI, UseMI->getDebugLoc(),
2711                                      get(NewUseOpc), NewReg)
2712                              .addReg(Reg1, getKillRegState(isKill))
2713                              .addImm(SOImmValV1)));
2714  UseMI->setDesc(get(NewUseOpc));
2715  UseMI->getOperand(1).setReg(NewReg);
2716  UseMI->getOperand(1).setIsKill();
2717  UseMI->getOperand(2).ChangeToImmediate(SOImmValV2);
2718  DefMI->eraseFromParent();
2719  return true;
2720}
2721
2722static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
2723                                        const MachineInstr *MI) {
2724  switch (MI->getOpcode()) {
2725  default: {
2726    const MCInstrDesc &Desc = MI->getDesc();
2727    int UOps = ItinData->getNumMicroOps(Desc.getSchedClass());
2728    assert(UOps >= 0 && "bad # UOps");
2729    return UOps;
2730  }
2731
2732  case ARM::LDRrs:
2733  case ARM::LDRBrs:
2734  case ARM::STRrs:
2735  case ARM::STRBrs: {
2736    unsigned ShOpVal = MI->getOperand(3).getImm();
2737    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
2738    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
2739    if (!isSub &&
2740        (ShImm == 0 ||
2741         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
2742          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
2743      return 1;
2744    return 2;
2745  }
2746
2747  case ARM::LDRH:
2748  case ARM::STRH: {
2749    if (!MI->getOperand(2).getReg())
2750      return 1;
2751
2752    unsigned ShOpVal = MI->getOperand(3).getImm();
2753    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
2754    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
2755    if (!isSub &&
2756        (ShImm == 0 ||
2757         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
2758          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
2759      return 1;
2760    return 2;
2761  }
2762
2763  case ARM::LDRSB:
2764  case ARM::LDRSH:
2765    return (ARM_AM::getAM3Op(MI->getOperand(3).getImm()) == ARM_AM::sub) ? 3:2;
2766
2767  case ARM::LDRSB_POST:
2768  case ARM::LDRSH_POST: {
2769    unsigned Rt = MI->getOperand(0).getReg();
2770    unsigned Rm = MI->getOperand(3).getReg();
2771    return (Rt == Rm) ? 4 : 3;
2772  }
2773
2774  case ARM::LDR_PRE_REG:
2775  case ARM::LDRB_PRE_REG: {
2776    unsigned Rt = MI->getOperand(0).getReg();
2777    unsigned Rm = MI->getOperand(3).getReg();
2778    if (Rt == Rm)
2779      return 3;
2780    unsigned ShOpVal = MI->getOperand(4).getImm();
2781    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
2782    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
2783    if (!isSub &&
2784        (ShImm == 0 ||
2785         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
2786          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
2787      return 2;
2788    return 3;
2789  }
2790
2791  case ARM::STR_PRE_REG:
2792  case ARM::STRB_PRE_REG: {
2793    unsigned ShOpVal = MI->getOperand(4).getImm();
2794    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
2795    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
2796    if (!isSub &&
2797        (ShImm == 0 ||
2798         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
2799          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
2800      return 2;
2801    return 3;
2802  }
2803
2804  case ARM::LDRH_PRE:
2805  case ARM::STRH_PRE: {
2806    unsigned Rt = MI->getOperand(0).getReg();
2807    unsigned Rm = MI->getOperand(3).getReg();
2808    if (!Rm)
2809      return 2;
2810    if (Rt == Rm)
2811      return 3;
2812    return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub)
2813      ? 3 : 2;
2814  }
2815
2816  case ARM::LDR_POST_REG:
2817  case ARM::LDRB_POST_REG:
2818  case ARM::LDRH_POST: {
2819    unsigned Rt = MI->getOperand(0).getReg();
2820    unsigned Rm = MI->getOperand(3).getReg();
2821    return (Rt == Rm) ? 3 : 2;
2822  }
2823
2824  case ARM::LDR_PRE_IMM:
2825  case ARM::LDRB_PRE_IMM:
2826  case ARM::LDR_POST_IMM:
2827  case ARM::LDRB_POST_IMM:
2828  case ARM::STRB_POST_IMM:
2829  case ARM::STRB_POST_REG:
2830  case ARM::STRB_PRE_IMM:
2831  case ARM::STRH_POST:
2832  case ARM::STR_POST_IMM:
2833  case ARM::STR_POST_REG:
2834  case ARM::STR_PRE_IMM:
2835    return 2;
2836
2837  case ARM::LDRSB_PRE:
2838  case ARM::LDRSH_PRE: {
2839    unsigned Rm = MI->getOperand(3).getReg();
2840    if (Rm == 0)
2841      return 3;
2842    unsigned Rt = MI->getOperand(0).getReg();
2843    if (Rt == Rm)
2844      return 4;
2845    unsigned ShOpVal = MI->getOperand(4).getImm();
2846    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
2847    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
2848    if (!isSub &&
2849        (ShImm == 0 ||
2850         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
2851          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
2852      return 3;
2853    return 4;
2854  }
2855
2856  case ARM::LDRD: {
2857    unsigned Rt = MI->getOperand(0).getReg();
2858    unsigned Rn = MI->getOperand(2).getReg();
2859    unsigned Rm = MI->getOperand(3).getReg();
2860    if (Rm)
2861      return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub) ?4:3;
2862    return (Rt == Rn) ? 3 : 2;
2863  }
2864
2865  case ARM::STRD: {
2866    unsigned Rm = MI->getOperand(3).getReg();
2867    if (Rm)
2868      return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub) ?4:3;
2869    return 2;
2870  }
2871
2872  case ARM::LDRD_POST:
2873  case ARM::t2LDRD_POST:
2874    return 3;
2875
2876  case ARM::STRD_POST:
2877  case ARM::t2STRD_POST:
2878    return 4;
2879
2880  case ARM::LDRD_PRE: {
2881    unsigned Rt = MI->getOperand(0).getReg();
2882    unsigned Rn = MI->getOperand(3).getReg();
2883    unsigned Rm = MI->getOperand(4).getReg();
2884    if (Rm)
2885      return (ARM_AM::getAM3Op(MI->getOperand(5).getImm()) == ARM_AM::sub) ?5:4;
2886    return (Rt == Rn) ? 4 : 3;
2887  }
2888
2889  case ARM::t2LDRD_PRE: {
2890    unsigned Rt = MI->getOperand(0).getReg();
2891    unsigned Rn = MI->getOperand(3).getReg();
2892    return (Rt == Rn) ? 4 : 3;
2893  }
2894
2895  case ARM::STRD_PRE: {
2896    unsigned Rm = MI->getOperand(4).getReg();
2897    if (Rm)
2898      return (ARM_AM::getAM3Op(MI->getOperand(5).getImm()) == ARM_AM::sub) ?5:4;
2899    return 3;
2900  }
2901
2902  case ARM::t2STRD_PRE:
2903    return 3;
2904
2905  case ARM::t2LDR_POST:
2906  case ARM::t2LDRB_POST:
2907  case ARM::t2LDRB_PRE:
2908  case ARM::t2LDRSBi12:
2909  case ARM::t2LDRSBi8:
2910  case ARM::t2LDRSBpci:
2911  case ARM::t2LDRSBs:
2912  case ARM::t2LDRH_POST:
2913  case ARM::t2LDRH_PRE:
2914  case ARM::t2LDRSBT:
2915  case ARM::t2LDRSB_POST:
2916  case ARM::t2LDRSB_PRE:
2917  case ARM::t2LDRSH_POST:
2918  case ARM::t2LDRSH_PRE:
2919  case ARM::t2LDRSHi12:
2920  case ARM::t2LDRSHi8:
2921  case ARM::t2LDRSHpci:
2922  case ARM::t2LDRSHs:
2923    return 2;
2924
2925  case ARM::t2LDRDi8: {
2926    unsigned Rt = MI->getOperand(0).getReg();
2927    unsigned Rn = MI->getOperand(2).getReg();
2928    return (Rt == Rn) ? 3 : 2;
2929  }
2930
2931  case ARM::t2STRB_POST:
2932  case ARM::t2STRB_PRE:
2933  case ARM::t2STRBs:
2934  case ARM::t2STRDi8:
2935  case ARM::t2STRH_POST:
2936  case ARM::t2STRH_PRE:
2937  case ARM::t2STRHs:
2938  case ARM::t2STR_POST:
2939  case ARM::t2STR_PRE:
2940  case ARM::t2STRs:
2941    return 2;
2942  }
2943}
2944
2945// Return the number of 32-bit words loaded by LDM or stored by STM. If this
2946// can't be easily determined return 0 (missing MachineMemOperand).
2947//
2948// FIXME: The current MachineInstr design does not support relying on machine
2949// mem operands to determine the width of a memory access. Instead, we expect
2950// the target to provide this information based on the instruction opcode and
2951// operands. However, using MachineMemOperand is the best solution now for
2952// two reasons:
2953//
2954// 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
2955// operands. This is much more dangerous than using the MachineMemOperand
2956// sizes because CodeGen passes can insert/remove optional machine operands. In
2957// fact, it's totally incorrect for preRA passes and appears to be wrong for
2958// postRA passes as well.
2959//
2960// 2) getNumLDMAddresses is only used by the scheduling machine model and any
2961// machine model that calls this should handle the unknown (zero size) case.
2962//
2963// Long term, we should require a target hook that verifies MachineMemOperand
2964// sizes during MC lowering. That target hook should be local to MC lowering
2965// because we can't ensure that it is aware of other MI forms. Doing this will
2966// ensure that MachineMemOperands are correctly propagated through all passes.
2967unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr *MI) const {
2968  unsigned Size = 0;
2969  for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
2970         E = MI->memoperands_end(); I != E; ++I) {
2971    Size += (*I)->getSize();
2972  }
2973  return Size / 4;
2974}
2975
2976unsigned
2977ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
2978                                 const MachineInstr *MI) const {
2979  if (!ItinData || ItinData->isEmpty())
2980    return 1;
2981
2982  const MCInstrDesc &Desc = MI->getDesc();
2983  unsigned Class = Desc.getSchedClass();
2984  int ItinUOps = ItinData->getNumMicroOps(Class);
2985  if (ItinUOps >= 0) {
2986    if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
2987      return getNumMicroOpsSwiftLdSt(ItinData, MI);
2988
2989    return ItinUOps;
2990  }
2991
2992  unsigned Opc = MI->getOpcode();
2993  switch (Opc) {
2994  default:
2995    llvm_unreachable("Unexpected multi-uops instruction!");
2996  case ARM::VLDMQIA:
2997  case ARM::VSTMQIA:
2998    return 2;
2999
3000  // The number of uOps for load / store multiple are determined by the number
3001  // registers.
3002  //
3003  // On Cortex-A8, each pair of register loads / stores can be scheduled on the
3004  // same cycle. The scheduling for the first load / store must be done
3005  // separately by assuming the address is not 64-bit aligned.
3006  //
3007  // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address
3008  // is not 64-bit aligned, then AGU would take an extra cycle.  For VFP / NEON
3009  // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1.
3010  case ARM::VLDMDIA:
3011  case ARM::VLDMDIA_UPD:
3012  case ARM::VLDMDDB_UPD:
3013  case ARM::VLDMSIA:
3014  case ARM::VLDMSIA_UPD:
3015  case ARM::VLDMSDB_UPD:
3016  case ARM::VSTMDIA:
3017  case ARM::VSTMDIA_UPD:
3018  case ARM::VSTMDDB_UPD:
3019  case ARM::VSTMSIA:
3020  case ARM::VSTMSIA_UPD:
3021  case ARM::VSTMSDB_UPD: {
3022    unsigned NumRegs = MI->getNumOperands() - Desc.getNumOperands();
3023    return (NumRegs / 2) + (NumRegs % 2) + 1;
3024  }
3025
3026  case ARM::LDMIA_RET:
3027  case ARM::LDMIA:
3028  case ARM::LDMDA:
3029  case ARM::LDMDB:
3030  case ARM::LDMIB:
3031  case ARM::LDMIA_UPD:
3032  case ARM::LDMDA_UPD:
3033  case ARM::LDMDB_UPD:
3034  case ARM::LDMIB_UPD:
3035  case ARM::STMIA:
3036  case ARM::STMDA:
3037  case ARM::STMDB:
3038  case ARM::STMIB:
3039  case ARM::STMIA_UPD:
3040  case ARM::STMDA_UPD:
3041  case ARM::STMDB_UPD:
3042  case ARM::STMIB_UPD:
3043  case ARM::tLDMIA:
3044  case ARM::tLDMIA_UPD:
3045  case ARM::tSTMIA_UPD:
3046  case ARM::tPOP_RET:
3047  case ARM::tPOP:
3048  case ARM::tPUSH:
3049  case ARM::t2LDMIA_RET:
3050  case ARM::t2LDMIA:
3051  case ARM::t2LDMDB:
3052  case ARM::t2LDMIA_UPD:
3053  case ARM::t2LDMDB_UPD:
3054  case ARM::t2STMIA:
3055  case ARM::t2STMDB:
3056  case ARM::t2STMIA_UPD:
3057  case ARM::t2STMDB_UPD: {
3058    unsigned NumRegs = MI->getNumOperands() - Desc.getNumOperands() + 1;
3059    if (Subtarget.isSwift()) {
3060      int UOps = 1 + NumRegs;  // One for address computation, one for each ld / st.
3061      switch (Opc) {
3062      default: break;
3063      case ARM::VLDMDIA_UPD:
3064      case ARM::VLDMDDB_UPD:
3065      case ARM::VLDMSIA_UPD:
3066      case ARM::VLDMSDB_UPD:
3067      case ARM::VSTMDIA_UPD:
3068      case ARM::VSTMDDB_UPD:
3069      case ARM::VSTMSIA_UPD:
3070      case ARM::VSTMSDB_UPD:
3071      case ARM::LDMIA_UPD:
3072      case ARM::LDMDA_UPD:
3073      case ARM::LDMDB_UPD:
3074      case ARM::LDMIB_UPD:
3075      case ARM::STMIA_UPD:
3076      case ARM::STMDA_UPD:
3077      case ARM::STMDB_UPD:
3078      case ARM::STMIB_UPD:
3079      case ARM::tLDMIA_UPD:
3080      case ARM::tSTMIA_UPD:
3081      case ARM::t2LDMIA_UPD:
3082      case ARM::t2LDMDB_UPD:
3083      case ARM::t2STMIA_UPD:
3084      case ARM::t2STMDB_UPD:
3085        ++UOps; // One for base register writeback.
3086        break;
3087      case ARM::LDMIA_RET:
3088      case ARM::tPOP_RET:
3089      case ARM::t2LDMIA_RET:
3090        UOps += 2; // One for base reg wb, one for write to pc.
3091        break;
3092      }
3093      return UOps;
3094    } else if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3095      if (NumRegs < 4)
3096        return 2;
3097      // 4 registers would be issued: 2, 2.
3098      // 5 registers would be issued: 2, 2, 1.
3099      int A8UOps = (NumRegs / 2);
3100      if (NumRegs % 2)
3101        ++A8UOps;
3102      return A8UOps;
3103    } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3104      int A9UOps = (NumRegs / 2);
3105      // If there are odd number of registers or if it's not 64-bit aligned,
3106      // then it takes an extra AGU (Address Generation Unit) cycle.
3107      if ((NumRegs % 2) ||
3108          !MI->hasOneMemOperand() ||
3109          (*MI->memoperands_begin())->getAlignment() < 8)
3110        ++A9UOps;
3111      return A9UOps;
3112    } else {
3113      // Assume the worst.
3114      return NumRegs;
3115    }
3116  }
3117  }
3118}
3119
3120int
3121ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
3122                                  const MCInstrDesc &DefMCID,
3123                                  unsigned DefClass,
3124                                  unsigned DefIdx, unsigned DefAlign) const {
3125  int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3126  if (RegNo <= 0)
3127    // Def is the address writeback.
3128    return ItinData->getOperandCycle(DefClass, DefIdx);
3129
3130  int DefCycle;
3131  if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3132    // (regno / 2) + (regno % 2) + 1
3133    DefCycle = RegNo / 2 + 1;
3134    if (RegNo % 2)
3135      ++DefCycle;
3136  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3137    DefCycle = RegNo;
3138    bool isSLoad = false;
3139
3140    switch (DefMCID.getOpcode()) {
3141    default: break;
3142    case ARM::VLDMSIA:
3143    case ARM::VLDMSIA_UPD:
3144    case ARM::VLDMSDB_UPD:
3145      isSLoad = true;
3146      break;
3147    }
3148
3149    // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3150    // then it takes an extra cycle.
3151    if ((isSLoad && (RegNo % 2)) || DefAlign < 8)
3152      ++DefCycle;
3153  } else {
3154    // Assume the worst.
3155    DefCycle = RegNo + 2;
3156  }
3157
3158  return DefCycle;
3159}
3160
3161int
3162ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
3163                                 const MCInstrDesc &DefMCID,
3164                                 unsigned DefClass,
3165                                 unsigned DefIdx, unsigned DefAlign) const {
3166  int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3167  if (RegNo <= 0)
3168    // Def is the address writeback.
3169    return ItinData->getOperandCycle(DefClass, DefIdx);
3170
3171  int DefCycle;
3172  if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3173    // 4 registers would be issued: 1, 2, 1.
3174    // 5 registers would be issued: 1, 2, 2.
3175    DefCycle = RegNo / 2;
3176    if (DefCycle < 1)
3177      DefCycle = 1;
3178    // Result latency is issue cycle + 2: E2.
3179    DefCycle += 2;
3180  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3181    DefCycle = (RegNo / 2);
3182    // If there are odd number of registers or if it's not 64-bit aligned,
3183    // then it takes an extra AGU (Address Generation Unit) cycle.
3184    if ((RegNo % 2) || DefAlign < 8)
3185      ++DefCycle;
3186    // Result latency is AGU cycles + 2.
3187    DefCycle += 2;
3188  } else {
3189    // Assume the worst.
3190    DefCycle = RegNo + 2;
3191  }
3192
3193  return DefCycle;
3194}
3195
3196int
3197ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
3198                                  const MCInstrDesc &UseMCID,
3199                                  unsigned UseClass,
3200                                  unsigned UseIdx, unsigned UseAlign) const {
3201  int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3202  if (RegNo <= 0)
3203    return ItinData->getOperandCycle(UseClass, UseIdx);
3204
3205  int UseCycle;
3206  if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3207    // (regno / 2) + (regno % 2) + 1
3208    UseCycle = RegNo / 2 + 1;
3209    if (RegNo % 2)
3210      ++UseCycle;
3211  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3212    UseCycle = RegNo;
3213    bool isSStore = false;
3214
3215    switch (UseMCID.getOpcode()) {
3216    default: break;
3217    case ARM::VSTMSIA:
3218    case ARM::VSTMSIA_UPD:
3219    case ARM::VSTMSDB_UPD:
3220      isSStore = true;
3221      break;
3222    }
3223
3224    // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3225    // then it takes an extra cycle.
3226    if ((isSStore && (RegNo % 2)) || UseAlign < 8)
3227      ++UseCycle;
3228  } else {
3229    // Assume the worst.
3230    UseCycle = RegNo + 2;
3231  }
3232
3233  return UseCycle;
3234}
3235
3236int
3237ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
3238                                 const MCInstrDesc &UseMCID,
3239                                 unsigned UseClass,
3240                                 unsigned UseIdx, unsigned UseAlign) const {
3241  int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3242  if (RegNo <= 0)
3243    return ItinData->getOperandCycle(UseClass, UseIdx);
3244
3245  int UseCycle;
3246  if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3247    UseCycle = RegNo / 2;
3248    if (UseCycle < 2)
3249      UseCycle = 2;
3250    // Read in E3.
3251    UseCycle += 2;
3252  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3253    UseCycle = (RegNo / 2);
3254    // If there are odd number of registers or if it's not 64-bit aligned,
3255    // then it takes an extra AGU (Address Generation Unit) cycle.
3256    if ((RegNo % 2) || UseAlign < 8)
3257      ++UseCycle;
3258  } else {
3259    // Assume the worst.
3260    UseCycle = 1;
3261  }
3262  return UseCycle;
3263}
3264
3265int
3266ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
3267                                    const MCInstrDesc &DefMCID,
3268                                    unsigned DefIdx, unsigned DefAlign,
3269                                    const MCInstrDesc &UseMCID,
3270                                    unsigned UseIdx, unsigned UseAlign) const {
3271  unsigned DefClass = DefMCID.getSchedClass();
3272  unsigned UseClass = UseMCID.getSchedClass();
3273
3274  if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands())
3275    return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
3276
3277  // This may be a def / use of a variable_ops instruction, the operand
3278  // latency might be determinable dynamically. Let the target try to
3279  // figure it out.
3280  int DefCycle = -1;
3281  bool LdmBypass = false;
3282  switch (DefMCID.getOpcode()) {
3283  default:
3284    DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
3285    break;
3286
3287  case ARM::VLDMDIA:
3288  case ARM::VLDMDIA_UPD:
3289  case ARM::VLDMDDB_UPD:
3290  case ARM::VLDMSIA:
3291  case ARM::VLDMSIA_UPD:
3292  case ARM::VLDMSDB_UPD:
3293    DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
3294    break;
3295
3296  case ARM::LDMIA_RET:
3297  case ARM::LDMIA:
3298  case ARM::LDMDA:
3299  case ARM::LDMDB:
3300  case ARM::LDMIB:
3301  case ARM::LDMIA_UPD:
3302  case ARM::LDMDA_UPD:
3303  case ARM::LDMDB_UPD:
3304  case ARM::LDMIB_UPD:
3305  case ARM::tLDMIA:
3306  case ARM::tLDMIA_UPD:
3307  case ARM::tPUSH:
3308  case ARM::t2LDMIA_RET:
3309  case ARM::t2LDMIA:
3310  case ARM::t2LDMDB:
3311  case ARM::t2LDMIA_UPD:
3312  case ARM::t2LDMDB_UPD:
3313    LdmBypass = 1;
3314    DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
3315    break;
3316  }
3317
3318  if (DefCycle == -1)
3319    // We can't seem to determine the result latency of the def, assume it's 2.
3320    DefCycle = 2;
3321
3322  int UseCycle = -1;
3323  switch (UseMCID.getOpcode()) {
3324  default:
3325    UseCycle = ItinData->getOperandCycle(UseClass, UseIdx);
3326    break;
3327
3328  case ARM::VSTMDIA:
3329  case ARM::VSTMDIA_UPD:
3330  case ARM::VSTMDDB_UPD:
3331  case ARM::VSTMSIA:
3332  case ARM::VSTMSIA_UPD:
3333  case ARM::VSTMSDB_UPD:
3334    UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
3335    break;
3336
3337  case ARM::STMIA:
3338  case ARM::STMDA:
3339  case ARM::STMDB:
3340  case ARM::STMIB:
3341  case ARM::STMIA_UPD:
3342  case ARM::STMDA_UPD:
3343  case ARM::STMDB_UPD:
3344  case ARM::STMIB_UPD:
3345  case ARM::tSTMIA_UPD:
3346  case ARM::tPOP_RET:
3347  case ARM::tPOP:
3348  case ARM::t2STMIA:
3349  case ARM::t2STMDB:
3350  case ARM::t2STMIA_UPD:
3351  case ARM::t2STMDB_UPD:
3352    UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
3353    break;
3354  }
3355
3356  if (UseCycle == -1)
3357    // Assume it's read in the first stage.
3358    UseCycle = 1;
3359
3360  UseCycle = DefCycle - UseCycle + 1;
3361  if (UseCycle > 0) {
3362    if (LdmBypass) {
3363      // It's a variable_ops instruction so we can't use DefIdx here. Just use
3364      // first def operand.
3365      if (ItinData->hasPipelineForwarding(DefClass, DefMCID.getNumOperands()-1,
3366                                          UseClass, UseIdx))
3367        --UseCycle;
3368    } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx,
3369                                               UseClass, UseIdx)) {
3370      --UseCycle;
3371    }
3372  }
3373
3374  return UseCycle;
3375}
3376
3377static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI,
3378                                           const MachineInstr *MI, unsigned Reg,
3379                                           unsigned &DefIdx, unsigned &Dist) {
3380  Dist = 0;
3381
3382  MachineBasicBlock::const_iterator I = MI; ++I;
3383  MachineBasicBlock::const_instr_iterator II = std::prev(I.getInstrIterator());
3384  assert(II->isInsideBundle() && "Empty bundle?");
3385
3386  int Idx = -1;
3387  while (II->isInsideBundle()) {
3388    Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI);
3389    if (Idx != -1)
3390      break;
3391    --II;
3392    ++Dist;
3393  }
3394
3395  assert(Idx != -1 && "Cannot find bundled definition!");
3396  DefIdx = Idx;
3397  return II;
3398}
3399
3400static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI,
3401                                           const MachineInstr *MI, unsigned Reg,
3402                                           unsigned &UseIdx, unsigned &Dist) {
3403  Dist = 0;
3404
3405  MachineBasicBlock::const_instr_iterator II = MI; ++II;
3406  assert(II->isInsideBundle() && "Empty bundle?");
3407  MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end();
3408
3409  // FIXME: This doesn't properly handle multiple uses.
3410  int Idx = -1;
3411  while (II != E && II->isInsideBundle()) {
3412    Idx = II->findRegisterUseOperandIdx(Reg, false, TRI);
3413    if (Idx != -1)
3414      break;
3415    if (II->getOpcode() != ARM::t2IT)
3416      ++Dist;
3417    ++II;
3418  }
3419
3420  if (Idx == -1) {
3421    Dist = 0;
3422    return nullptr;
3423  }
3424
3425  UseIdx = Idx;
3426  return II;
3427}
3428
3429/// Return the number of cycles to add to (or subtract from) the static
3430/// itinerary based on the def opcode and alignment. The caller will ensure that
3431/// adjusted latency is at least one cycle.
3432static int adjustDefLatency(const ARMSubtarget &Subtarget,
3433                            const MachineInstr *DefMI,
3434                            const MCInstrDesc *DefMCID, unsigned DefAlign) {
3435  int Adjust = 0;
3436  if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) {
3437    // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
3438    // variants are one cycle cheaper.
3439    switch (DefMCID->getOpcode()) {
3440    default: break;
3441    case ARM::LDRrs:
3442    case ARM::LDRBrs: {
3443      unsigned ShOpVal = DefMI->getOperand(3).getImm();
3444      unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3445      if (ShImm == 0 ||
3446          (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
3447        --Adjust;
3448      break;
3449    }
3450    case ARM::t2LDRs:
3451    case ARM::t2LDRBs:
3452    case ARM::t2LDRHs:
3453    case ARM::t2LDRSHs: {
3454      // Thumb2 mode: lsl only.
3455      unsigned ShAmt = DefMI->getOperand(3).getImm();
3456      if (ShAmt == 0 || ShAmt == 2)
3457        --Adjust;
3458      break;
3459    }
3460    }
3461  } else if (Subtarget.isSwift()) {
3462    // FIXME: Properly handle all of the latency adjustments for address
3463    // writeback.
3464    switch (DefMCID->getOpcode()) {
3465    default: break;
3466    case ARM::LDRrs:
3467    case ARM::LDRBrs: {
3468      unsigned ShOpVal = DefMI->getOperand(3).getImm();
3469      bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3470      unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3471      if (!isSub &&
3472          (ShImm == 0 ||
3473           ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3474            ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3475        Adjust -= 2;
3476      else if (!isSub &&
3477               ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
3478        --Adjust;
3479      break;
3480    }
3481    case ARM::t2LDRs:
3482    case ARM::t2LDRBs:
3483    case ARM::t2LDRHs:
3484    case ARM::t2LDRSHs: {
3485      // Thumb2 mode: lsl only.
3486      unsigned ShAmt = DefMI->getOperand(3).getImm();
3487      if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
3488        Adjust -= 2;
3489      break;
3490    }
3491    }
3492  }
3493
3494  if (DefAlign < 8 && Subtarget.isLikeA9()) {
3495    switch (DefMCID->getOpcode()) {
3496    default: break;
3497    case ARM::VLD1q8:
3498    case ARM::VLD1q16:
3499    case ARM::VLD1q32:
3500    case ARM::VLD1q64:
3501    case ARM::VLD1q8wb_fixed:
3502    case ARM::VLD1q16wb_fixed:
3503    case ARM::VLD1q32wb_fixed:
3504    case ARM::VLD1q64wb_fixed:
3505    case ARM::VLD1q8wb_register:
3506    case ARM::VLD1q16wb_register:
3507    case ARM::VLD1q32wb_register:
3508    case ARM::VLD1q64wb_register:
3509    case ARM::VLD2d8:
3510    case ARM::VLD2d16:
3511    case ARM::VLD2d32:
3512    case ARM::VLD2q8:
3513    case ARM::VLD2q16:
3514    case ARM::VLD2q32:
3515    case ARM::VLD2d8wb_fixed:
3516    case ARM::VLD2d16wb_fixed:
3517    case ARM::VLD2d32wb_fixed:
3518    case ARM::VLD2q8wb_fixed:
3519    case ARM::VLD2q16wb_fixed:
3520    case ARM::VLD2q32wb_fixed:
3521    case ARM::VLD2d8wb_register:
3522    case ARM::VLD2d16wb_register:
3523    case ARM::VLD2d32wb_register:
3524    case ARM::VLD2q8wb_register:
3525    case ARM::VLD2q16wb_register:
3526    case ARM::VLD2q32wb_register:
3527    case ARM::VLD3d8:
3528    case ARM::VLD3d16:
3529    case ARM::VLD3d32:
3530    case ARM::VLD1d64T:
3531    case ARM::VLD3d8_UPD:
3532    case ARM::VLD3d16_UPD:
3533    case ARM::VLD3d32_UPD:
3534    case ARM::VLD1d64Twb_fixed:
3535    case ARM::VLD1d64Twb_register:
3536    case ARM::VLD3q8_UPD:
3537    case ARM::VLD3q16_UPD:
3538    case ARM::VLD3q32_UPD:
3539    case ARM::VLD4d8:
3540    case ARM::VLD4d16:
3541    case ARM::VLD4d32:
3542    case ARM::VLD1d64Q:
3543    case ARM::VLD4d8_UPD:
3544    case ARM::VLD4d16_UPD:
3545    case ARM::VLD4d32_UPD:
3546    case ARM::VLD1d64Qwb_fixed:
3547    case ARM::VLD1d64Qwb_register:
3548    case ARM::VLD4q8_UPD:
3549    case ARM::VLD4q16_UPD:
3550    case ARM::VLD4q32_UPD:
3551    case ARM::VLD1DUPq8:
3552    case ARM::VLD1DUPq16:
3553    case ARM::VLD1DUPq32:
3554    case ARM::VLD1DUPq8wb_fixed:
3555    case ARM::VLD1DUPq16wb_fixed:
3556    case ARM::VLD1DUPq32wb_fixed:
3557    case ARM::VLD1DUPq8wb_register:
3558    case ARM::VLD1DUPq16wb_register:
3559    case ARM::VLD1DUPq32wb_register:
3560    case ARM::VLD2DUPd8:
3561    case ARM::VLD2DUPd16:
3562    case ARM::VLD2DUPd32:
3563    case ARM::VLD2DUPd8wb_fixed:
3564    case ARM::VLD2DUPd16wb_fixed:
3565    case ARM::VLD2DUPd32wb_fixed:
3566    case ARM::VLD2DUPd8wb_register:
3567    case ARM::VLD2DUPd16wb_register:
3568    case ARM::VLD2DUPd32wb_register:
3569    case ARM::VLD4DUPd8:
3570    case ARM::VLD4DUPd16:
3571    case ARM::VLD4DUPd32:
3572    case ARM::VLD4DUPd8_UPD:
3573    case ARM::VLD4DUPd16_UPD:
3574    case ARM::VLD4DUPd32_UPD:
3575    case ARM::VLD1LNd8:
3576    case ARM::VLD1LNd16:
3577    case ARM::VLD1LNd32:
3578    case ARM::VLD1LNd8_UPD:
3579    case ARM::VLD1LNd16_UPD:
3580    case ARM::VLD1LNd32_UPD:
3581    case ARM::VLD2LNd8:
3582    case ARM::VLD2LNd16:
3583    case ARM::VLD2LNd32:
3584    case ARM::VLD2LNq16:
3585    case ARM::VLD2LNq32:
3586    case ARM::VLD2LNd8_UPD:
3587    case ARM::VLD2LNd16_UPD:
3588    case ARM::VLD2LNd32_UPD:
3589    case ARM::VLD2LNq16_UPD:
3590    case ARM::VLD2LNq32_UPD:
3591    case ARM::VLD4LNd8:
3592    case ARM::VLD4LNd16:
3593    case ARM::VLD4LNd32:
3594    case ARM::VLD4LNq16:
3595    case ARM::VLD4LNq32:
3596    case ARM::VLD4LNd8_UPD:
3597    case ARM::VLD4LNd16_UPD:
3598    case ARM::VLD4LNd32_UPD:
3599    case ARM::VLD4LNq16_UPD:
3600    case ARM::VLD4LNq32_UPD:
3601      // If the address is not 64-bit aligned, the latencies of these
3602      // instructions increases by one.
3603      ++Adjust;
3604      break;
3605    }
3606  }
3607  return Adjust;
3608}
3609
3610
3611
3612int
3613ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
3614                                    const MachineInstr *DefMI, unsigned DefIdx,
3615                                    const MachineInstr *UseMI,
3616                                    unsigned UseIdx) const {
3617  // No operand latency. The caller may fall back to getInstrLatency.
3618  if (!ItinData || ItinData->isEmpty())
3619    return -1;
3620
3621  const MachineOperand &DefMO = DefMI->getOperand(DefIdx);
3622  unsigned Reg = DefMO.getReg();
3623  const MCInstrDesc *DefMCID = &DefMI->getDesc();
3624  const MCInstrDesc *UseMCID = &UseMI->getDesc();
3625
3626  unsigned DefAdj = 0;
3627  if (DefMI->isBundle()) {
3628    DefMI = getBundledDefMI(&getRegisterInfo(), DefMI, Reg, DefIdx, DefAdj);
3629    DefMCID = &DefMI->getDesc();
3630  }
3631  if (DefMI->isCopyLike() || DefMI->isInsertSubreg() ||
3632      DefMI->isRegSequence() || DefMI->isImplicitDef()) {
3633    return 1;
3634  }
3635
3636  unsigned UseAdj = 0;
3637  if (UseMI->isBundle()) {
3638    unsigned NewUseIdx;
3639    const MachineInstr *NewUseMI = getBundledUseMI(&getRegisterInfo(), UseMI,
3640                                                   Reg, NewUseIdx, UseAdj);
3641    if (!NewUseMI)
3642      return -1;
3643
3644    UseMI = NewUseMI;
3645    UseIdx = NewUseIdx;
3646    UseMCID = &UseMI->getDesc();
3647  }
3648
3649  if (Reg == ARM::CPSR) {
3650    if (DefMI->getOpcode() == ARM::FMSTAT) {
3651      // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
3652      return Subtarget.isLikeA9() ? 1 : 20;
3653    }
3654
3655    // CPSR set and branch can be paired in the same cycle.
3656    if (UseMI->isBranch())
3657      return 0;
3658
3659    // Otherwise it takes the instruction latency (generally one).
3660    unsigned Latency = getInstrLatency(ItinData, DefMI);
3661
3662    // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to
3663    // its uses. Instructions which are otherwise scheduled between them may
3664    // incur a code size penalty (not able to use the CPSR setting 16-bit
3665    // instructions).
3666    if (Latency > 0 && Subtarget.isThumb2()) {
3667      const MachineFunction *MF = DefMI->getParent()->getParent();
3668      if (MF->getFunction()->hasFnAttribute(Attribute::OptimizeForSize))
3669        --Latency;
3670    }
3671    return Latency;
3672  }
3673
3674  if (DefMO.isImplicit() || UseMI->getOperand(UseIdx).isImplicit())
3675    return -1;
3676
3677  unsigned DefAlign = DefMI->hasOneMemOperand()
3678    ? (*DefMI->memoperands_begin())->getAlignment() : 0;
3679  unsigned UseAlign = UseMI->hasOneMemOperand()
3680    ? (*UseMI->memoperands_begin())->getAlignment() : 0;
3681
3682  // Get the itinerary's latency if possible, and handle variable_ops.
3683  int Latency = getOperandLatency(ItinData, *DefMCID, DefIdx, DefAlign,
3684                                  *UseMCID, UseIdx, UseAlign);
3685  // Unable to find operand latency. The caller may resort to getInstrLatency.
3686  if (Latency < 0)
3687    return Latency;
3688
3689  // Adjust for IT block position.
3690  int Adj = DefAdj + UseAdj;
3691
3692  // Adjust for dynamic def-side opcode variants not captured by the itinerary.
3693  Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign);
3694  if (Adj >= 0 || (int)Latency > -Adj) {
3695    return Latency + Adj;
3696  }
3697  // Return the itinerary latency, which may be zero but not less than zero.
3698  return Latency;
3699}
3700
3701int
3702ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
3703                                    SDNode *DefNode, unsigned DefIdx,
3704                                    SDNode *UseNode, unsigned UseIdx) const {
3705  if (!DefNode->isMachineOpcode())
3706    return 1;
3707
3708  const MCInstrDesc &DefMCID = get(DefNode->getMachineOpcode());
3709
3710  if (isZeroCost(DefMCID.Opcode))
3711    return 0;
3712
3713  if (!ItinData || ItinData->isEmpty())
3714    return DefMCID.mayLoad() ? 3 : 1;
3715
3716  if (!UseNode->isMachineOpcode()) {
3717    int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx);
3718    if (Subtarget.isLikeA9() || Subtarget.isSwift())
3719      return Latency <= 2 ? 1 : Latency - 1;
3720    else
3721      return Latency <= 3 ? 1 : Latency - 2;
3722  }
3723
3724  const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode());
3725  const MachineSDNode *DefMN = dyn_cast<MachineSDNode>(DefNode);
3726  unsigned DefAlign = !DefMN->memoperands_empty()
3727    ? (*DefMN->memoperands_begin())->getAlignment() : 0;
3728  const MachineSDNode *UseMN = dyn_cast<MachineSDNode>(UseNode);
3729  unsigned UseAlign = !UseMN->memoperands_empty()
3730    ? (*UseMN->memoperands_begin())->getAlignment() : 0;
3731  int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign,
3732                                  UseMCID, UseIdx, UseAlign);
3733
3734  if (Latency > 1 &&
3735      (Subtarget.isCortexA8() || Subtarget.isLikeA9() ||
3736       Subtarget.isCortexA7())) {
3737    // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
3738    // variants are one cycle cheaper.
3739    switch (DefMCID.getOpcode()) {
3740    default: break;
3741    case ARM::LDRrs:
3742    case ARM::LDRBrs: {
3743      unsigned ShOpVal =
3744        cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
3745      unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3746      if (ShImm == 0 ||
3747          (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
3748        --Latency;
3749      break;
3750    }
3751    case ARM::t2LDRs:
3752    case ARM::t2LDRBs:
3753    case ARM::t2LDRHs:
3754    case ARM::t2LDRSHs: {
3755      // Thumb2 mode: lsl only.
3756      unsigned ShAmt =
3757        cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
3758      if (ShAmt == 0 || ShAmt == 2)
3759        --Latency;
3760      break;
3761    }
3762    }
3763  } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) {
3764    // FIXME: Properly handle all of the latency adjustments for address
3765    // writeback.
3766    switch (DefMCID.getOpcode()) {
3767    default: break;
3768    case ARM::LDRrs:
3769    case ARM::LDRBrs: {
3770      unsigned ShOpVal =
3771        cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
3772      unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3773      if (ShImm == 0 ||
3774          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3775           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
3776        Latency -= 2;
3777      else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
3778        --Latency;
3779      break;
3780    }
3781    case ARM::t2LDRs:
3782    case ARM::t2LDRBs:
3783    case ARM::t2LDRHs:
3784    case ARM::t2LDRSHs: {
3785      // Thumb2 mode: lsl 0-3 only.
3786      Latency -= 2;
3787      break;
3788    }
3789    }
3790  }
3791
3792  if (DefAlign < 8 && Subtarget.isLikeA9())
3793    switch (DefMCID.getOpcode()) {
3794    default: break;
3795    case ARM::VLD1q8:
3796    case ARM::VLD1q16:
3797    case ARM::VLD1q32:
3798    case ARM::VLD1q64:
3799    case ARM::VLD1q8wb_register:
3800    case ARM::VLD1q16wb_register:
3801    case ARM::VLD1q32wb_register:
3802    case ARM::VLD1q64wb_register:
3803    case ARM::VLD1q8wb_fixed:
3804    case ARM::VLD1q16wb_fixed:
3805    case ARM::VLD1q32wb_fixed:
3806    case ARM::VLD1q64wb_fixed:
3807    case ARM::VLD2d8:
3808    case ARM::VLD2d16:
3809    case ARM::VLD2d32:
3810    case ARM::VLD2q8Pseudo:
3811    case ARM::VLD2q16Pseudo:
3812    case ARM::VLD2q32Pseudo:
3813    case ARM::VLD2d8wb_fixed:
3814    case ARM::VLD2d16wb_fixed:
3815    case ARM::VLD2d32wb_fixed:
3816    case ARM::VLD2q8PseudoWB_fixed:
3817    case ARM::VLD2q16PseudoWB_fixed:
3818    case ARM::VLD2q32PseudoWB_fixed:
3819    case ARM::VLD2d8wb_register:
3820    case ARM::VLD2d16wb_register:
3821    case ARM::VLD2d32wb_register:
3822    case ARM::VLD2q8PseudoWB_register:
3823    case ARM::VLD2q16PseudoWB_register:
3824    case ARM::VLD2q32PseudoWB_register:
3825    case ARM::VLD3d8Pseudo:
3826    case ARM::VLD3d16Pseudo:
3827    case ARM::VLD3d32Pseudo:
3828    case ARM::VLD1d64TPseudo:
3829    case ARM::VLD1d64TPseudoWB_fixed:
3830    case ARM::VLD3d8Pseudo_UPD:
3831    case ARM::VLD3d16Pseudo_UPD:
3832    case ARM::VLD3d32Pseudo_UPD:
3833    case ARM::VLD3q8Pseudo_UPD:
3834    case ARM::VLD3q16Pseudo_UPD:
3835    case ARM::VLD3q32Pseudo_UPD:
3836    case ARM::VLD3q8oddPseudo:
3837    case ARM::VLD3q16oddPseudo:
3838    case ARM::VLD3q32oddPseudo:
3839    case ARM::VLD3q8oddPseudo_UPD:
3840    case ARM::VLD3q16oddPseudo_UPD:
3841    case ARM::VLD3q32oddPseudo_UPD:
3842    case ARM::VLD4d8Pseudo:
3843    case ARM::VLD4d16Pseudo:
3844    case ARM::VLD4d32Pseudo:
3845    case ARM::VLD1d64QPseudo:
3846    case ARM::VLD1d64QPseudoWB_fixed:
3847    case ARM::VLD4d8Pseudo_UPD:
3848    case ARM::VLD4d16Pseudo_UPD:
3849    case ARM::VLD4d32Pseudo_UPD:
3850    case ARM::VLD4q8Pseudo_UPD:
3851    case ARM::VLD4q16Pseudo_UPD:
3852    case ARM::VLD4q32Pseudo_UPD:
3853    case ARM::VLD4q8oddPseudo:
3854    case ARM::VLD4q16oddPseudo:
3855    case ARM::VLD4q32oddPseudo:
3856    case ARM::VLD4q8oddPseudo_UPD:
3857    case ARM::VLD4q16oddPseudo_UPD:
3858    case ARM::VLD4q32oddPseudo_UPD:
3859    case ARM::VLD1DUPq8:
3860    case ARM::VLD1DUPq16:
3861    case ARM::VLD1DUPq32:
3862    case ARM::VLD1DUPq8wb_fixed:
3863    case ARM::VLD1DUPq16wb_fixed:
3864    case ARM::VLD1DUPq32wb_fixed:
3865    case ARM::VLD1DUPq8wb_register:
3866    case ARM::VLD1DUPq16wb_register:
3867    case ARM::VLD1DUPq32wb_register:
3868    case ARM::VLD2DUPd8:
3869    case ARM::VLD2DUPd16:
3870    case ARM::VLD2DUPd32:
3871    case ARM::VLD2DUPd8wb_fixed:
3872    case ARM::VLD2DUPd16wb_fixed:
3873    case ARM::VLD2DUPd32wb_fixed:
3874    case ARM::VLD2DUPd8wb_register:
3875    case ARM::VLD2DUPd16wb_register:
3876    case ARM::VLD2DUPd32wb_register:
3877    case ARM::VLD4DUPd8Pseudo:
3878    case ARM::VLD4DUPd16Pseudo:
3879    case ARM::VLD4DUPd32Pseudo:
3880    case ARM::VLD4DUPd8Pseudo_UPD:
3881    case ARM::VLD4DUPd16Pseudo_UPD:
3882    case ARM::VLD4DUPd32Pseudo_UPD:
3883    case ARM::VLD1LNq8Pseudo:
3884    case ARM::VLD1LNq16Pseudo:
3885    case ARM::VLD1LNq32Pseudo:
3886    case ARM::VLD1LNq8Pseudo_UPD:
3887    case ARM::VLD1LNq16Pseudo_UPD:
3888    case ARM::VLD1LNq32Pseudo_UPD:
3889    case ARM::VLD2LNd8Pseudo:
3890    case ARM::VLD2LNd16Pseudo:
3891    case ARM::VLD2LNd32Pseudo:
3892    case ARM::VLD2LNq16Pseudo:
3893    case ARM::VLD2LNq32Pseudo:
3894    case ARM::VLD2LNd8Pseudo_UPD:
3895    case ARM::VLD2LNd16Pseudo_UPD:
3896    case ARM::VLD2LNd32Pseudo_UPD:
3897    case ARM::VLD2LNq16Pseudo_UPD:
3898    case ARM::VLD2LNq32Pseudo_UPD:
3899    case ARM::VLD4LNd8Pseudo:
3900    case ARM::VLD4LNd16Pseudo:
3901    case ARM::VLD4LNd32Pseudo:
3902    case ARM::VLD4LNq16Pseudo:
3903    case ARM::VLD4LNq32Pseudo:
3904    case ARM::VLD4LNd8Pseudo_UPD:
3905    case ARM::VLD4LNd16Pseudo_UPD:
3906    case ARM::VLD4LNd32Pseudo_UPD:
3907    case ARM::VLD4LNq16Pseudo_UPD:
3908    case ARM::VLD4LNq32Pseudo_UPD:
3909      // If the address is not 64-bit aligned, the latencies of these
3910      // instructions increases by one.
3911      ++Latency;
3912      break;
3913    }
3914
3915  return Latency;
3916}
3917
3918unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr *MI) const {
3919   if (MI->isCopyLike() || MI->isInsertSubreg() ||
3920      MI->isRegSequence() || MI->isImplicitDef())
3921    return 0;
3922
3923  if (MI->isBundle())
3924    return 0;
3925
3926  const MCInstrDesc &MCID = MI->getDesc();
3927
3928  if (MCID.isCall() || MCID.hasImplicitDefOfPhysReg(ARM::CPSR)) {
3929    // When predicated, CPSR is an additional source operand for CPSR updating
3930    // instructions, this apparently increases their latencies.
3931    return 1;
3932  }
3933  return 0;
3934}
3935
3936unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
3937                                           const MachineInstr *MI,
3938                                           unsigned *PredCost) const {
3939  if (MI->isCopyLike() || MI->isInsertSubreg() ||
3940      MI->isRegSequence() || MI->isImplicitDef())
3941    return 1;
3942
3943  // An instruction scheduler typically runs on unbundled instructions, however
3944  // other passes may query the latency of a bundled instruction.
3945  if (MI->isBundle()) {
3946    unsigned Latency = 0;
3947    MachineBasicBlock::const_instr_iterator I = MI;
3948    MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end();
3949    while (++I != E && I->isInsideBundle()) {
3950      if (I->getOpcode() != ARM::t2IT)
3951        Latency += getInstrLatency(ItinData, I, PredCost);
3952    }
3953    return Latency;
3954  }
3955
3956  const MCInstrDesc &MCID = MI->getDesc();
3957  if (PredCost && (MCID.isCall() || MCID.hasImplicitDefOfPhysReg(ARM::CPSR))) {
3958    // When predicated, CPSR is an additional source operand for CPSR updating
3959    // instructions, this apparently increases their latencies.
3960    *PredCost = 1;
3961  }
3962  // Be sure to call getStageLatency for an empty itinerary in case it has a
3963  // valid MinLatency property.
3964  if (!ItinData)
3965    return MI->mayLoad() ? 3 : 1;
3966
3967  unsigned Class = MCID.getSchedClass();
3968
3969  // For instructions with variable uops, use uops as latency.
3970  if (!ItinData->isEmpty() && ItinData->getNumMicroOps(Class) < 0)
3971    return getNumMicroOps(ItinData, MI);
3972
3973  // For the common case, fall back on the itinerary's latency.
3974  unsigned Latency = ItinData->getStageLatency(Class);
3975
3976  // Adjust for dynamic def-side opcode variants not captured by the itinerary.
3977  unsigned DefAlign = MI->hasOneMemOperand()
3978    ? (*MI->memoperands_begin())->getAlignment() : 0;
3979  int Adj = adjustDefLatency(Subtarget, MI, &MCID, DefAlign);
3980  if (Adj >= 0 || (int)Latency > -Adj) {
3981    return Latency + Adj;
3982  }
3983  return Latency;
3984}
3985
3986int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
3987                                      SDNode *Node) const {
3988  if (!Node->isMachineOpcode())
3989    return 1;
3990
3991  if (!ItinData || ItinData->isEmpty())
3992    return 1;
3993
3994  unsigned Opcode = Node->getMachineOpcode();
3995  switch (Opcode) {
3996  default:
3997    return ItinData->getStageLatency(get(Opcode).getSchedClass());
3998  case ARM::VLDMQIA:
3999  case ARM::VSTMQIA:
4000    return 2;
4001  }
4002}
4003
4004bool ARMBaseInstrInfo::
4005hasHighOperandLatency(const InstrItineraryData *ItinData,
4006                      const MachineRegisterInfo *MRI,
4007                      const MachineInstr *DefMI, unsigned DefIdx,
4008                      const MachineInstr *UseMI, unsigned UseIdx) const {
4009  unsigned DDomain = DefMI->getDesc().TSFlags & ARMII::DomainMask;
4010  unsigned UDomain = UseMI->getDesc().TSFlags & ARMII::DomainMask;
4011  if (Subtarget.isCortexA8() &&
4012      (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP))
4013    // CortexA8 VFP instructions are not pipelined.
4014    return true;
4015
4016  // Hoist VFP / NEON instructions with 4 or higher latency.
4017  int Latency = computeOperandLatency(ItinData, DefMI, DefIdx, UseMI, UseIdx);
4018  if (Latency < 0)
4019    Latency = getInstrLatency(ItinData, DefMI);
4020  if (Latency <= 3)
4021    return false;
4022  return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
4023         UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON;
4024}
4025
4026bool ARMBaseInstrInfo::
4027hasLowDefLatency(const InstrItineraryData *ItinData,
4028                 const MachineInstr *DefMI, unsigned DefIdx) const {
4029  if (!ItinData || ItinData->isEmpty())
4030    return false;
4031
4032  unsigned DDomain = DefMI->getDesc().TSFlags & ARMII::DomainMask;
4033  if (DDomain == ARMII::DomainGeneral) {
4034    unsigned DefClass = DefMI->getDesc().getSchedClass();
4035    int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
4036    return (DefCycle != -1 && DefCycle <= 2);
4037  }
4038  return false;
4039}
4040
4041bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr *MI,
4042                                         StringRef &ErrInfo) const {
4043  if (convertAddSubFlagsOpcode(MI->getOpcode())) {
4044    ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
4045    return false;
4046  }
4047  return true;
4048}
4049
4050// LoadStackGuard has so far only been implemented for MachO. Different code
4051// sequence is needed for other targets.
4052void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
4053                                                unsigned LoadImmOpc,
4054                                                unsigned LoadOpc,
4055                                                Reloc::Model RM) const {
4056  MachineBasicBlock &MBB = *MI->getParent();
4057  DebugLoc DL = MI->getDebugLoc();
4058  unsigned Reg = MI->getOperand(0).getReg();
4059  const GlobalValue *GV =
4060      cast<GlobalValue>((*MI->memoperands_begin())->getValue());
4061  MachineInstrBuilder MIB;
4062
4063  BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg)
4064      .addGlobalAddress(GV, 0, ARMII::MO_NONLAZY);
4065
4066  if (Subtarget.GVIsIndirectSymbol(GV, RM)) {
4067    MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4068    MIB.addReg(Reg, RegState::Kill).addImm(0);
4069    unsigned Flag = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant;
4070    MachineMemOperand *MMO = MBB.getParent()->
4071        getMachineMemOperand(MachinePointerInfo::getGOT(), Flag, 4, 4);
4072    MIB.addMemOperand(MMO);
4073    AddDefaultPred(MIB);
4074  }
4075
4076  MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4077  MIB.addReg(Reg, RegState::Kill).addImm(0);
4078  MIB.setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
4079  AddDefaultPred(MIB);
4080}
4081
4082bool
4083ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
4084                                     unsigned &AddSubOpc,
4085                                     bool &NegAcc, bool &HasLane) const {
4086  DenseMap<unsigned, unsigned>::const_iterator I = MLxEntryMap.find(Opcode);
4087  if (I == MLxEntryMap.end())
4088    return false;
4089
4090  const ARM_MLxEntry &Entry = ARM_MLxTable[I->second];
4091  MulOpc = Entry.MulOpc;
4092  AddSubOpc = Entry.AddSubOpc;
4093  NegAcc = Entry.NegAcc;
4094  HasLane = Entry.HasLane;
4095  return true;
4096}
4097
4098//===----------------------------------------------------------------------===//
4099// Execution domains.
4100//===----------------------------------------------------------------------===//
4101//
4102// Some instructions go down the NEON pipeline, some go down the VFP pipeline,
4103// and some can go down both.  The vmov instructions go down the VFP pipeline,
4104// but they can be changed to vorr equivalents that are executed by the NEON
4105// pipeline.
4106//
4107// We use the following execution domain numbering:
4108//
4109enum ARMExeDomain {
4110  ExeGeneric = 0,
4111  ExeVFP = 1,
4112  ExeNEON = 2
4113};
4114//
4115// Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
4116//
4117std::pair<uint16_t, uint16_t>
4118ARMBaseInstrInfo::getExecutionDomain(const MachineInstr *MI) const {
4119  // If we don't have access to NEON instructions then we won't be able
4120  // to swizzle anything to the NEON domain. Check to make sure.
4121  if (Subtarget.hasNEON()) {
4122    // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON
4123    // if they are not predicated.
4124    if (MI->getOpcode() == ARM::VMOVD && !isPredicated(MI))
4125      return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
4126
4127    // CortexA9 is particularly picky about mixing the two and wants these
4128    // converted.
4129    if (Subtarget.isCortexA9() && !isPredicated(MI) &&
4130        (MI->getOpcode() == ARM::VMOVRS || MI->getOpcode() == ARM::VMOVSR ||
4131         MI->getOpcode() == ARM::VMOVS))
4132      return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
4133  }
4134  // No other instructions can be swizzled, so just determine their domain.
4135  unsigned Domain = MI->getDesc().TSFlags & ARMII::DomainMask;
4136
4137  if (Domain & ARMII::DomainNEON)
4138    return std::make_pair(ExeNEON, 0);
4139
4140  // Certain instructions can go either way on Cortex-A8.
4141  // Treat them as NEON instructions.
4142  if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
4143    return std::make_pair(ExeNEON, 0);
4144
4145  if (Domain & ARMII::DomainVFP)
4146    return std::make_pair(ExeVFP, 0);
4147
4148  return std::make_pair(ExeGeneric, 0);
4149}
4150
4151static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
4152                                            unsigned SReg, unsigned &Lane) {
4153  unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass);
4154  Lane = 0;
4155
4156  if (DReg != ARM::NoRegister)
4157   return DReg;
4158
4159  Lane = 1;
4160  DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass);
4161
4162  assert(DReg && "S-register with no D super-register?");
4163  return DReg;
4164}
4165
4166/// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
4167/// set ImplicitSReg to a register number that must be marked as implicit-use or
4168/// zero if no register needs to be defined as implicit-use.
4169///
4170/// If the function cannot determine if an SPR should be marked implicit use or
4171/// not, it returns false.
4172///
4173/// This function handles cases where an instruction is being modified from taking
4174/// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
4175/// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
4176/// lane of the DPR).
4177///
4178/// If the other SPR is defined, an implicit-use of it should be added. Else,
4179/// (including the case where the DPR itself is defined), it should not.
4180///
4181static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
4182                                       MachineInstr *MI,
4183                                       unsigned DReg, unsigned Lane,
4184                                       unsigned &ImplicitSReg) {
4185  // If the DPR is defined or used already, the other SPR lane will be chained
4186  // correctly, so there is nothing to be done.
4187  if (MI->definesRegister(DReg, TRI) || MI->readsRegister(DReg, TRI)) {
4188    ImplicitSReg = 0;
4189    return true;
4190  }
4191
4192  // Otherwise we need to go searching to see if the SPR is set explicitly.
4193  ImplicitSReg = TRI->getSubReg(DReg,
4194                                (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
4195  MachineBasicBlock::LivenessQueryResult LQR =
4196    MI->getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI);
4197
4198  if (LQR == MachineBasicBlock::LQR_Live)
4199    return true;
4200  else if (LQR == MachineBasicBlock::LQR_Unknown)
4201    return false;
4202
4203  // If the register is known not to be live, there is no need to add an
4204  // implicit-use.
4205  ImplicitSReg = 0;
4206  return true;
4207}
4208
4209void
4210ARMBaseInstrInfo::setExecutionDomain(MachineInstr *MI, unsigned Domain) const {
4211  unsigned DstReg, SrcReg, DReg;
4212  unsigned Lane;
4213  MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI);
4214  const TargetRegisterInfo *TRI = &getRegisterInfo();
4215  switch (MI->getOpcode()) {
4216    default:
4217      llvm_unreachable("cannot handle opcode!");
4218      break;
4219    case ARM::VMOVD:
4220      if (Domain != ExeNEON)
4221        break;
4222
4223      // Zap the predicate operands.
4224      assert(!isPredicated(MI) && "Cannot predicate a VORRd");
4225
4226      // Make sure we've got NEON instructions.
4227      assert(Subtarget.hasNEON() && "VORRd requires NEON");
4228
4229      // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
4230      DstReg = MI->getOperand(0).getReg();
4231      SrcReg = MI->getOperand(1).getReg();
4232
4233      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
4234        MI->RemoveOperand(i-1);
4235
4236      // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
4237      MI->setDesc(get(ARM::VORRd));
4238      AddDefaultPred(MIB.addReg(DstReg, RegState::Define)
4239                        .addReg(SrcReg)
4240                        .addReg(SrcReg));
4241      break;
4242    case ARM::VMOVRS:
4243      if (Domain != ExeNEON)
4244        break;
4245      assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
4246
4247      // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
4248      DstReg = MI->getOperand(0).getReg();
4249      SrcReg = MI->getOperand(1).getReg();
4250
4251      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
4252        MI->RemoveOperand(i-1);
4253
4254      DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane);
4255
4256      // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
4257      // Note that DSrc has been widened and the other lane may be undef, which
4258      // contaminates the entire register.
4259      MI->setDesc(get(ARM::VGETLNi32));
4260      AddDefaultPred(MIB.addReg(DstReg, RegState::Define)
4261                        .addReg(DReg, RegState::Undef)
4262                        .addImm(Lane));
4263
4264      // The old source should be an implicit use, otherwise we might think it
4265      // was dead before here.
4266      MIB.addReg(SrcReg, RegState::Implicit);
4267      break;
4268    case ARM::VMOVSR: {
4269      if (Domain != ExeNEON)
4270        break;
4271      assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
4272
4273      // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
4274      DstReg = MI->getOperand(0).getReg();
4275      SrcReg = MI->getOperand(1).getReg();
4276
4277      DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane);
4278
4279      unsigned ImplicitSReg;
4280      if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
4281        break;
4282
4283      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
4284        MI->RemoveOperand(i-1);
4285
4286      // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
4287      // Again DDst may be undefined at the beginning of this instruction.
4288      MI->setDesc(get(ARM::VSETLNi32));
4289      MIB.addReg(DReg, RegState::Define)
4290         .addReg(DReg, getUndefRegState(!MI->readsRegister(DReg, TRI)))
4291         .addReg(SrcReg)
4292         .addImm(Lane);
4293      AddDefaultPred(MIB);
4294
4295      // The narrower destination must be marked as set to keep previous chains
4296      // in place.
4297      MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
4298      if (ImplicitSReg != 0)
4299        MIB.addReg(ImplicitSReg, RegState::Implicit);
4300      break;
4301    }
4302    case ARM::VMOVS: {
4303      if (Domain != ExeNEON)
4304        break;
4305
4306      // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
4307      DstReg = MI->getOperand(0).getReg();
4308      SrcReg = MI->getOperand(1).getReg();
4309
4310      unsigned DstLane = 0, SrcLane = 0, DDst, DSrc;
4311      DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane);
4312      DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane);
4313
4314      unsigned ImplicitSReg;
4315      if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg))
4316        break;
4317
4318      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
4319        MI->RemoveOperand(i-1);
4320
4321      if (DSrc == DDst) {
4322        // Destination can be:
4323        //     %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
4324        MI->setDesc(get(ARM::VDUPLN32d));
4325        MIB.addReg(DDst, RegState::Define)
4326           .addReg(DDst, getUndefRegState(!MI->readsRegister(DDst, TRI)))
4327           .addImm(SrcLane);
4328        AddDefaultPred(MIB);
4329
4330        // Neither the source or the destination are naturally represented any
4331        // more, so add them in manually.
4332        MIB.addReg(DstReg, RegState::Implicit | RegState::Define);
4333        MIB.addReg(SrcReg, RegState::Implicit);
4334        if (ImplicitSReg != 0)
4335          MIB.addReg(ImplicitSReg, RegState::Implicit);
4336        break;
4337      }
4338
4339      // In general there's no single instruction that can perform an S <-> S
4340      // move in NEON space, but a pair of VEXT instructions *can* do the
4341      // job. It turns out that the VEXTs needed will only use DSrc once, with
4342      // the position based purely on the combination of lane-0 and lane-1
4343      // involved. For example
4344      //     vmov s0, s2 -> vext.32 d0, d0, d1, #1  vext.32 d0, d0, d0, #1
4345      //     vmov s1, s3 -> vext.32 d0, d1, d0, #1  vext.32 d0, d0, d0, #1
4346      //     vmov s0, s3 -> vext.32 d0, d0, d0, #1  vext.32 d0, d1, d0, #1
4347      //     vmov s1, s2 -> vext.32 d0, d0, d0, #1  vext.32 d0, d0, d1, #1
4348      //
4349      // Pattern of the MachineInstrs is:
4350      //     %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
4351      MachineInstrBuilder NewMIB;
4352      NewMIB = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
4353                       get(ARM::VEXTd32), DDst);
4354
4355      // On the first instruction, both DSrc and DDst may be <undef> if present.
4356      // Specifically when the original instruction didn't have them as an
4357      // <imp-use>.
4358      unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
4359      bool CurUndef = !MI->readsRegister(CurReg, TRI);
4360      NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
4361
4362      CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
4363      CurUndef = !MI->readsRegister(CurReg, TRI);
4364      NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
4365
4366      NewMIB.addImm(1);
4367      AddDefaultPred(NewMIB);
4368
4369      if (SrcLane == DstLane)
4370        NewMIB.addReg(SrcReg, RegState::Implicit);
4371
4372      MI->setDesc(get(ARM::VEXTd32));
4373      MIB.addReg(DDst, RegState::Define);
4374
4375      // On the second instruction, DDst has definitely been defined above, so
4376      // it is not <undef>. DSrc, if present, can be <undef> as above.
4377      CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
4378      CurUndef = CurReg == DSrc && !MI->readsRegister(CurReg, TRI);
4379      MIB.addReg(CurReg, getUndefRegState(CurUndef));
4380
4381      CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
4382      CurUndef = CurReg == DSrc && !MI->readsRegister(CurReg, TRI);
4383      MIB.addReg(CurReg, getUndefRegState(CurUndef));
4384
4385      MIB.addImm(1);
4386      AddDefaultPred(MIB);
4387
4388      if (SrcLane != DstLane)
4389        MIB.addReg(SrcReg, RegState::Implicit);
4390
4391      // As before, the original destination is no longer represented, add it
4392      // implicitly.
4393      MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
4394      if (ImplicitSReg != 0)
4395        MIB.addReg(ImplicitSReg, RegState::Implicit);
4396      break;
4397    }
4398  }
4399
4400}
4401
4402//===----------------------------------------------------------------------===//
4403// Partial register updates
4404//===----------------------------------------------------------------------===//
4405//
4406// Swift renames NEON registers with 64-bit granularity.  That means any
4407// instruction writing an S-reg implicitly reads the containing D-reg.  The
4408// problem is mostly avoided by translating f32 operations to v2f32 operations
4409// on D-registers, but f32 loads are still a problem.
4410//
4411// These instructions can load an f32 into a NEON register:
4412//
4413// VLDRS - Only writes S, partial D update.
4414// VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
4415// VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
4416//
4417// FCONSTD can be used as a dependency-breaking instruction.
4418unsigned ARMBaseInstrInfo::
4419getPartialRegUpdateClearance(const MachineInstr *MI,
4420                             unsigned OpNum,
4421                             const TargetRegisterInfo *TRI) const {
4422  if (!SwiftPartialUpdateClearance ||
4423      !(Subtarget.isSwift() || Subtarget.isCortexA15()))
4424    return 0;
4425
4426  assert(TRI && "Need TRI instance");
4427
4428  const MachineOperand &MO = MI->getOperand(OpNum);
4429  if (MO.readsReg())
4430    return 0;
4431  unsigned Reg = MO.getReg();
4432  int UseOp = -1;
4433
4434  switch(MI->getOpcode()) {
4435    // Normal instructions writing only an S-register.
4436  case ARM::VLDRS:
4437  case ARM::FCONSTS:
4438  case ARM::VMOVSR:
4439  case ARM::VMOVv8i8:
4440  case ARM::VMOVv4i16:
4441  case ARM::VMOVv2i32:
4442  case ARM::VMOVv2f32:
4443  case ARM::VMOVv1i64:
4444    UseOp = MI->findRegisterUseOperandIdx(Reg, false, TRI);
4445    break;
4446
4447    // Explicitly reads the dependency.
4448  case ARM::VLD1LNd32:
4449    UseOp = 3;
4450    break;
4451  default:
4452    return 0;
4453  }
4454
4455  // If this instruction actually reads a value from Reg, there is no unwanted
4456  // dependency.
4457  if (UseOp != -1 && MI->getOperand(UseOp).readsReg())
4458    return 0;
4459
4460  // We must be able to clobber the whole D-reg.
4461  if (TargetRegisterInfo::isVirtualRegister(Reg)) {
4462    // Virtual register must be a foo:ssub_0<def,undef> operand.
4463    if (!MO.getSubReg() || MI->readsVirtualRegister(Reg))
4464      return 0;
4465  } else if (ARM::SPRRegClass.contains(Reg)) {
4466    // Physical register: MI must define the full D-reg.
4467    unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0,
4468                                             &ARM::DPRRegClass);
4469    if (!DReg || !MI->definesRegister(DReg, TRI))
4470      return 0;
4471  }
4472
4473  // MI has an unwanted D-register dependency.
4474  // Avoid defs in the previous N instructrions.
4475  return SwiftPartialUpdateClearance;
4476}
4477
4478// Break a partial register dependency after getPartialRegUpdateClearance
4479// returned non-zero.
4480void ARMBaseInstrInfo::
4481breakPartialRegDependency(MachineBasicBlock::iterator MI,
4482                          unsigned OpNum,
4483                          const TargetRegisterInfo *TRI) const {
4484  assert(MI && OpNum < MI->getDesc().getNumDefs() && "OpNum is not a def");
4485  assert(TRI && "Need TRI instance");
4486
4487  const MachineOperand &MO = MI->getOperand(OpNum);
4488  unsigned Reg = MO.getReg();
4489  assert(TargetRegisterInfo::isPhysicalRegister(Reg) &&
4490         "Can't break virtual register dependencies.");
4491  unsigned DReg = Reg;
4492
4493  // If MI defines an S-reg, find the corresponding D super-register.
4494  if (ARM::SPRRegClass.contains(Reg)) {
4495    DReg = ARM::D0 + (Reg - ARM::S0) / 2;
4496    assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
4497  }
4498
4499  assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
4500  assert(MI->definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
4501
4502  // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
4503  // the full D-register by loading the same value to both lanes.  The
4504  // instruction is micro-coded with 2 uops, so don't do this until we can
4505  // properly schedule micro-coded instructions.  The dispatcher stalls cause
4506  // too big regressions.
4507
4508  // Insert the dependency-breaking FCONSTD before MI.
4509  // 96 is the encoding of 0.5, but the actual value doesn't matter here.
4510  AddDefaultPred(BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
4511                         get(ARM::FCONSTD), DReg).addImm(96));
4512  MI->addRegisterKilled(DReg, TRI, true);
4513}
4514
4515bool ARMBaseInstrInfo::hasNOP() const {
4516  return (Subtarget.getFeatureBits() & ARM::HasV6KOps) != 0;
4517}
4518
4519bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
4520  if (MI->getNumOperands() < 4)
4521    return true;
4522  unsigned ShOpVal = MI->getOperand(3).getImm();
4523  unsigned ShImm = ARM_AM::getSORegOffset(ShOpVal);
4524  // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1.
4525  if ((ShImm == 1 && ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsr) ||
4526      ((ShImm == 1 || ShImm == 2) &&
4527       ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsl))
4528    return true;
4529
4530  return false;
4531}
4532
4533bool ARMBaseInstrInfo::getRegSequenceLikeInputs(
4534    const MachineInstr &MI, unsigned DefIdx,
4535    SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
4536  assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
4537  assert(MI.isRegSequenceLike() && "Invalid kind of instruction");
4538
4539  switch (MI.getOpcode()) {
4540  case ARM::VMOVDRR:
4541    // dX = VMOVDRR rY, rZ
4542    // is the same as:
4543    // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1
4544    // Populate the InputRegs accordingly.
4545    // rY
4546    const MachineOperand *MOReg = &MI.getOperand(1);
4547    InputRegs.push_back(
4548        RegSubRegPairAndIdx(MOReg->getReg(), MOReg->getSubReg(), ARM::ssub_0));
4549    // rZ
4550    MOReg = &MI.getOperand(2);
4551    InputRegs.push_back(
4552        RegSubRegPairAndIdx(MOReg->getReg(), MOReg->getSubReg(), ARM::ssub_1));
4553    return true;
4554  }
4555  llvm_unreachable("Target dependent opcode missing");
4556}
4557
4558bool ARMBaseInstrInfo::getExtractSubregLikeInputs(
4559    const MachineInstr &MI, unsigned DefIdx,
4560    RegSubRegPairAndIdx &InputReg) const {
4561  assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
4562  assert(MI.isExtractSubregLike() && "Invalid kind of instruction");
4563
4564  switch (MI.getOpcode()) {
4565  case ARM::VMOVRRD:
4566    // rX, rY = VMOVRRD dZ
4567    // is the same as:
4568    // rX = EXTRACT_SUBREG dZ, ssub_0
4569    // rY = EXTRACT_SUBREG dZ, ssub_1
4570    const MachineOperand &MOReg = MI.getOperand(2);
4571    InputReg.Reg = MOReg.getReg();
4572    InputReg.SubReg = MOReg.getSubReg();
4573    InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1;
4574    return true;
4575  }
4576  llvm_unreachable("Target dependent opcode missing");
4577}
4578
4579bool ARMBaseInstrInfo::getInsertSubregLikeInputs(
4580    const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg,
4581    RegSubRegPairAndIdx &InsertedReg) const {
4582  assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
4583  assert(MI.isInsertSubregLike() && "Invalid kind of instruction");
4584
4585  switch (MI.getOpcode()) {
4586  case ARM::VSETLNi32:
4587    // dX = VSETLNi32 dY, rZ, imm
4588    const MachineOperand &MOBaseReg = MI.getOperand(1);
4589    const MachineOperand &MOInsertedReg = MI.getOperand(2);
4590    const MachineOperand &MOIndex = MI.getOperand(3);
4591    BaseReg.Reg = MOBaseReg.getReg();
4592    BaseReg.SubReg = MOBaseReg.getSubReg();
4593
4594    InsertedReg.Reg = MOInsertedReg.getReg();
4595    InsertedReg.SubReg = MOInsertedReg.getSubReg();
4596    InsertedReg.SubIdx = MOIndex.getImm() == 0 ? ARM::ssub_0 : ARM::ssub_1;
4597    return true;
4598  }
4599  llvm_unreachable("Target dependent opcode missing");
4600}
4601