1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMTargetMachine.h"
17#include "MCTargetDesc/ARMAddressingModes.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/SelectionDAG.h"
23#include "llvm/CodeGen/SelectionDAGISel.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Target/TargetLowering.h"
35#include "llvm/Target/TargetOptions.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE "arm-isel"
40
41static cl::opt<bool>
42DisableShifterOp("disable-shifter-op", cl::Hidden,
43  cl::desc("Disable isel of shifter-op"),
44  cl::init(false));
45
46static cl::opt<bool>
47CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
48  cl::desc("Check fp vmla / vmls hazard at isel time"),
49  cl::init(true));
50
51//===--------------------------------------------------------------------===//
52/// ARMDAGToDAGISel - ARM specific code to select ARM machine
53/// instructions for SelectionDAG operations.
54///
55namespace {
56
57enum AddrMode2Type {
58  AM2_BASE, // Simple AM2 (+-imm12)
59  AM2_SHOP  // Shifter-op AM2
60};
61
62class ARMDAGToDAGISel : public SelectionDAGISel {
63  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
64  /// make the right decision when generating code for different targets.
65  const ARMSubtarget *Subtarget;
66
67public:
68  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
69      : SelectionDAGISel(tm, OptLevel) {}
70
71  bool runOnMachineFunction(MachineFunction &MF) override {
72    // Reset the subtarget each time through.
73    Subtarget = &MF.getTarget().getSubtarget<ARMSubtarget>();
74    SelectionDAGISel::runOnMachineFunction(MF);
75    return true;
76  }
77
78  const char *getPassName() const override {
79    return "ARM Instruction Selection";
80  }
81
82  void PreprocessISelDAG() override;
83
84  /// getI32Imm - Return a target constant of type i32 with the specified
85  /// value.
86  inline SDValue getI32Imm(unsigned Imm) {
87    return CurDAG->getTargetConstant(Imm, MVT::i32);
88  }
89
90  SDNode *Select(SDNode *N) override;
91
92
93  bool hasNoVMLxHazardUse(SDNode *N) const;
94  bool isShifterOpProfitable(const SDValue &Shift,
95                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
96  bool SelectRegShifterOperand(SDValue N, SDValue &A,
97                               SDValue &B, SDValue &C,
98                               bool CheckProfitability = true);
99  bool SelectImmShifterOperand(SDValue N, SDValue &A,
100                               SDValue &B, bool CheckProfitability = true);
101  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
102                                    SDValue &B, SDValue &C) {
103    // Don't apply the profitability check
104    return SelectRegShifterOperand(N, A, B, C, false);
105  }
106  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
107                                    SDValue &B) {
108    // Don't apply the profitability check
109    return SelectImmShifterOperand(N, A, B, false);
110  }
111
112  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
113  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
114
115  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
116                                      SDValue &Offset, SDValue &Opc);
117  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
118                           SDValue &Opc) {
119    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
120  }
121
122  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
123                           SDValue &Opc) {
124    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
125  }
126
127  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
128                       SDValue &Opc) {
129    SelectAddrMode2Worker(N, Base, Offset, Opc);
130//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
131    // This always matches one way or another.
132    return true;
133  }
134
135  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
136    const ConstantSDNode *CN = cast<ConstantSDNode>(N);
137    Pred = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
138    Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
139    return true;
140  }
141
142  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
143                             SDValue &Offset, SDValue &Opc);
144  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
145                             SDValue &Offset, SDValue &Opc);
146  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
147                             SDValue &Offset, SDValue &Opc);
148  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
149  bool SelectAddrMode3(SDValue N, SDValue &Base,
150                       SDValue &Offset, SDValue &Opc);
151  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
152                             SDValue &Offset, SDValue &Opc);
153  bool SelectAddrMode5(SDValue N, SDValue &Base,
154                       SDValue &Offset);
155  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
156  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
157
158  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
159
160  // Thumb Addressing Modes:
161  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
162  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
163                             unsigned Scale);
164  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
165  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
166  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
167  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
168                                SDValue &OffImm);
169  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
170                                 SDValue &OffImm);
171  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
172                                 SDValue &OffImm);
173  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
174                                 SDValue &OffImm);
175  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
176
177  // Thumb 2 Addressing Modes:
178  bool SelectT2ShifterOperandReg(SDValue N,
179                                 SDValue &BaseReg, SDValue &Opc);
180  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
181  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
182                            SDValue &OffImm);
183  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
184                                 SDValue &OffImm);
185  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
186                             SDValue &OffReg, SDValue &ShImm);
187  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
188
189  inline bool is_so_imm(unsigned Imm) const {
190    return ARM_AM::getSOImmVal(Imm) != -1;
191  }
192
193  inline bool is_so_imm_not(unsigned Imm) const {
194    return ARM_AM::getSOImmVal(~Imm) != -1;
195  }
196
197  inline bool is_t2_so_imm(unsigned Imm) const {
198    return ARM_AM::getT2SOImmVal(Imm) != -1;
199  }
200
201  inline bool is_t2_so_imm_not(unsigned Imm) const {
202    return ARM_AM::getT2SOImmVal(~Imm) != -1;
203  }
204
205  // Include the pieces autogenerated from the target description.
206#include "ARMGenDAGISel.inc"
207
208private:
209  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
210  /// ARM.
211  SDNode *SelectARMIndexedLoad(SDNode *N);
212  SDNode *SelectT2IndexedLoad(SDNode *N);
213
214  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
215  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
216  /// loads of D registers and even subregs and odd subregs of Q registers.
217  /// For NumVecs <= 2, QOpcodes1 is not used.
218  SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
219                    const uint16_t *DOpcodes,
220                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
221
222  /// SelectVST - Select NEON store intrinsics.  NumVecs should
223  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
224  /// stores of D registers and even subregs and odd subregs of Q registers.
225  /// For NumVecs <= 2, QOpcodes1 is not used.
226  SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
227                    const uint16_t *DOpcodes,
228                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
229
230  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
231  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
232  /// load/store of D registers and Q registers.
233  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
234                          bool isUpdating, unsigned NumVecs,
235                          const uint16_t *DOpcodes, const uint16_t *QOpcodes);
236
237  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
238  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
239  /// for loading D registers.  (Q registers are not supported.)
240  SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
241                       const uint16_t *Opcodes);
242
243  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
244  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
245  /// generated to force the table registers to be consecutive.
246  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
247
248  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
249  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
250
251  // Select special operations if node forms integer ABS pattern
252  SDNode *SelectABSOp(SDNode *N);
253
254  SDNode *SelectInlineAsm(SDNode *N);
255
256  SDNode *SelectConcatVector(SDNode *N);
257
258  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
259  /// inline asm expressions.
260  bool SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
261                                    std::vector<SDValue> &OutOps) override;
262
263  // Form pairs of consecutive R, S, D, or Q registers.
264  SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
265  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
266  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
267  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
268
269  // Form sequences of 4 consecutive S, D, or Q registers.
270  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
271  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
272  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
273
274  // Get the alignment operand for a NEON VLD or VST instruction.
275  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
276};
277}
278
279/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
280/// operand. If so Imm will receive the 32-bit value.
281static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
282  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
283    Imm = cast<ConstantSDNode>(N)->getZExtValue();
284    return true;
285  }
286  return false;
287}
288
289// isInt32Immediate - This method tests to see if a constant operand.
290// If so Imm will receive the 32 bit value.
291static bool isInt32Immediate(SDValue N, unsigned &Imm) {
292  return isInt32Immediate(N.getNode(), Imm);
293}
294
295// isOpcWithIntImmediate - This method tests to see if the node is a specific
296// opcode and that it has a immediate integer right operand.
297// If so Imm will receive the 32 bit value.
298static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
299  return N->getOpcode() == Opc &&
300         isInt32Immediate(N->getOperand(1).getNode(), Imm);
301}
302
303/// \brief Check whether a particular node is a constant value representable as
304/// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
305///
306/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
307static bool isScaledConstantInRange(SDValue Node, int Scale,
308                                    int RangeMin, int RangeMax,
309                                    int &ScaledConstant) {
310  assert(Scale > 0 && "Invalid scale!");
311
312  // Check that this is a constant.
313  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
314  if (!C)
315    return false;
316
317  ScaledConstant = (int) C->getZExtValue();
318  if ((ScaledConstant % Scale) != 0)
319    return false;
320
321  ScaledConstant /= Scale;
322  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
323}
324
325void ARMDAGToDAGISel::PreprocessISelDAG() {
326  if (!Subtarget->hasV6T2Ops())
327    return;
328
329  bool isThumb2 = Subtarget->isThumb();
330  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
331       E = CurDAG->allnodes_end(); I != E; ) {
332    SDNode *N = I++;  // Preincrement iterator to avoid invalidation issues.
333
334    if (N->getOpcode() != ISD::ADD)
335      continue;
336
337    // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
338    // leading zeros, followed by consecutive set bits, followed by 1 or 2
339    // trailing zeros, e.g. 1020.
340    // Transform the expression to
341    // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
342    // of trailing zeros of c2. The left shift would be folded as an shifter
343    // operand of 'add' and the 'and' and 'srl' would become a bits extraction
344    // node (UBFX).
345
346    SDValue N0 = N->getOperand(0);
347    SDValue N1 = N->getOperand(1);
348    unsigned And_imm = 0;
349    if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
350      if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
351        std::swap(N0, N1);
352    }
353    if (!And_imm)
354      continue;
355
356    // Check if the AND mask is an immediate of the form: 000.....1111111100
357    unsigned TZ = countTrailingZeros(And_imm);
358    if (TZ != 1 && TZ != 2)
359      // Be conservative here. Shifter operands aren't always free. e.g. On
360      // Swift, left shifter operand of 1 / 2 for free but others are not.
361      // e.g.
362      //  ubfx   r3, r1, #16, #8
363      //  ldr.w  r3, [r0, r3, lsl #2]
364      // vs.
365      //  mov.w  r9, #1020
366      //  and.w  r2, r9, r1, lsr #14
367      //  ldr    r2, [r0, r2]
368      continue;
369    And_imm >>= TZ;
370    if (And_imm & (And_imm + 1))
371      continue;
372
373    // Look for (and (srl X, c1), c2).
374    SDValue Srl = N1.getOperand(0);
375    unsigned Srl_imm = 0;
376    if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
377        (Srl_imm <= 2))
378      continue;
379
380    // Make sure first operand is not a shifter operand which would prevent
381    // folding of the left shift.
382    SDValue CPTmp0;
383    SDValue CPTmp1;
384    SDValue CPTmp2;
385    if (isThumb2) {
386      if (SelectT2ShifterOperandReg(N0, CPTmp0, CPTmp1))
387        continue;
388    } else {
389      if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
390          SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
391        continue;
392    }
393
394    // Now make the transformation.
395    Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
396                          Srl.getOperand(0),
397                          CurDAG->getConstant(Srl_imm+TZ, MVT::i32));
398    N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
399                         Srl, CurDAG->getConstant(And_imm, MVT::i32));
400    N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
401                         N1, CurDAG->getConstant(TZ, MVT::i32));
402    CurDAG->UpdateNodeOperands(N, N0, N1);
403  }
404}
405
406/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
407/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
408/// least on current ARM implementations) which should be avoidded.
409bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
410  if (OptLevel == CodeGenOpt::None)
411    return true;
412
413  if (!CheckVMLxHazard)
414    return true;
415
416  if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() &&
417      !Subtarget->isCortexA9() && !Subtarget->isSwift())
418    return true;
419
420  if (!N->hasOneUse())
421    return false;
422
423  SDNode *Use = *N->use_begin();
424  if (Use->getOpcode() == ISD::CopyToReg)
425    return true;
426  if (Use->isMachineOpcode()) {
427    const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
428        CurDAG->getTarget().getInstrInfo());
429
430    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
431    if (MCID.mayStore())
432      return true;
433    unsigned Opcode = MCID.getOpcode();
434    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
435      return true;
436    // vmlx feeding into another vmlx. We actually want to unfold
437    // the use later in the MLxExpansion pass. e.g.
438    // vmla
439    // vmla (stall 8 cycles)
440    //
441    // vmul (5 cycles)
442    // vadd (5 cycles)
443    // vmla
444    // This adds up to about 18 - 19 cycles.
445    //
446    // vmla
447    // vmul (stall 4 cycles)
448    // vadd adds up to about 14 cycles.
449    return TII->isFpMLxInstruction(Opcode);
450  }
451
452  return false;
453}
454
455bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
456                                            ARM_AM::ShiftOpc ShOpcVal,
457                                            unsigned ShAmt) {
458  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
459    return true;
460  if (Shift.hasOneUse())
461    return true;
462  // R << 2 is free.
463  return ShOpcVal == ARM_AM::lsl &&
464         (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
465}
466
467bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
468                                              SDValue &BaseReg,
469                                              SDValue &Opc,
470                                              bool CheckProfitability) {
471  if (DisableShifterOp)
472    return false;
473
474  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
475
476  // Don't match base register only case. That is matched to a separate
477  // lower complexity pattern with explicit register operand.
478  if (ShOpcVal == ARM_AM::no_shift) return false;
479
480  BaseReg = N.getOperand(0);
481  unsigned ShImmVal = 0;
482  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
483  if (!RHS) return false;
484  ShImmVal = RHS->getZExtValue() & 31;
485  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
486                                  MVT::i32);
487  return true;
488}
489
490bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
491                                              SDValue &BaseReg,
492                                              SDValue &ShReg,
493                                              SDValue &Opc,
494                                              bool CheckProfitability) {
495  if (DisableShifterOp)
496    return false;
497
498  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
499
500  // Don't match base register only case. That is matched to a separate
501  // lower complexity pattern with explicit register operand.
502  if (ShOpcVal == ARM_AM::no_shift) return false;
503
504  BaseReg = N.getOperand(0);
505  unsigned ShImmVal = 0;
506  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
507  if (RHS) return false;
508
509  ShReg = N.getOperand(1);
510  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
511    return false;
512  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
513                                  MVT::i32);
514  return true;
515}
516
517
518bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
519                                          SDValue &Base,
520                                          SDValue &OffImm) {
521  // Match simple R + imm12 operands.
522
523  // Base only.
524  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
525      !CurDAG->isBaseWithConstantOffset(N)) {
526    if (N.getOpcode() == ISD::FrameIndex) {
527      // Match frame index.
528      int FI = cast<FrameIndexSDNode>(N)->getIndex();
529      Base = CurDAG->getTargetFrameIndex(FI,
530                                         getTargetLowering()->getPointerTy());
531      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
532      return true;
533    }
534
535    if (N.getOpcode() == ARMISD::Wrapper &&
536        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
537      Base = N.getOperand(0);
538    } else
539      Base = N;
540    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
541    return true;
542  }
543
544  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
545    int RHSC = (int)RHS->getZExtValue();
546    if (N.getOpcode() == ISD::SUB)
547      RHSC = -RHSC;
548
549    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
550      Base   = N.getOperand(0);
551      if (Base.getOpcode() == ISD::FrameIndex) {
552        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
553        Base = CurDAG->getTargetFrameIndex(FI,
554                                           getTargetLowering()->getPointerTy());
555      }
556      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
557      return true;
558    }
559  }
560
561  // Base only.
562  Base = N;
563  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
564  return true;
565}
566
567
568
569bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
570                                      SDValue &Opc) {
571  if (N.getOpcode() == ISD::MUL &&
572      ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
573    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
574      // X * [3,5,9] -> X + X * [2,4,8] etc.
575      int RHSC = (int)RHS->getZExtValue();
576      if (RHSC & 1) {
577        RHSC = RHSC & ~1;
578        ARM_AM::AddrOpc AddSub = ARM_AM::add;
579        if (RHSC < 0) {
580          AddSub = ARM_AM::sub;
581          RHSC = - RHSC;
582        }
583        if (isPowerOf2_32(RHSC)) {
584          unsigned ShAmt = Log2_32(RHSC);
585          Base = Offset = N.getOperand(0);
586          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
587                                                            ARM_AM::lsl),
588                                          MVT::i32);
589          return true;
590        }
591      }
592    }
593  }
594
595  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
596      // ISD::OR that is equivalent to an ISD::ADD.
597      !CurDAG->isBaseWithConstantOffset(N))
598    return false;
599
600  // Leave simple R +/- imm12 operands for LDRi12
601  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
602    int RHSC;
603    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
604                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
605      return false;
606  }
607
608  // Otherwise this is R +/- [possibly shifted] R.
609  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
610  ARM_AM::ShiftOpc ShOpcVal =
611    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
612  unsigned ShAmt = 0;
613
614  Base   = N.getOperand(0);
615  Offset = N.getOperand(1);
616
617  if (ShOpcVal != ARM_AM::no_shift) {
618    // Check to see if the RHS of the shift is a constant, if not, we can't fold
619    // it.
620    if (ConstantSDNode *Sh =
621           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
622      ShAmt = Sh->getZExtValue();
623      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
624        Offset = N.getOperand(1).getOperand(0);
625      else {
626        ShAmt = 0;
627        ShOpcVal = ARM_AM::no_shift;
628      }
629    } else {
630      ShOpcVal = ARM_AM::no_shift;
631    }
632  }
633
634  // Try matching (R shl C) + (R).
635  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
636      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
637        N.getOperand(0).hasOneUse())) {
638    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
639    if (ShOpcVal != ARM_AM::no_shift) {
640      // Check to see if the RHS of the shift is a constant, if not, we can't
641      // fold it.
642      if (ConstantSDNode *Sh =
643          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
644        ShAmt = Sh->getZExtValue();
645        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
646          Offset = N.getOperand(0).getOperand(0);
647          Base = N.getOperand(1);
648        } else {
649          ShAmt = 0;
650          ShOpcVal = ARM_AM::no_shift;
651        }
652      } else {
653        ShOpcVal = ARM_AM::no_shift;
654      }
655    }
656  }
657
658  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
659                                  MVT::i32);
660  return true;
661}
662
663
664//-----
665
666AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
667                                                     SDValue &Base,
668                                                     SDValue &Offset,
669                                                     SDValue &Opc) {
670  if (N.getOpcode() == ISD::MUL &&
671      (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
672    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
673      // X * [3,5,9] -> X + X * [2,4,8] etc.
674      int RHSC = (int)RHS->getZExtValue();
675      if (RHSC & 1) {
676        RHSC = RHSC & ~1;
677        ARM_AM::AddrOpc AddSub = ARM_AM::add;
678        if (RHSC < 0) {
679          AddSub = ARM_AM::sub;
680          RHSC = - RHSC;
681        }
682        if (isPowerOf2_32(RHSC)) {
683          unsigned ShAmt = Log2_32(RHSC);
684          Base = Offset = N.getOperand(0);
685          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
686                                                            ARM_AM::lsl),
687                                          MVT::i32);
688          return AM2_SHOP;
689        }
690      }
691    }
692  }
693
694  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
695      // ISD::OR that is equivalent to an ADD.
696      !CurDAG->isBaseWithConstantOffset(N)) {
697    Base = N;
698    if (N.getOpcode() == ISD::FrameIndex) {
699      int FI = cast<FrameIndexSDNode>(N)->getIndex();
700      Base = CurDAG->getTargetFrameIndex(FI,
701                                         getTargetLowering()->getPointerTy());
702    } else if (N.getOpcode() == ARMISD::Wrapper &&
703               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
704      Base = N.getOperand(0);
705    }
706    Offset = CurDAG->getRegister(0, MVT::i32);
707    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
708                                                      ARM_AM::no_shift),
709                                    MVT::i32);
710    return AM2_BASE;
711  }
712
713  // Match simple R +/- imm12 operands.
714  if (N.getOpcode() != ISD::SUB) {
715    int RHSC;
716    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
717                                -0x1000+1, 0x1000, RHSC)) { // 12 bits.
718      Base = N.getOperand(0);
719      if (Base.getOpcode() == ISD::FrameIndex) {
720        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
721        Base = CurDAG->getTargetFrameIndex(FI,
722                                           getTargetLowering()->getPointerTy());
723      }
724      Offset = CurDAG->getRegister(0, MVT::i32);
725
726      ARM_AM::AddrOpc AddSub = ARM_AM::add;
727      if (RHSC < 0) {
728        AddSub = ARM_AM::sub;
729        RHSC = - RHSC;
730      }
731      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
732                                                        ARM_AM::no_shift),
733                                      MVT::i32);
734      return AM2_BASE;
735    }
736  }
737
738  if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) {
739    // Compute R +/- (R << N) and reuse it.
740    Base = N;
741    Offset = CurDAG->getRegister(0, MVT::i32);
742    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
743                                                      ARM_AM::no_shift),
744                                    MVT::i32);
745    return AM2_BASE;
746  }
747
748  // Otherwise this is R +/- [possibly shifted] R.
749  ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
750  ARM_AM::ShiftOpc ShOpcVal =
751    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
752  unsigned ShAmt = 0;
753
754  Base   = N.getOperand(0);
755  Offset = N.getOperand(1);
756
757  if (ShOpcVal != ARM_AM::no_shift) {
758    // Check to see if the RHS of the shift is a constant, if not, we can't fold
759    // it.
760    if (ConstantSDNode *Sh =
761           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
762      ShAmt = Sh->getZExtValue();
763      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
764        Offset = N.getOperand(1).getOperand(0);
765      else {
766        ShAmt = 0;
767        ShOpcVal = ARM_AM::no_shift;
768      }
769    } else {
770      ShOpcVal = ARM_AM::no_shift;
771    }
772  }
773
774  // Try matching (R shl C) + (R).
775  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
776      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
777        N.getOperand(0).hasOneUse())) {
778    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
779    if (ShOpcVal != ARM_AM::no_shift) {
780      // Check to see if the RHS of the shift is a constant, if not, we can't
781      // fold it.
782      if (ConstantSDNode *Sh =
783          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
784        ShAmt = Sh->getZExtValue();
785        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
786          Offset = N.getOperand(0).getOperand(0);
787          Base = N.getOperand(1);
788        } else {
789          ShAmt = 0;
790          ShOpcVal = ARM_AM::no_shift;
791        }
792      } else {
793        ShOpcVal = ARM_AM::no_shift;
794      }
795    }
796  }
797
798  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
799                                  MVT::i32);
800  return AM2_SHOP;
801}
802
803bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
804                                            SDValue &Offset, SDValue &Opc) {
805  unsigned Opcode = Op->getOpcode();
806  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
807    ? cast<LoadSDNode>(Op)->getAddressingMode()
808    : cast<StoreSDNode>(Op)->getAddressingMode();
809  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
810    ? ARM_AM::add : ARM_AM::sub;
811  int Val;
812  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
813    return false;
814
815  Offset = N;
816  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
817  unsigned ShAmt = 0;
818  if (ShOpcVal != ARM_AM::no_shift) {
819    // Check to see if the RHS of the shift is a constant, if not, we can't fold
820    // it.
821    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
822      ShAmt = Sh->getZExtValue();
823      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
824        Offset = N.getOperand(0);
825      else {
826        ShAmt = 0;
827        ShOpcVal = ARM_AM::no_shift;
828      }
829    } else {
830      ShOpcVal = ARM_AM::no_shift;
831    }
832  }
833
834  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
835                                  MVT::i32);
836  return true;
837}
838
839bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
840                                            SDValue &Offset, SDValue &Opc) {
841  unsigned Opcode = Op->getOpcode();
842  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
843    ? cast<LoadSDNode>(Op)->getAddressingMode()
844    : cast<StoreSDNode>(Op)->getAddressingMode();
845  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
846    ? ARM_AM::add : ARM_AM::sub;
847  int Val;
848  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
849    if (AddSub == ARM_AM::sub) Val *= -1;
850    Offset = CurDAG->getRegister(0, MVT::i32);
851    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
852    return true;
853  }
854
855  return false;
856}
857
858
859bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
860                                            SDValue &Offset, SDValue &Opc) {
861  unsigned Opcode = Op->getOpcode();
862  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
863    ? cast<LoadSDNode>(Op)->getAddressingMode()
864    : cast<StoreSDNode>(Op)->getAddressingMode();
865  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
866    ? ARM_AM::add : ARM_AM::sub;
867  int Val;
868  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
869    Offset = CurDAG->getRegister(0, MVT::i32);
870    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
871                                                      ARM_AM::no_shift),
872                                    MVT::i32);
873    return true;
874  }
875
876  return false;
877}
878
879bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
880  Base = N;
881  return true;
882}
883
884bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
885                                      SDValue &Base, SDValue &Offset,
886                                      SDValue &Opc) {
887  if (N.getOpcode() == ISD::SUB) {
888    // X - C  is canonicalize to X + -C, no need to handle it here.
889    Base = N.getOperand(0);
890    Offset = N.getOperand(1);
891    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
892    return true;
893  }
894
895  if (!CurDAG->isBaseWithConstantOffset(N)) {
896    Base = N;
897    if (N.getOpcode() == ISD::FrameIndex) {
898      int FI = cast<FrameIndexSDNode>(N)->getIndex();
899      Base = CurDAG->getTargetFrameIndex(FI,
900                                         getTargetLowering()->getPointerTy());
901    }
902    Offset = CurDAG->getRegister(0, MVT::i32);
903    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
904    return true;
905  }
906
907  // If the RHS is +/- imm8, fold into addr mode.
908  int RHSC;
909  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
910                              -256 + 1, 256, RHSC)) { // 8 bits.
911    Base = N.getOperand(0);
912    if (Base.getOpcode() == ISD::FrameIndex) {
913      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
914      Base = CurDAG->getTargetFrameIndex(FI,
915                                         getTargetLowering()->getPointerTy());
916    }
917    Offset = CurDAG->getRegister(0, MVT::i32);
918
919    ARM_AM::AddrOpc AddSub = ARM_AM::add;
920    if (RHSC < 0) {
921      AddSub = ARM_AM::sub;
922      RHSC = -RHSC;
923    }
924    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
925    return true;
926  }
927
928  Base = N.getOperand(0);
929  Offset = N.getOperand(1);
930  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
931  return true;
932}
933
934bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
935                                            SDValue &Offset, SDValue &Opc) {
936  unsigned Opcode = Op->getOpcode();
937  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
938    ? cast<LoadSDNode>(Op)->getAddressingMode()
939    : cast<StoreSDNode>(Op)->getAddressingMode();
940  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
941    ? ARM_AM::add : ARM_AM::sub;
942  int Val;
943  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
944    Offset = CurDAG->getRegister(0, MVT::i32);
945    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
946    return true;
947  }
948
949  Offset = N;
950  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
951  return true;
952}
953
954bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
955                                      SDValue &Base, SDValue &Offset) {
956  if (!CurDAG->isBaseWithConstantOffset(N)) {
957    Base = N;
958    if (N.getOpcode() == ISD::FrameIndex) {
959      int FI = cast<FrameIndexSDNode>(N)->getIndex();
960      Base = CurDAG->getTargetFrameIndex(FI,
961                                         getTargetLowering()->getPointerTy());
962    } else if (N.getOpcode() == ARMISD::Wrapper &&
963               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
964      Base = N.getOperand(0);
965    }
966    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
967                                       MVT::i32);
968    return true;
969  }
970
971  // If the RHS is +/- imm8, fold into addr mode.
972  int RHSC;
973  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
974                              -256 + 1, 256, RHSC)) {
975    Base = N.getOperand(0);
976    if (Base.getOpcode() == ISD::FrameIndex) {
977      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
978      Base = CurDAG->getTargetFrameIndex(FI,
979                                         getTargetLowering()->getPointerTy());
980    }
981
982    ARM_AM::AddrOpc AddSub = ARM_AM::add;
983    if (RHSC < 0) {
984      AddSub = ARM_AM::sub;
985      RHSC = -RHSC;
986    }
987    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
988                                       MVT::i32);
989    return true;
990  }
991
992  Base = N;
993  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
994                                     MVT::i32);
995  return true;
996}
997
998bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
999                                      SDValue &Align) {
1000  Addr = N;
1001
1002  unsigned Alignment = 0;
1003  if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
1004    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
1005    // The maximum alignment is equal to the memory size being referenced.
1006    unsigned LSNAlign = LSN->getAlignment();
1007    unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
1008    if (LSNAlign >= MemSize && MemSize > 1)
1009      Alignment = MemSize;
1010  } else {
1011    // All other uses of addrmode6 are for intrinsics.  For now just record
1012    // the raw alignment value; it will be refined later based on the legal
1013    // alignment operands for the intrinsic.
1014    Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
1015  }
1016
1017  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1018  return true;
1019}
1020
1021bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1022                                            SDValue &Offset) {
1023  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1024  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
1025  if (AM != ISD::POST_INC)
1026    return false;
1027  Offset = N;
1028  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1029    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1030      Offset = CurDAG->getRegister(0, MVT::i32);
1031  }
1032  return true;
1033}
1034
1035bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1036                                       SDValue &Offset, SDValue &Label) {
1037  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1038    Offset = N.getOperand(0);
1039    SDValue N1 = N.getOperand(1);
1040    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1041                                      MVT::i32);
1042    return true;
1043  }
1044
1045  return false;
1046}
1047
1048
1049//===----------------------------------------------------------------------===//
1050//                         Thumb Addressing Modes
1051//===----------------------------------------------------------------------===//
1052
1053bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
1054                                            SDValue &Base, SDValue &Offset){
1055  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1056    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
1057    if (!NC || !NC->isNullValue())
1058      return false;
1059
1060    Base = Offset = N;
1061    return true;
1062  }
1063
1064  Base = N.getOperand(0);
1065  Offset = N.getOperand(1);
1066  return true;
1067}
1068
1069bool
1070ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
1071                                       SDValue &Offset, unsigned Scale) {
1072  if (Scale == 4) {
1073    SDValue TmpBase, TmpOffImm;
1074    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1075      return false;  // We want to select tLDRspi / tSTRspi instead.
1076
1077    if (N.getOpcode() == ARMISD::Wrapper &&
1078        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1079      return false;  // We want to select tLDRpci instead.
1080  }
1081
1082  if (!CurDAG->isBaseWithConstantOffset(N))
1083    return false;
1084
1085  // Thumb does not have [sp, r] address mode.
1086  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1087  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1088  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1089      (RHSR && RHSR->getReg() == ARM::SP))
1090    return false;
1091
1092  // FIXME: Why do we explicitly check for a match here and then return false?
1093  // Presumably to allow something else to match, but shouldn't this be
1094  // documented?
1095  int RHSC;
1096  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
1097    return false;
1098
1099  Base = N.getOperand(0);
1100  Offset = N.getOperand(1);
1101  return true;
1102}
1103
1104bool
1105ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
1106                                          SDValue &Base,
1107                                          SDValue &Offset) {
1108  return SelectThumbAddrModeRI(N, Base, Offset, 1);
1109}
1110
1111bool
1112ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
1113                                          SDValue &Base,
1114                                          SDValue &Offset) {
1115  return SelectThumbAddrModeRI(N, Base, Offset, 2);
1116}
1117
1118bool
1119ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
1120                                          SDValue &Base,
1121                                          SDValue &Offset) {
1122  return SelectThumbAddrModeRI(N, Base, Offset, 4);
1123}
1124
1125bool
1126ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1127                                          SDValue &Base, SDValue &OffImm) {
1128  if (Scale == 4) {
1129    SDValue TmpBase, TmpOffImm;
1130    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1131      return false;  // We want to select tLDRspi / tSTRspi instead.
1132
1133    if (N.getOpcode() == ARMISD::Wrapper &&
1134        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1135      return false;  // We want to select tLDRpci instead.
1136  }
1137
1138  if (!CurDAG->isBaseWithConstantOffset(N)) {
1139    if (N.getOpcode() == ARMISD::Wrapper &&
1140        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
1141      Base = N.getOperand(0);
1142    } else {
1143      Base = N;
1144    }
1145
1146    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1147    return true;
1148  }
1149
1150  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1151  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1152  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1153      (RHSR && RHSR->getReg() == ARM::SP)) {
1154    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1155    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1156    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1157    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1158
1159    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1160    if (LHSC != 0 || RHSC != 0) return false;
1161
1162    Base = N;
1163    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1164    return true;
1165  }
1166
1167  // If the RHS is + imm5 * scale, fold into addr mode.
1168  int RHSC;
1169  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1170    Base = N.getOperand(0);
1171    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1172    return true;
1173  }
1174
1175  Base = N.getOperand(0);
1176  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1177  return true;
1178}
1179
1180bool
1181ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1182                                           SDValue &OffImm) {
1183  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1184}
1185
1186bool
1187ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1188                                           SDValue &OffImm) {
1189  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1190}
1191
1192bool
1193ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1194                                           SDValue &OffImm) {
1195  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1196}
1197
1198bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1199                                            SDValue &Base, SDValue &OffImm) {
1200  if (N.getOpcode() == ISD::FrameIndex) {
1201    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1202    Base = CurDAG->getTargetFrameIndex(FI,
1203                                       getTargetLowering()->getPointerTy());
1204    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1205    return true;
1206  }
1207
1208  if (!CurDAG->isBaseWithConstantOffset(N))
1209    return false;
1210
1211  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1212  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1213      (LHSR && LHSR->getReg() == ARM::SP)) {
1214    // If the RHS is + imm8 * scale, fold into addr mode.
1215    int RHSC;
1216    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1217      Base = N.getOperand(0);
1218      if (Base.getOpcode() == ISD::FrameIndex) {
1219        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1220        Base = CurDAG->getTargetFrameIndex(FI,
1221                                           getTargetLowering()->getPointerTy());
1222      }
1223      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1224      return true;
1225    }
1226  }
1227
1228  return false;
1229}
1230
1231
1232//===----------------------------------------------------------------------===//
1233//                        Thumb 2 Addressing Modes
1234//===----------------------------------------------------------------------===//
1235
1236
1237bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1238                                                SDValue &Opc) {
1239  if (DisableShifterOp)
1240    return false;
1241
1242  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1243
1244  // Don't match base register only case. That is matched to a separate
1245  // lower complexity pattern with explicit register operand.
1246  if (ShOpcVal == ARM_AM::no_shift) return false;
1247
1248  BaseReg = N.getOperand(0);
1249  unsigned ShImmVal = 0;
1250  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1251    ShImmVal = RHS->getZExtValue() & 31;
1252    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1253    return true;
1254  }
1255
1256  return false;
1257}
1258
1259bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1260                                            SDValue &Base, SDValue &OffImm) {
1261  // Match simple R + imm12 operands.
1262
1263  // Base only.
1264  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1265      !CurDAG->isBaseWithConstantOffset(N)) {
1266    if (N.getOpcode() == ISD::FrameIndex) {
1267      // Match frame index.
1268      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1269      Base = CurDAG->getTargetFrameIndex(FI,
1270                                         getTargetLowering()->getPointerTy());
1271      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1272      return true;
1273    }
1274
1275    if (N.getOpcode() == ARMISD::Wrapper &&
1276        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
1277      Base = N.getOperand(0);
1278      if (Base.getOpcode() == ISD::TargetConstantPool)
1279        return false;  // We want to select t2LDRpci instead.
1280    } else
1281      Base = N;
1282    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1283    return true;
1284  }
1285
1286  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1287    if (SelectT2AddrModeImm8(N, Base, OffImm))
1288      // Let t2LDRi8 handle (R - imm8).
1289      return false;
1290
1291    int RHSC = (int)RHS->getZExtValue();
1292    if (N.getOpcode() == ISD::SUB)
1293      RHSC = -RHSC;
1294
1295    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1296      Base   = N.getOperand(0);
1297      if (Base.getOpcode() == ISD::FrameIndex) {
1298        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1299        Base = CurDAG->getTargetFrameIndex(FI,
1300                                           getTargetLowering()->getPointerTy());
1301      }
1302      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1303      return true;
1304    }
1305  }
1306
1307  // Base only.
1308  Base = N;
1309  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1310  return true;
1311}
1312
1313bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1314                                           SDValue &Base, SDValue &OffImm) {
1315  // Match simple R - imm8 operands.
1316  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1317      !CurDAG->isBaseWithConstantOffset(N))
1318    return false;
1319
1320  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1321    int RHSC = (int)RHS->getSExtValue();
1322    if (N.getOpcode() == ISD::SUB)
1323      RHSC = -RHSC;
1324
1325    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1326      Base = N.getOperand(0);
1327      if (Base.getOpcode() == ISD::FrameIndex) {
1328        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1329        Base = CurDAG->getTargetFrameIndex(FI,
1330                                           getTargetLowering()->getPointerTy());
1331      }
1332      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1333      return true;
1334    }
1335  }
1336
1337  return false;
1338}
1339
1340bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1341                                                 SDValue &OffImm){
1342  unsigned Opcode = Op->getOpcode();
1343  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1344    ? cast<LoadSDNode>(Op)->getAddressingMode()
1345    : cast<StoreSDNode>(Op)->getAddressingMode();
1346  int RHSC;
1347  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1348    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1349      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1350      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1351    return true;
1352  }
1353
1354  return false;
1355}
1356
1357bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1358                                            SDValue &Base,
1359                                            SDValue &OffReg, SDValue &ShImm) {
1360  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1361  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1362    return false;
1363
1364  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1365  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1366    int RHSC = (int)RHS->getZExtValue();
1367    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1368      return false;
1369    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1370      return false;
1371  }
1372
1373  // Look for (R + R) or (R + (R << [1,2,3])).
1374  unsigned ShAmt = 0;
1375  Base   = N.getOperand(0);
1376  OffReg = N.getOperand(1);
1377
1378  // Swap if it is ((R << c) + R).
1379  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1380  if (ShOpcVal != ARM_AM::lsl) {
1381    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1382    if (ShOpcVal == ARM_AM::lsl)
1383      std::swap(Base, OffReg);
1384  }
1385
1386  if (ShOpcVal == ARM_AM::lsl) {
1387    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1388    // it.
1389    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1390      ShAmt = Sh->getZExtValue();
1391      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1392        OffReg = OffReg.getOperand(0);
1393      else {
1394        ShAmt = 0;
1395        ShOpcVal = ARM_AM::no_shift;
1396      }
1397    } else {
1398      ShOpcVal = ARM_AM::no_shift;
1399    }
1400  }
1401
1402  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1403
1404  return true;
1405}
1406
1407bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1408                                                SDValue &OffImm) {
1409  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1410  // instructions.
1411  Base = N;
1412  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1413
1414  if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1415    return true;
1416
1417  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1418  if (!RHS)
1419    return true;
1420
1421  uint32_t RHSC = (int)RHS->getZExtValue();
1422  if (RHSC > 1020 || RHSC % 4 != 0)
1423    return true;
1424
1425  Base = N.getOperand(0);
1426  if (Base.getOpcode() == ISD::FrameIndex) {
1427    int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1428    Base = CurDAG->getTargetFrameIndex(FI, getTargetLowering()->getPointerTy());
1429  }
1430
1431  OffImm = CurDAG->getTargetConstant(RHSC / 4, MVT::i32);
1432  return true;
1433}
1434
1435//===--------------------------------------------------------------------===//
1436
1437/// getAL - Returns a ARMCC::AL immediate node.
1438static inline SDValue getAL(SelectionDAG *CurDAG) {
1439  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1440}
1441
1442SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1443  LoadSDNode *LD = cast<LoadSDNode>(N);
1444  ISD::MemIndexedMode AM = LD->getAddressingMode();
1445  if (AM == ISD::UNINDEXED)
1446    return nullptr;
1447
1448  EVT LoadedVT = LD->getMemoryVT();
1449  SDValue Offset, AMOpc;
1450  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1451  unsigned Opcode = 0;
1452  bool Match = false;
1453  if (LoadedVT == MVT::i32 && isPre &&
1454      SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1455    Opcode = ARM::LDR_PRE_IMM;
1456    Match = true;
1457  } else if (LoadedVT == MVT::i32 && !isPre &&
1458      SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1459    Opcode = ARM::LDR_POST_IMM;
1460    Match = true;
1461  } else if (LoadedVT == MVT::i32 &&
1462      SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1463    Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1464    Match = true;
1465
1466  } else if (LoadedVT == MVT::i16 &&
1467             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1468    Match = true;
1469    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1470      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1471      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1472  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1473    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1474      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1475        Match = true;
1476        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1477      }
1478    } else {
1479      if (isPre &&
1480          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1481        Match = true;
1482        Opcode = ARM::LDRB_PRE_IMM;
1483      } else if (!isPre &&
1484                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1485        Match = true;
1486        Opcode = ARM::LDRB_POST_IMM;
1487      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1488        Match = true;
1489        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1490      }
1491    }
1492  }
1493
1494  if (Match) {
1495    if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1496      SDValue Chain = LD->getChain();
1497      SDValue Base = LD->getBasePtr();
1498      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
1499                       CurDAG->getRegister(0, MVT::i32), Chain };
1500      return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1501                                    MVT::i32, MVT::Other, Ops);
1502    } else {
1503      SDValue Chain = LD->getChain();
1504      SDValue Base = LD->getBasePtr();
1505      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1506                       CurDAG->getRegister(0, MVT::i32), Chain };
1507      return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1508                                    MVT::i32, MVT::Other, Ops);
1509    }
1510  }
1511
1512  return nullptr;
1513}
1514
1515SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1516  LoadSDNode *LD = cast<LoadSDNode>(N);
1517  ISD::MemIndexedMode AM = LD->getAddressingMode();
1518  if (AM == ISD::UNINDEXED)
1519    return nullptr;
1520
1521  EVT LoadedVT = LD->getMemoryVT();
1522  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1523  SDValue Offset;
1524  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1525  unsigned Opcode = 0;
1526  bool Match = false;
1527  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1528    switch (LoadedVT.getSimpleVT().SimpleTy) {
1529    case MVT::i32:
1530      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1531      break;
1532    case MVT::i16:
1533      if (isSExtLd)
1534        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1535      else
1536        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1537      break;
1538    case MVT::i8:
1539    case MVT::i1:
1540      if (isSExtLd)
1541        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1542      else
1543        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1544      break;
1545    default:
1546      return nullptr;
1547    }
1548    Match = true;
1549  }
1550
1551  if (Match) {
1552    SDValue Chain = LD->getChain();
1553    SDValue Base = LD->getBasePtr();
1554    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1555                     CurDAG->getRegister(0, MVT::i32), Chain };
1556    return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1557                                  MVT::Other, Ops);
1558  }
1559
1560  return nullptr;
1561}
1562
1563/// \brief Form a GPRPair pseudo register from a pair of GPR regs.
1564SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1565  SDLoc dl(V0.getNode());
1566  SDValue RegClass =
1567    CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32);
1568  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
1569  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
1570  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1571  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1572}
1573
1574/// \brief Form a D register from a pair of S registers.
1575SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1576  SDLoc dl(V0.getNode());
1577  SDValue RegClass =
1578    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1579  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1580  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1581  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1582  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1583}
1584
1585/// \brief Form a quad register from a pair of D registers.
1586SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1587  SDLoc dl(V0.getNode());
1588  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1589  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1590  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1591  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1592  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1593}
1594
1595/// \brief Form 4 consecutive D registers from a pair of Q registers.
1596SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1597  SDLoc dl(V0.getNode());
1598  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1599  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1600  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1601  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1602  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1603}
1604
1605/// \brief Form 4 consecutive S registers.
1606SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1607                                   SDValue V2, SDValue V3) {
1608  SDLoc dl(V0.getNode());
1609  SDValue RegClass =
1610    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1611  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1612  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1613  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1614  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1615  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1616                                    V2, SubReg2, V3, SubReg3 };
1617  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1618}
1619
1620/// \brief Form 4 consecutive D registers.
1621SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1622                                   SDValue V2, SDValue V3) {
1623  SDLoc dl(V0.getNode());
1624  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1625  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1626  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1627  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1628  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1629  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1630                                    V2, SubReg2, V3, SubReg3 };
1631  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1632}
1633
1634/// \brief Form 4 consecutive Q registers.
1635SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1636                                   SDValue V2, SDValue V3) {
1637  SDLoc dl(V0.getNode());
1638  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1639  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1640  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1641  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1642  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1643  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1644                                    V2, SubReg2, V3, SubReg3 };
1645  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1646}
1647
1648/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1649/// of a NEON VLD or VST instruction.  The supported values depend on the
1650/// number of registers being loaded.
1651SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1652                                       bool is64BitVector) {
1653  unsigned NumRegs = NumVecs;
1654  if (!is64BitVector && NumVecs < 3)
1655    NumRegs *= 2;
1656
1657  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1658  if (Alignment >= 32 && NumRegs == 4)
1659    Alignment = 32;
1660  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1661    Alignment = 16;
1662  else if (Alignment >= 8)
1663    Alignment = 8;
1664  else
1665    Alignment = 0;
1666
1667  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1668}
1669
1670static bool isVLDfixed(unsigned Opc)
1671{
1672  switch (Opc) {
1673  default: return false;
1674  case ARM::VLD1d8wb_fixed : return true;
1675  case ARM::VLD1d16wb_fixed : return true;
1676  case ARM::VLD1d64Qwb_fixed : return true;
1677  case ARM::VLD1d32wb_fixed : return true;
1678  case ARM::VLD1d64wb_fixed : return true;
1679  case ARM::VLD1d64TPseudoWB_fixed : return true;
1680  case ARM::VLD1d64QPseudoWB_fixed : return true;
1681  case ARM::VLD1q8wb_fixed : return true;
1682  case ARM::VLD1q16wb_fixed : return true;
1683  case ARM::VLD1q32wb_fixed : return true;
1684  case ARM::VLD1q64wb_fixed : return true;
1685  case ARM::VLD2d8wb_fixed : return true;
1686  case ARM::VLD2d16wb_fixed : return true;
1687  case ARM::VLD2d32wb_fixed : return true;
1688  case ARM::VLD2q8PseudoWB_fixed : return true;
1689  case ARM::VLD2q16PseudoWB_fixed : return true;
1690  case ARM::VLD2q32PseudoWB_fixed : return true;
1691  case ARM::VLD2DUPd8wb_fixed : return true;
1692  case ARM::VLD2DUPd16wb_fixed : return true;
1693  case ARM::VLD2DUPd32wb_fixed : return true;
1694  }
1695}
1696
1697static bool isVSTfixed(unsigned Opc)
1698{
1699  switch (Opc) {
1700  default: return false;
1701  case ARM::VST1d8wb_fixed : return true;
1702  case ARM::VST1d16wb_fixed : return true;
1703  case ARM::VST1d32wb_fixed : return true;
1704  case ARM::VST1d64wb_fixed : return true;
1705  case ARM::VST1q8wb_fixed : return true;
1706  case ARM::VST1q16wb_fixed : return true;
1707  case ARM::VST1q32wb_fixed : return true;
1708  case ARM::VST1q64wb_fixed : return true;
1709  case ARM::VST1d64TPseudoWB_fixed : return true;
1710  case ARM::VST1d64QPseudoWB_fixed : return true;
1711  case ARM::VST2d8wb_fixed : return true;
1712  case ARM::VST2d16wb_fixed : return true;
1713  case ARM::VST2d32wb_fixed : return true;
1714  case ARM::VST2q8PseudoWB_fixed : return true;
1715  case ARM::VST2q16PseudoWB_fixed : return true;
1716  case ARM::VST2q32PseudoWB_fixed : return true;
1717  }
1718}
1719
1720// Get the register stride update opcode of a VLD/VST instruction that
1721// is otherwise equivalent to the given fixed stride updating instruction.
1722static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1723  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1724    && "Incorrect fixed stride updating instruction.");
1725  switch (Opc) {
1726  default: break;
1727  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1728  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1729  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1730  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1731  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1732  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1733  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1734  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1735  case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
1736  case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
1737  case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
1738  case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
1739
1740  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1741  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1742  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1743  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1744  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1745  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1746  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1747  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1748  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1749  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1750
1751  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1752  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1753  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1754  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1755  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1756  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1757
1758  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1759  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1760  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1761  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1762  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1763  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1764
1765  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1766  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1767  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1768  }
1769  return Opc; // If not one we handle, return it unchanged.
1770}
1771
1772SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1773                                   const uint16_t *DOpcodes,
1774                                   const uint16_t *QOpcodes0,
1775                                   const uint16_t *QOpcodes1) {
1776  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1777  SDLoc dl(N);
1778
1779  SDValue MemAddr, Align;
1780  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1781  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1782    return nullptr;
1783
1784  SDValue Chain = N->getOperand(0);
1785  EVT VT = N->getValueType(0);
1786  bool is64BitVector = VT.is64BitVector();
1787  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1788
1789  unsigned OpcodeIndex;
1790  switch (VT.getSimpleVT().SimpleTy) {
1791  default: llvm_unreachable("unhandled vld type");
1792    // Double-register operations:
1793  case MVT::v8i8:  OpcodeIndex = 0; break;
1794  case MVT::v4i16: OpcodeIndex = 1; break;
1795  case MVT::v2f32:
1796  case MVT::v2i32: OpcodeIndex = 2; break;
1797  case MVT::v1i64: OpcodeIndex = 3; break;
1798    // Quad-register operations:
1799  case MVT::v16i8: OpcodeIndex = 0; break;
1800  case MVT::v8i16: OpcodeIndex = 1; break;
1801  case MVT::v4f32:
1802  case MVT::v4i32: OpcodeIndex = 2; break;
1803  case MVT::v2i64: OpcodeIndex = 3;
1804    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1805    break;
1806  }
1807
1808  EVT ResTy;
1809  if (NumVecs == 1)
1810    ResTy = VT;
1811  else {
1812    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1813    if (!is64BitVector)
1814      ResTyElts *= 2;
1815    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1816  }
1817  std::vector<EVT> ResTys;
1818  ResTys.push_back(ResTy);
1819  if (isUpdating)
1820    ResTys.push_back(MVT::i32);
1821  ResTys.push_back(MVT::Other);
1822
1823  SDValue Pred = getAL(CurDAG);
1824  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1825  SDNode *VLd;
1826  SmallVector<SDValue, 7> Ops;
1827
1828  // Double registers and VLD1/VLD2 quad registers are directly supported.
1829  if (is64BitVector || NumVecs <= 2) {
1830    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1831                    QOpcodes0[OpcodeIndex]);
1832    Ops.push_back(MemAddr);
1833    Ops.push_back(Align);
1834    if (isUpdating) {
1835      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1836      // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
1837      // case entirely when the rest are updated to that form, too.
1838      if ((NumVecs <= 2) && !isa<ConstantSDNode>(Inc.getNode()))
1839        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1840      // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1841      // check for that explicitly too. Horribly hacky, but temporary.
1842      if ((NumVecs > 2 && !isVLDfixed(Opc)) ||
1843          !isa<ConstantSDNode>(Inc.getNode()))
1844        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1845    }
1846    Ops.push_back(Pred);
1847    Ops.push_back(Reg0);
1848    Ops.push_back(Chain);
1849    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1850
1851  } else {
1852    // Otherwise, quad registers are loaded with two separate instructions,
1853    // where one loads the even registers and the other loads the odd registers.
1854    EVT AddrTy = MemAddr.getValueType();
1855
1856    // Load the even subregs.  This is always an updating load, so that it
1857    // provides the address to the second load for the odd subregs.
1858    SDValue ImplDef =
1859      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1860    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1861    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1862                                          ResTy, AddrTy, MVT::Other, OpsA);
1863    Chain = SDValue(VLdA, 2);
1864
1865    // Load the odd subregs.
1866    Ops.push_back(SDValue(VLdA, 1));
1867    Ops.push_back(Align);
1868    if (isUpdating) {
1869      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1870      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1871             "only constant post-increment update allowed for VLD3/4");
1872      (void)Inc;
1873      Ops.push_back(Reg0);
1874    }
1875    Ops.push_back(SDValue(VLdA, 0));
1876    Ops.push_back(Pred);
1877    Ops.push_back(Reg0);
1878    Ops.push_back(Chain);
1879    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1880  }
1881
1882  // Transfer memoperands.
1883  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1884  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1885  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1886
1887  if (NumVecs == 1)
1888    return VLd;
1889
1890  // Extract out the subregisters.
1891  SDValue SuperReg = SDValue(VLd, 0);
1892  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1893         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1894  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1895  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1896    ReplaceUses(SDValue(N, Vec),
1897                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1898  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1899  if (isUpdating)
1900    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1901  return nullptr;
1902}
1903
1904SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1905                                   const uint16_t *DOpcodes,
1906                                   const uint16_t *QOpcodes0,
1907                                   const uint16_t *QOpcodes1) {
1908  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1909  SDLoc dl(N);
1910
1911  SDValue MemAddr, Align;
1912  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1913  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1914  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1915    return nullptr;
1916
1917  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1918  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1919
1920  SDValue Chain = N->getOperand(0);
1921  EVT VT = N->getOperand(Vec0Idx).getValueType();
1922  bool is64BitVector = VT.is64BitVector();
1923  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1924
1925  unsigned OpcodeIndex;
1926  switch (VT.getSimpleVT().SimpleTy) {
1927  default: llvm_unreachable("unhandled vst type");
1928    // Double-register operations:
1929  case MVT::v8i8:  OpcodeIndex = 0; break;
1930  case MVT::v4i16: OpcodeIndex = 1; break;
1931  case MVT::v2f32:
1932  case MVT::v2i32: OpcodeIndex = 2; break;
1933  case MVT::v1i64: OpcodeIndex = 3; break;
1934    // Quad-register operations:
1935  case MVT::v16i8: OpcodeIndex = 0; break;
1936  case MVT::v8i16: OpcodeIndex = 1; break;
1937  case MVT::v4f32:
1938  case MVT::v4i32: OpcodeIndex = 2; break;
1939  case MVT::v2i64: OpcodeIndex = 3;
1940    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1941    break;
1942  }
1943
1944  std::vector<EVT> ResTys;
1945  if (isUpdating)
1946    ResTys.push_back(MVT::i32);
1947  ResTys.push_back(MVT::Other);
1948
1949  SDValue Pred = getAL(CurDAG);
1950  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1951  SmallVector<SDValue, 7> Ops;
1952
1953  // Double registers and VST1/VST2 quad registers are directly supported.
1954  if (is64BitVector || NumVecs <= 2) {
1955    SDValue SrcReg;
1956    if (NumVecs == 1) {
1957      SrcReg = N->getOperand(Vec0Idx);
1958    } else if (is64BitVector) {
1959      // Form a REG_SEQUENCE to force register allocation.
1960      SDValue V0 = N->getOperand(Vec0Idx + 0);
1961      SDValue V1 = N->getOperand(Vec0Idx + 1);
1962      if (NumVecs == 2)
1963        SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
1964      else {
1965        SDValue V2 = N->getOperand(Vec0Idx + 2);
1966        // If it's a vst3, form a quad D-register and leave the last part as
1967        // an undef.
1968        SDValue V3 = (NumVecs == 3)
1969          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1970          : N->getOperand(Vec0Idx + 3);
1971        SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
1972      }
1973    } else {
1974      // Form a QQ register.
1975      SDValue Q0 = N->getOperand(Vec0Idx);
1976      SDValue Q1 = N->getOperand(Vec0Idx + 1);
1977      SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
1978    }
1979
1980    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1981                    QOpcodes0[OpcodeIndex]);
1982    Ops.push_back(MemAddr);
1983    Ops.push_back(Align);
1984    if (isUpdating) {
1985      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1986      // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
1987      // case entirely when the rest are updated to that form, too.
1988      if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
1989        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1990      // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
1991      // check for that explicitly too. Horribly hacky, but temporary.
1992      if  (!isa<ConstantSDNode>(Inc.getNode()))
1993        Ops.push_back(Inc);
1994      else if (NumVecs > 2 && !isVSTfixed(Opc))
1995        Ops.push_back(Reg0);
1996    }
1997    Ops.push_back(SrcReg);
1998    Ops.push_back(Pred);
1999    Ops.push_back(Reg0);
2000    Ops.push_back(Chain);
2001    SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2002
2003    // Transfer memoperands.
2004    cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
2005
2006    return VSt;
2007  }
2008
2009  // Otherwise, quad registers are stored with two separate instructions,
2010  // where one stores the even registers and the other stores the odd registers.
2011
2012  // Form the QQQQ REG_SEQUENCE.
2013  SDValue V0 = N->getOperand(Vec0Idx + 0);
2014  SDValue V1 = N->getOperand(Vec0Idx + 1);
2015  SDValue V2 = N->getOperand(Vec0Idx + 2);
2016  SDValue V3 = (NumVecs == 3)
2017    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2018    : N->getOperand(Vec0Idx + 3);
2019  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2020
2021  // Store the even D registers.  This is always an updating store, so that it
2022  // provides the address to the second store for the odd subregs.
2023  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2024  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2025                                        MemAddr.getValueType(),
2026                                        MVT::Other, OpsA);
2027  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
2028  Chain = SDValue(VStA, 1);
2029
2030  // Store the odd D registers.
2031  Ops.push_back(SDValue(VStA, 0));
2032  Ops.push_back(Align);
2033  if (isUpdating) {
2034    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2035    assert(isa<ConstantSDNode>(Inc.getNode()) &&
2036           "only constant post-increment update allowed for VST3/4");
2037    (void)Inc;
2038    Ops.push_back(Reg0);
2039  }
2040  Ops.push_back(RegSeq);
2041  Ops.push_back(Pred);
2042  Ops.push_back(Reg0);
2043  Ops.push_back(Chain);
2044  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2045                                        Ops);
2046  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
2047  return VStB;
2048}
2049
2050SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
2051                                         bool isUpdating, unsigned NumVecs,
2052                                         const uint16_t *DOpcodes,
2053                                         const uint16_t *QOpcodes) {
2054  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2055  SDLoc dl(N);
2056
2057  SDValue MemAddr, Align;
2058  unsigned AddrOpIdx = isUpdating ? 1 : 2;
2059  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2060  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2061    return nullptr;
2062
2063  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2064  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2065
2066  SDValue Chain = N->getOperand(0);
2067  unsigned Lane =
2068    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2069  EVT VT = N->getOperand(Vec0Idx).getValueType();
2070  bool is64BitVector = VT.is64BitVector();
2071
2072  unsigned Alignment = 0;
2073  if (NumVecs != 3) {
2074    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2075    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2076    if (Alignment > NumBytes)
2077      Alignment = NumBytes;
2078    if (Alignment < 8 && Alignment < NumBytes)
2079      Alignment = 0;
2080    // Alignment must be a power of two; make sure of that.
2081    Alignment = (Alignment & -Alignment);
2082    if (Alignment == 1)
2083      Alignment = 0;
2084  }
2085  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2086
2087  unsigned OpcodeIndex;
2088  switch (VT.getSimpleVT().SimpleTy) {
2089  default: llvm_unreachable("unhandled vld/vst lane type");
2090    // Double-register operations:
2091  case MVT::v8i8:  OpcodeIndex = 0; break;
2092  case MVT::v4i16: OpcodeIndex = 1; break;
2093  case MVT::v2f32:
2094  case MVT::v2i32: OpcodeIndex = 2; break;
2095    // Quad-register operations:
2096  case MVT::v8i16: OpcodeIndex = 0; break;
2097  case MVT::v4f32:
2098  case MVT::v4i32: OpcodeIndex = 1; break;
2099  }
2100
2101  std::vector<EVT> ResTys;
2102  if (IsLoad) {
2103    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2104    if (!is64BitVector)
2105      ResTyElts *= 2;
2106    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2107                                      MVT::i64, ResTyElts));
2108  }
2109  if (isUpdating)
2110    ResTys.push_back(MVT::i32);
2111  ResTys.push_back(MVT::Other);
2112
2113  SDValue Pred = getAL(CurDAG);
2114  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2115
2116  SmallVector<SDValue, 8> Ops;
2117  Ops.push_back(MemAddr);
2118  Ops.push_back(Align);
2119  if (isUpdating) {
2120    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2121    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
2122  }
2123
2124  SDValue SuperReg;
2125  SDValue V0 = N->getOperand(Vec0Idx + 0);
2126  SDValue V1 = N->getOperand(Vec0Idx + 1);
2127  if (NumVecs == 2) {
2128    if (is64BitVector)
2129      SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2130    else
2131      SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2132  } else {
2133    SDValue V2 = N->getOperand(Vec0Idx + 2);
2134    SDValue V3 = (NumVecs == 3)
2135      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2136      : N->getOperand(Vec0Idx + 3);
2137    if (is64BitVector)
2138      SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2139    else
2140      SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2141  }
2142  Ops.push_back(SuperReg);
2143  Ops.push_back(getI32Imm(Lane));
2144  Ops.push_back(Pred);
2145  Ops.push_back(Reg0);
2146  Ops.push_back(Chain);
2147
2148  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2149                                  QOpcodes[OpcodeIndex]);
2150  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2151  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
2152  if (!IsLoad)
2153    return VLdLn;
2154
2155  // Extract the subregisters.
2156  SuperReg = SDValue(VLdLn, 0);
2157  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
2158         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
2159  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2160  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2161    ReplaceUses(SDValue(N, Vec),
2162                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2163  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2164  if (isUpdating)
2165    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2166  return nullptr;
2167}
2168
2169SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
2170                                      unsigned NumVecs,
2171                                      const uint16_t *Opcodes) {
2172  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2173  SDLoc dl(N);
2174
2175  SDValue MemAddr, Align;
2176  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2177    return nullptr;
2178
2179  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2180  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2181
2182  SDValue Chain = N->getOperand(0);
2183  EVT VT = N->getValueType(0);
2184
2185  unsigned Alignment = 0;
2186  if (NumVecs != 3) {
2187    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2188    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2189    if (Alignment > NumBytes)
2190      Alignment = NumBytes;
2191    if (Alignment < 8 && Alignment < NumBytes)
2192      Alignment = 0;
2193    // Alignment must be a power of two; make sure of that.
2194    Alignment = (Alignment & -Alignment);
2195    if (Alignment == 1)
2196      Alignment = 0;
2197  }
2198  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2199
2200  unsigned OpcodeIndex;
2201  switch (VT.getSimpleVT().SimpleTy) {
2202  default: llvm_unreachable("unhandled vld-dup type");
2203  case MVT::v8i8:  OpcodeIndex = 0; break;
2204  case MVT::v4i16: OpcodeIndex = 1; break;
2205  case MVT::v2f32:
2206  case MVT::v2i32: OpcodeIndex = 2; break;
2207  }
2208
2209  SDValue Pred = getAL(CurDAG);
2210  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2211  SDValue SuperReg;
2212  unsigned Opc = Opcodes[OpcodeIndex];
2213  SmallVector<SDValue, 6> Ops;
2214  Ops.push_back(MemAddr);
2215  Ops.push_back(Align);
2216  if (isUpdating) {
2217    // fixed-stride update instructions don't have an explicit writeback
2218    // operand. It's implicit in the opcode itself.
2219    SDValue Inc = N->getOperand(2);
2220    if (!isa<ConstantSDNode>(Inc.getNode()))
2221      Ops.push_back(Inc);
2222    // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2223    else if (NumVecs > 2)
2224      Ops.push_back(Reg0);
2225  }
2226  Ops.push_back(Pred);
2227  Ops.push_back(Reg0);
2228  Ops.push_back(Chain);
2229
2230  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2231  std::vector<EVT> ResTys;
2232  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2233  if (isUpdating)
2234    ResTys.push_back(MVT::i32);
2235  ResTys.push_back(MVT::Other);
2236  SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2237  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2238  SuperReg = SDValue(VLdDup, 0);
2239
2240  // Extract the subregisters.
2241  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
2242  unsigned SubIdx = ARM::dsub_0;
2243  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2244    ReplaceUses(SDValue(N, Vec),
2245                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2246  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2247  if (isUpdating)
2248    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2249  return nullptr;
2250}
2251
2252SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
2253                                    unsigned Opc) {
2254  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
2255  SDLoc dl(N);
2256  EVT VT = N->getValueType(0);
2257  unsigned FirstTblReg = IsExt ? 2 : 1;
2258
2259  // Form a REG_SEQUENCE to force register allocation.
2260  SDValue RegSeq;
2261  SDValue V0 = N->getOperand(FirstTblReg + 0);
2262  SDValue V1 = N->getOperand(FirstTblReg + 1);
2263  if (NumVecs == 2)
2264    RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0);
2265  else {
2266    SDValue V2 = N->getOperand(FirstTblReg + 2);
2267    // If it's a vtbl3, form a quad D-register and leave the last part as
2268    // an undef.
2269    SDValue V3 = (NumVecs == 3)
2270      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2271      : N->getOperand(FirstTblReg + 3);
2272    RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2273  }
2274
2275  SmallVector<SDValue, 6> Ops;
2276  if (IsExt)
2277    Ops.push_back(N->getOperand(1));
2278  Ops.push_back(RegSeq);
2279  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
2280  Ops.push_back(getAL(CurDAG)); // predicate
2281  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
2282  return CurDAG->getMachineNode(Opc, dl, VT, Ops);
2283}
2284
2285SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
2286                                                     bool isSigned) {
2287  if (!Subtarget->hasV6T2Ops())
2288    return nullptr;
2289
2290  unsigned Opc = isSigned
2291    ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2292    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2293
2294  // For unsigned extracts, check for a shift right and mask
2295  unsigned And_imm = 0;
2296  if (N->getOpcode() == ISD::AND) {
2297    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2298
2299      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2300      if (And_imm & (And_imm + 1))
2301        return nullptr;
2302
2303      unsigned Srl_imm = 0;
2304      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2305                                Srl_imm)) {
2306        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2307
2308        // Note: The width operand is encoded as width-1.
2309        unsigned Width = CountTrailingOnes_32(And_imm) - 1;
2310        unsigned LSB = Srl_imm;
2311
2312        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2313
2314        if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2315          // It's cheaper to use a right shift to extract the top bits.
2316          if (Subtarget->isThumb()) {
2317            Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2318            SDValue Ops[] = { N->getOperand(0).getOperand(0),
2319                              CurDAG->getTargetConstant(LSB, MVT::i32),
2320                              getAL(CurDAG), Reg0, Reg0 };
2321            return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2322          }
2323
2324          // ARM models shift instructions as MOVsi with shifter operand.
2325          ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL);
2326          SDValue ShOpc =
2327            CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB),
2328                                      MVT::i32);
2329          SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2330                            getAL(CurDAG), Reg0, Reg0 };
2331          return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2332        }
2333
2334        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2335                          CurDAG->getTargetConstant(LSB, MVT::i32),
2336                          CurDAG->getTargetConstant(Width, MVT::i32),
2337                          getAL(CurDAG), Reg0 };
2338        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2339      }
2340    }
2341    return nullptr;
2342  }
2343
2344  // Otherwise, we're looking for a shift of a shift
2345  unsigned Shl_imm = 0;
2346  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2347    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2348    unsigned Srl_imm = 0;
2349    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2350      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2351      // Note: The width operand is encoded as width-1.
2352      unsigned Width = 32 - Srl_imm - 1;
2353      int LSB = Srl_imm - Shl_imm;
2354      if (LSB < 0)
2355        return nullptr;
2356      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2357      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2358                        CurDAG->getTargetConstant(LSB, MVT::i32),
2359                        CurDAG->getTargetConstant(Width, MVT::i32),
2360                        getAL(CurDAG), Reg0 };
2361      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2362    }
2363  }
2364  return nullptr;
2365}
2366
2367/// Target-specific DAG combining for ISD::XOR.
2368/// Target-independent combining lowers SELECT_CC nodes of the form
2369/// select_cc setg[ge] X,  0,  X, -X
2370/// select_cc setgt    X, -1,  X, -X
2371/// select_cc setl[te] X,  0, -X,  X
2372/// select_cc setlt    X,  1, -X,  X
2373/// which represent Integer ABS into:
2374/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2375/// ARM instruction selection detects the latter and matches it to
2376/// ARM::ABS or ARM::t2ABS machine node.
2377SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
2378  SDValue XORSrc0 = N->getOperand(0);
2379  SDValue XORSrc1 = N->getOperand(1);
2380  EVT VT = N->getValueType(0);
2381
2382  if (Subtarget->isThumb1Only())
2383    return nullptr;
2384
2385  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2386    return nullptr;
2387
2388  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2389  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2390  SDValue SRASrc0 = XORSrc1.getOperand(0);
2391  SDValue SRASrc1 = XORSrc1.getOperand(1);
2392  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
2393  EVT XType = SRASrc0.getValueType();
2394  unsigned Size = XType.getSizeInBits() - 1;
2395
2396  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2397      XType.isInteger() && SRAConstant != nullptr &&
2398      Size == SRAConstant->getZExtValue()) {
2399    unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2400    return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2401  }
2402
2403  return nullptr;
2404}
2405
2406SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2407  // The only time a CONCAT_VECTORS operation can have legal types is when
2408  // two 64-bit vectors are concatenated to a 128-bit vector.
2409  EVT VT = N->getValueType(0);
2410  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2411    llvm_unreachable("unexpected CONCAT_VECTORS");
2412  return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1));
2413}
2414
2415SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2416  SDLoc dl(N);
2417
2418  if (N->isMachineOpcode()) {
2419    N->setNodeId(-1);
2420    return nullptr;   // Already selected.
2421  }
2422
2423  switch (N->getOpcode()) {
2424  default: break;
2425  case ISD::INLINEASM: {
2426    SDNode *ResNode = SelectInlineAsm(N);
2427    if (ResNode)
2428      return ResNode;
2429    break;
2430  }
2431  case ISD::XOR: {
2432    // Select special operations if XOR node forms integer ABS pattern
2433    SDNode *ResNode = SelectABSOp(N);
2434    if (ResNode)
2435      return ResNode;
2436    // Other cases are autogenerated.
2437    break;
2438  }
2439  case ISD::Constant: {
2440    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2441    bool UseCP = true;
2442    if (Subtarget->useMovt(*MF))
2443      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2444      // be done with MOV + MOVT, at worst.
2445      UseCP = false;
2446    else {
2447      if (Subtarget->isThumb()) {
2448        UseCP = (Val > 255 &&                                  // MOV
2449                 ~Val > 255 &&                                 // MOV + MVN
2450                 !ARM_AM::isThumbImmShiftedVal(Val) &&         // MOV + LSL
2451                 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW
2452      } else
2453        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&             // MOV
2454                 ARM_AM::getSOImmVal(~Val) == -1 &&            // MVN
2455                 !ARM_AM::isSOImmTwoPartVal(Val) &&            // two instrs.
2456                 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW
2457    }
2458
2459    if (UseCP) {
2460      SDValue CPIdx =
2461        CurDAG->getTargetConstantPool(ConstantInt::get(
2462                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
2463                                      getTargetLowering()->getPointerTy());
2464
2465      SDNode *ResNode;
2466      if (Subtarget->isThumb()) {
2467        SDValue Pred = getAL(CurDAG);
2468        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2469        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2470        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2471                                         Ops);
2472      } else {
2473        SDValue Ops[] = {
2474          CPIdx,
2475          CurDAG->getTargetConstant(0, MVT::i32),
2476          getAL(CurDAG),
2477          CurDAG->getRegister(0, MVT::i32),
2478          CurDAG->getEntryNode()
2479        };
2480        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2481                                       Ops);
2482      }
2483      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2484      return nullptr;
2485    }
2486
2487    // Other cases are autogenerated.
2488    break;
2489  }
2490  case ISD::FrameIndex: {
2491    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2492    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2493    SDValue TFI = CurDAG->getTargetFrameIndex(FI,
2494                                           getTargetLowering()->getPointerTy());
2495    if (Subtarget->isThumb1Only()) {
2496      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2497                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2498      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops);
2499    } else {
2500      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2501                      ARM::t2ADDri : ARM::ADDri);
2502      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2503                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2504                        CurDAG->getRegister(0, MVT::i32) };
2505      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2506    }
2507  }
2508  case ISD::SRL:
2509    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2510      return I;
2511    break;
2512  case ISD::SRA:
2513    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2514      return I;
2515    break;
2516  case ISD::MUL:
2517    if (Subtarget->isThumb1Only())
2518      break;
2519    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2520      unsigned RHSV = C->getZExtValue();
2521      if (!RHSV) break;
2522      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2523        unsigned ShImm = Log2_32(RHSV-1);
2524        if (ShImm >= 32)
2525          break;
2526        SDValue V = N->getOperand(0);
2527        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2528        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2529        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2530        if (Subtarget->isThumb()) {
2531          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2532          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2533        } else {
2534          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2535          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2536        }
2537      }
2538      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2539        unsigned ShImm = Log2_32(RHSV+1);
2540        if (ShImm >= 32)
2541          break;
2542        SDValue V = N->getOperand(0);
2543        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2544        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2545        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2546        if (Subtarget->isThumb()) {
2547          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2548          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2549        } else {
2550          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2551          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2552        }
2553      }
2554    }
2555    break;
2556  case ISD::AND: {
2557    // Check for unsigned bitfield extract
2558    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2559      return I;
2560
2561    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2562    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2563    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2564    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2565    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2566    EVT VT = N->getValueType(0);
2567    if (VT != MVT::i32)
2568      break;
2569    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2570      ? ARM::t2MOVTi16
2571      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2572    if (!Opc)
2573      break;
2574    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2575    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2576    if (!N1C)
2577      break;
2578    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2579      SDValue N2 = N0.getOperand(1);
2580      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2581      if (!N2C)
2582        break;
2583      unsigned N1CVal = N1C->getZExtValue();
2584      unsigned N2CVal = N2C->getZExtValue();
2585      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2586          (N1CVal & 0xffffU) == 0xffffU &&
2587          (N2CVal & 0xffffU) == 0x0U) {
2588        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2589                                                  MVT::i32);
2590        SDValue Ops[] = { N0.getOperand(0), Imm16,
2591                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2592        return CurDAG->getMachineNode(Opc, dl, VT, Ops);
2593      }
2594    }
2595    break;
2596  }
2597  case ARMISD::VMOVRRD:
2598    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2599                                  N->getOperand(0), getAL(CurDAG),
2600                                  CurDAG->getRegister(0, MVT::i32));
2601  case ISD::UMUL_LOHI: {
2602    if (Subtarget->isThumb1Only())
2603      break;
2604    if (Subtarget->isThumb()) {
2605      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2606                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2607      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops);
2608    } else {
2609      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2610                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2611                        CurDAG->getRegister(0, MVT::i32) };
2612      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2613                                    ARM::UMULL : ARM::UMULLv5,
2614                                    dl, MVT::i32, MVT::i32, Ops);
2615    }
2616  }
2617  case ISD::SMUL_LOHI: {
2618    if (Subtarget->isThumb1Only())
2619      break;
2620    if (Subtarget->isThumb()) {
2621      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2622                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2623      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops);
2624    } else {
2625      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2626                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2627                        CurDAG->getRegister(0, MVT::i32) };
2628      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2629                                    ARM::SMULL : ARM::SMULLv5,
2630                                    dl, MVT::i32, MVT::i32, Ops);
2631    }
2632  }
2633  case ARMISD::UMLAL:{
2634    if (Subtarget->isThumb()) {
2635      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2636                        N->getOperand(3), getAL(CurDAG),
2637                        CurDAG->getRegister(0, MVT::i32)};
2638      return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops);
2639    }else{
2640      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2641                        N->getOperand(3), getAL(CurDAG),
2642                        CurDAG->getRegister(0, MVT::i32),
2643                        CurDAG->getRegister(0, MVT::i32) };
2644      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2645                                      ARM::UMLAL : ARM::UMLALv5,
2646                                      dl, MVT::i32, MVT::i32, Ops);
2647    }
2648  }
2649  case ARMISD::SMLAL:{
2650    if (Subtarget->isThumb()) {
2651      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2652                        N->getOperand(3), getAL(CurDAG),
2653                        CurDAG->getRegister(0, MVT::i32)};
2654      return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops);
2655    }else{
2656      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2657                        N->getOperand(3), getAL(CurDAG),
2658                        CurDAG->getRegister(0, MVT::i32),
2659                        CurDAG->getRegister(0, MVT::i32) };
2660      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2661                                      ARM::SMLAL : ARM::SMLALv5,
2662                                      dl, MVT::i32, MVT::i32, Ops);
2663    }
2664  }
2665  case ISD::LOAD: {
2666    SDNode *ResNode = nullptr;
2667    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2668      ResNode = SelectT2IndexedLoad(N);
2669    else
2670      ResNode = SelectARMIndexedLoad(N);
2671    if (ResNode)
2672      return ResNode;
2673    // Other cases are autogenerated.
2674    break;
2675  }
2676  case ARMISD::BRCOND: {
2677    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2678    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2679    // Pattern complexity = 6  cost = 1  size = 0
2680
2681    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2682    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2683    // Pattern complexity = 6  cost = 1  size = 0
2684
2685    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2686    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2687    // Pattern complexity = 6  cost = 1  size = 0
2688
2689    unsigned Opc = Subtarget->isThumb() ?
2690      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2691    SDValue Chain = N->getOperand(0);
2692    SDValue N1 = N->getOperand(1);
2693    SDValue N2 = N->getOperand(2);
2694    SDValue N3 = N->getOperand(3);
2695    SDValue InFlag = N->getOperand(4);
2696    assert(N1.getOpcode() == ISD::BasicBlock);
2697    assert(N2.getOpcode() == ISD::Constant);
2698    assert(N3.getOpcode() == ISD::Register);
2699
2700    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2701                               cast<ConstantSDNode>(N2)->getZExtValue()),
2702                               MVT::i32);
2703    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2704    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2705                                             MVT::Glue, Ops);
2706    Chain = SDValue(ResNode, 0);
2707    if (N->getNumValues() == 2) {
2708      InFlag = SDValue(ResNode, 1);
2709      ReplaceUses(SDValue(N, 1), InFlag);
2710    }
2711    ReplaceUses(SDValue(N, 0),
2712                SDValue(Chain.getNode(), Chain.getResNo()));
2713    return nullptr;
2714  }
2715  case ARMISD::VZIP: {
2716    unsigned Opc = 0;
2717    EVT VT = N->getValueType(0);
2718    switch (VT.getSimpleVT().SimpleTy) {
2719    default: return nullptr;
2720    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2721    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2722    case MVT::v2f32:
2723    // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2724    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2725    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2726    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2727    case MVT::v4f32:
2728    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2729    }
2730    SDValue Pred = getAL(CurDAG);
2731    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2732    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2733    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2734  }
2735  case ARMISD::VUZP: {
2736    unsigned Opc = 0;
2737    EVT VT = N->getValueType(0);
2738    switch (VT.getSimpleVT().SimpleTy) {
2739    default: return nullptr;
2740    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2741    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2742    case MVT::v2f32:
2743    // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2744    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2745    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2746    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2747    case MVT::v4f32:
2748    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2749    }
2750    SDValue Pred = getAL(CurDAG);
2751    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2752    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2753    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2754  }
2755  case ARMISD::VTRN: {
2756    unsigned Opc = 0;
2757    EVT VT = N->getValueType(0);
2758    switch (VT.getSimpleVT().SimpleTy) {
2759    default: return nullptr;
2760    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2761    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2762    case MVT::v2f32:
2763    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2764    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2765    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2766    case MVT::v4f32:
2767    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2768    }
2769    SDValue Pred = getAL(CurDAG);
2770    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2771    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2772    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2773  }
2774  case ARMISD::BUILD_VECTOR: {
2775    EVT VecVT = N->getValueType(0);
2776    EVT EltVT = VecVT.getVectorElementType();
2777    unsigned NumElts = VecVT.getVectorNumElements();
2778    if (EltVT == MVT::f64) {
2779      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2780      return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1));
2781    }
2782    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2783    if (NumElts == 2)
2784      return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1));
2785    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2786    return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
2787                     N->getOperand(2), N->getOperand(3));
2788  }
2789
2790  case ARMISD::VLD2DUP: {
2791    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
2792                                        ARM::VLD2DUPd32 };
2793    return SelectVLDDup(N, false, 2, Opcodes);
2794  }
2795
2796  case ARMISD::VLD3DUP: {
2797    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
2798                                        ARM::VLD3DUPd16Pseudo,
2799                                        ARM::VLD3DUPd32Pseudo };
2800    return SelectVLDDup(N, false, 3, Opcodes);
2801  }
2802
2803  case ARMISD::VLD4DUP: {
2804    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
2805                                        ARM::VLD4DUPd16Pseudo,
2806                                        ARM::VLD4DUPd32Pseudo };
2807    return SelectVLDDup(N, false, 4, Opcodes);
2808  }
2809
2810  case ARMISD::VLD2DUP_UPD: {
2811    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
2812                                        ARM::VLD2DUPd16wb_fixed,
2813                                        ARM::VLD2DUPd32wb_fixed };
2814    return SelectVLDDup(N, true, 2, Opcodes);
2815  }
2816
2817  case ARMISD::VLD3DUP_UPD: {
2818    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
2819                                        ARM::VLD3DUPd16Pseudo_UPD,
2820                                        ARM::VLD3DUPd32Pseudo_UPD };
2821    return SelectVLDDup(N, true, 3, Opcodes);
2822  }
2823
2824  case ARMISD::VLD4DUP_UPD: {
2825    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
2826                                        ARM::VLD4DUPd16Pseudo_UPD,
2827                                        ARM::VLD4DUPd32Pseudo_UPD };
2828    return SelectVLDDup(N, true, 4, Opcodes);
2829  }
2830
2831  case ARMISD::VLD1_UPD: {
2832    static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
2833                                         ARM::VLD1d16wb_fixed,
2834                                         ARM::VLD1d32wb_fixed,
2835                                         ARM::VLD1d64wb_fixed };
2836    static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
2837                                         ARM::VLD1q16wb_fixed,
2838                                         ARM::VLD1q32wb_fixed,
2839                                         ARM::VLD1q64wb_fixed };
2840    return SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
2841  }
2842
2843  case ARMISD::VLD2_UPD: {
2844    static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
2845                                         ARM::VLD2d16wb_fixed,
2846                                         ARM::VLD2d32wb_fixed,
2847                                         ARM::VLD1q64wb_fixed};
2848    static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
2849                                         ARM::VLD2q16PseudoWB_fixed,
2850                                         ARM::VLD2q32PseudoWB_fixed };
2851    return SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
2852  }
2853
2854  case ARMISD::VLD3_UPD: {
2855    static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
2856                                         ARM::VLD3d16Pseudo_UPD,
2857                                         ARM::VLD3d32Pseudo_UPD,
2858                                         ARM::VLD1d64TPseudoWB_fixed};
2859    static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2860                                          ARM::VLD3q16Pseudo_UPD,
2861                                          ARM::VLD3q32Pseudo_UPD };
2862    static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2863                                          ARM::VLD3q16oddPseudo_UPD,
2864                                          ARM::VLD3q32oddPseudo_UPD };
2865    return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2866  }
2867
2868  case ARMISD::VLD4_UPD: {
2869    static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
2870                                         ARM::VLD4d16Pseudo_UPD,
2871                                         ARM::VLD4d32Pseudo_UPD,
2872                                         ARM::VLD1d64QPseudoWB_fixed};
2873    static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2874                                          ARM::VLD4q16Pseudo_UPD,
2875                                          ARM::VLD4q32Pseudo_UPD };
2876    static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2877                                          ARM::VLD4q16oddPseudo_UPD,
2878                                          ARM::VLD4q32oddPseudo_UPD };
2879    return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2880  }
2881
2882  case ARMISD::VLD2LN_UPD: {
2883    static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
2884                                         ARM::VLD2LNd16Pseudo_UPD,
2885                                         ARM::VLD2LNd32Pseudo_UPD };
2886    static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2887                                         ARM::VLD2LNq32Pseudo_UPD };
2888    return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2889  }
2890
2891  case ARMISD::VLD3LN_UPD: {
2892    static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
2893                                         ARM::VLD3LNd16Pseudo_UPD,
2894                                         ARM::VLD3LNd32Pseudo_UPD };
2895    static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2896                                         ARM::VLD3LNq32Pseudo_UPD };
2897    return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2898  }
2899
2900  case ARMISD::VLD4LN_UPD: {
2901    static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
2902                                         ARM::VLD4LNd16Pseudo_UPD,
2903                                         ARM::VLD4LNd32Pseudo_UPD };
2904    static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
2905                                         ARM::VLD4LNq32Pseudo_UPD };
2906    return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
2907  }
2908
2909  case ARMISD::VST1_UPD: {
2910    static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
2911                                         ARM::VST1d16wb_fixed,
2912                                         ARM::VST1d32wb_fixed,
2913                                         ARM::VST1d64wb_fixed };
2914    static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
2915                                         ARM::VST1q16wb_fixed,
2916                                         ARM::VST1q32wb_fixed,
2917                                         ARM::VST1q64wb_fixed };
2918    return SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
2919  }
2920
2921  case ARMISD::VST2_UPD: {
2922    static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
2923                                         ARM::VST2d16wb_fixed,
2924                                         ARM::VST2d32wb_fixed,
2925                                         ARM::VST1q64wb_fixed};
2926    static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
2927                                         ARM::VST2q16PseudoWB_fixed,
2928                                         ARM::VST2q32PseudoWB_fixed };
2929    return SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
2930  }
2931
2932  case ARMISD::VST3_UPD: {
2933    static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
2934                                         ARM::VST3d16Pseudo_UPD,
2935                                         ARM::VST3d32Pseudo_UPD,
2936                                         ARM::VST1d64TPseudoWB_fixed};
2937    static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2938                                          ARM::VST3q16Pseudo_UPD,
2939                                          ARM::VST3q32Pseudo_UPD };
2940    static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2941                                          ARM::VST3q16oddPseudo_UPD,
2942                                          ARM::VST3q32oddPseudo_UPD };
2943    return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2944  }
2945
2946  case ARMISD::VST4_UPD: {
2947    static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
2948                                         ARM::VST4d16Pseudo_UPD,
2949                                         ARM::VST4d32Pseudo_UPD,
2950                                         ARM::VST1d64QPseudoWB_fixed};
2951    static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2952                                          ARM::VST4q16Pseudo_UPD,
2953                                          ARM::VST4q32Pseudo_UPD };
2954    static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2955                                          ARM::VST4q16oddPseudo_UPD,
2956                                          ARM::VST4q32oddPseudo_UPD };
2957    return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2958  }
2959
2960  case ARMISD::VST2LN_UPD: {
2961    static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
2962                                         ARM::VST2LNd16Pseudo_UPD,
2963                                         ARM::VST2LNd32Pseudo_UPD };
2964    static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
2965                                         ARM::VST2LNq32Pseudo_UPD };
2966    return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
2967  }
2968
2969  case ARMISD::VST3LN_UPD: {
2970    static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
2971                                         ARM::VST3LNd16Pseudo_UPD,
2972                                         ARM::VST3LNd32Pseudo_UPD };
2973    static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
2974                                         ARM::VST3LNq32Pseudo_UPD };
2975    return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
2976  }
2977
2978  case ARMISD::VST4LN_UPD: {
2979    static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
2980                                         ARM::VST4LNd16Pseudo_UPD,
2981                                         ARM::VST4LNd32Pseudo_UPD };
2982    static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
2983                                         ARM::VST4LNq32Pseudo_UPD };
2984    return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
2985  }
2986
2987  case ISD::INTRINSIC_VOID:
2988  case ISD::INTRINSIC_W_CHAIN: {
2989    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2990    switch (IntNo) {
2991    default:
2992      break;
2993
2994    case Intrinsic::arm_ldaexd:
2995    case Intrinsic::arm_ldrexd: {
2996      SDLoc dl(N);
2997      SDValue Chain = N->getOperand(0);
2998      SDValue MemAddr = N->getOperand(2);
2999      bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3000
3001      bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3002      unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
3003                                : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
3004
3005      // arm_ldrexd returns a i64 value in {i32, i32}
3006      std::vector<EVT> ResTys;
3007      if (isThumb) {
3008        ResTys.push_back(MVT::i32);
3009        ResTys.push_back(MVT::i32);
3010      } else
3011        ResTys.push_back(MVT::Untyped);
3012      ResTys.push_back(MVT::Other);
3013
3014      // Place arguments in the right order.
3015      SmallVector<SDValue, 7> Ops;
3016      Ops.push_back(MemAddr);
3017      Ops.push_back(getAL(CurDAG));
3018      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3019      Ops.push_back(Chain);
3020      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3021      // Transfer memoperands.
3022      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3023      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3024      cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3025
3026      // Remap uses.
3027      SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3028      if (!SDValue(N, 0).use_empty()) {
3029        SDValue Result;
3030        if (isThumb)
3031          Result = SDValue(Ld, 0);
3032        else {
3033          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
3034          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3035              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3036          Result = SDValue(ResNode,0);
3037        }
3038        ReplaceUses(SDValue(N, 0), Result);
3039      }
3040      if (!SDValue(N, 1).use_empty()) {
3041        SDValue Result;
3042        if (isThumb)
3043          Result = SDValue(Ld, 1);
3044        else {
3045          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
3046          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3047              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3048          Result = SDValue(ResNode,0);
3049        }
3050        ReplaceUses(SDValue(N, 1), Result);
3051      }
3052      ReplaceUses(SDValue(N, 2), OutChain);
3053      return nullptr;
3054    }
3055    case Intrinsic::arm_stlexd:
3056    case Intrinsic::arm_strexd: {
3057      SDLoc dl(N);
3058      SDValue Chain = N->getOperand(0);
3059      SDValue Val0 = N->getOperand(2);
3060      SDValue Val1 = N->getOperand(3);
3061      SDValue MemAddr = N->getOperand(4);
3062
3063      // Store exclusive double return a i32 value which is the return status
3064      // of the issued store.
3065      EVT ResTys[] = { MVT::i32, MVT::Other };
3066
3067      bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3068      // Place arguments in the right order.
3069      SmallVector<SDValue, 7> Ops;
3070      if (isThumb) {
3071        Ops.push_back(Val0);
3072        Ops.push_back(Val1);
3073      } else
3074        // arm_strexd uses GPRPair.
3075        Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3076      Ops.push_back(MemAddr);
3077      Ops.push_back(getAL(CurDAG));
3078      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3079      Ops.push_back(Chain);
3080
3081      bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3082      unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
3083                                : (IsRelease ? ARM::STLEXD : ARM::STREXD);
3084
3085      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3086      // Transfer memoperands.
3087      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3088      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3089      cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3090
3091      return St;
3092    }
3093
3094    case Intrinsic::arm_neon_vld1: {
3095      static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3096                                           ARM::VLD1d32, ARM::VLD1d64 };
3097      static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3098                                           ARM::VLD1q32, ARM::VLD1q64};
3099      return SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3100    }
3101
3102    case Intrinsic::arm_neon_vld2: {
3103      static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3104                                           ARM::VLD2d32, ARM::VLD1q64 };
3105      static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3106                                           ARM::VLD2q32Pseudo };
3107      return SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3108    }
3109
3110    case Intrinsic::arm_neon_vld3: {
3111      static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3112                                           ARM::VLD3d16Pseudo,
3113                                           ARM::VLD3d32Pseudo,
3114                                           ARM::VLD1d64TPseudo };
3115      static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3116                                            ARM::VLD3q16Pseudo_UPD,
3117                                            ARM::VLD3q32Pseudo_UPD };
3118      static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3119                                            ARM::VLD3q16oddPseudo,
3120                                            ARM::VLD3q32oddPseudo };
3121      return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3122    }
3123
3124    case Intrinsic::arm_neon_vld4: {
3125      static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3126                                           ARM::VLD4d16Pseudo,
3127                                           ARM::VLD4d32Pseudo,
3128                                           ARM::VLD1d64QPseudo };
3129      static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3130                                            ARM::VLD4q16Pseudo_UPD,
3131                                            ARM::VLD4q32Pseudo_UPD };
3132      static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3133                                            ARM::VLD4q16oddPseudo,
3134                                            ARM::VLD4q32oddPseudo };
3135      return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3136    }
3137
3138    case Intrinsic::arm_neon_vld2lane: {
3139      static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3140                                           ARM::VLD2LNd16Pseudo,
3141                                           ARM::VLD2LNd32Pseudo };
3142      static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3143                                           ARM::VLD2LNq32Pseudo };
3144      return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3145    }
3146
3147    case Intrinsic::arm_neon_vld3lane: {
3148      static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3149                                           ARM::VLD3LNd16Pseudo,
3150                                           ARM::VLD3LNd32Pseudo };
3151      static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3152                                           ARM::VLD3LNq32Pseudo };
3153      return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3154    }
3155
3156    case Intrinsic::arm_neon_vld4lane: {
3157      static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3158                                           ARM::VLD4LNd16Pseudo,
3159                                           ARM::VLD4LNd32Pseudo };
3160      static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3161                                           ARM::VLD4LNq32Pseudo };
3162      return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3163    }
3164
3165    case Intrinsic::arm_neon_vst1: {
3166      static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3167                                           ARM::VST1d32, ARM::VST1d64 };
3168      static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3169                                           ARM::VST1q32, ARM::VST1q64 };
3170      return SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3171    }
3172
3173    case Intrinsic::arm_neon_vst2: {
3174      static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3175                                           ARM::VST2d32, ARM::VST1q64 };
3176      static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3177                                     ARM::VST2q32Pseudo };
3178      return SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3179    }
3180
3181    case Intrinsic::arm_neon_vst3: {
3182      static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3183                                           ARM::VST3d16Pseudo,
3184                                           ARM::VST3d32Pseudo,
3185                                           ARM::VST1d64TPseudo };
3186      static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3187                                            ARM::VST3q16Pseudo_UPD,
3188                                            ARM::VST3q32Pseudo_UPD };
3189      static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3190                                            ARM::VST3q16oddPseudo,
3191                                            ARM::VST3q32oddPseudo };
3192      return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3193    }
3194
3195    case Intrinsic::arm_neon_vst4: {
3196      static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3197                                           ARM::VST4d16Pseudo,
3198                                           ARM::VST4d32Pseudo,
3199                                           ARM::VST1d64QPseudo };
3200      static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3201                                            ARM::VST4q16Pseudo_UPD,
3202                                            ARM::VST4q32Pseudo_UPD };
3203      static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3204                                            ARM::VST4q16oddPseudo,
3205                                            ARM::VST4q32oddPseudo };
3206      return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3207    }
3208
3209    case Intrinsic::arm_neon_vst2lane: {
3210      static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3211                                           ARM::VST2LNd16Pseudo,
3212                                           ARM::VST2LNd32Pseudo };
3213      static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3214                                           ARM::VST2LNq32Pseudo };
3215      return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3216    }
3217
3218    case Intrinsic::arm_neon_vst3lane: {
3219      static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3220                                           ARM::VST3LNd16Pseudo,
3221                                           ARM::VST3LNd32Pseudo };
3222      static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3223                                           ARM::VST3LNq32Pseudo };
3224      return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3225    }
3226
3227    case Intrinsic::arm_neon_vst4lane: {
3228      static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3229                                           ARM::VST4LNd16Pseudo,
3230                                           ARM::VST4LNd32Pseudo };
3231      static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3232                                           ARM::VST4LNq32Pseudo };
3233      return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3234    }
3235    }
3236    break;
3237  }
3238
3239  case ISD::INTRINSIC_WO_CHAIN: {
3240    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3241    switch (IntNo) {
3242    default:
3243      break;
3244
3245    case Intrinsic::arm_neon_vtbl2:
3246      return SelectVTBL(N, false, 2, ARM::VTBL2);
3247    case Intrinsic::arm_neon_vtbl3:
3248      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
3249    case Intrinsic::arm_neon_vtbl4:
3250      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
3251
3252    case Intrinsic::arm_neon_vtbx2:
3253      return SelectVTBL(N, true, 2, ARM::VTBX2);
3254    case Intrinsic::arm_neon_vtbx3:
3255      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
3256    case Intrinsic::arm_neon_vtbx4:
3257      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
3258    }
3259    break;
3260  }
3261
3262  case ARMISD::VTBL1: {
3263    SDLoc dl(N);
3264    EVT VT = N->getValueType(0);
3265    SmallVector<SDValue, 6> Ops;
3266
3267    Ops.push_back(N->getOperand(0));
3268    Ops.push_back(N->getOperand(1));
3269    Ops.push_back(getAL(CurDAG));                    // Predicate
3270    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3271    return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops);
3272  }
3273  case ARMISD::VTBL2: {
3274    SDLoc dl(N);
3275    EVT VT = N->getValueType(0);
3276
3277    // Form a REG_SEQUENCE to force register allocation.
3278    SDValue V0 = N->getOperand(0);
3279    SDValue V1 = N->getOperand(1);
3280    SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0);
3281
3282    SmallVector<SDValue, 6> Ops;
3283    Ops.push_back(RegSeq);
3284    Ops.push_back(N->getOperand(2));
3285    Ops.push_back(getAL(CurDAG));                    // Predicate
3286    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3287    return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops);
3288  }
3289
3290  case ISD::CONCAT_VECTORS:
3291    return SelectConcatVector(N);
3292  }
3293
3294  return SelectCode(N);
3295}
3296
3297SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){
3298  std::vector<SDValue> AsmNodeOperands;
3299  unsigned Flag, Kind;
3300  bool Changed = false;
3301  unsigned NumOps = N->getNumOperands();
3302
3303  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
3304  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
3305  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
3306  // respectively. Since there is no constraint to explicitly specify a
3307  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
3308  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
3309  // them into a GPRPair.
3310
3311  SDLoc dl(N);
3312  SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
3313                                   : SDValue(nullptr,0);
3314
3315  SmallVector<bool, 8> OpChanged;
3316  // Glue node will be appended late.
3317  for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
3318    SDValue op = N->getOperand(i);
3319    AsmNodeOperands.push_back(op);
3320
3321    if (i < InlineAsm::Op_FirstOperand)
3322      continue;
3323
3324    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
3325      Flag = C->getZExtValue();
3326      Kind = InlineAsm::getKind(Flag);
3327    }
3328    else
3329      continue;
3330
3331    // Immediate operands to inline asm in the SelectionDAG are modeled with
3332    // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
3333    // the second is a constant with the value of the immediate. If we get here
3334    // and we have a Kind_Imm, skip the next operand, and continue.
3335    if (Kind == InlineAsm::Kind_Imm) {
3336      SDValue op = N->getOperand(++i);
3337      AsmNodeOperands.push_back(op);
3338      continue;
3339    }
3340
3341    unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
3342    if (NumRegs)
3343      OpChanged.push_back(false);
3344
3345    unsigned DefIdx = 0;
3346    bool IsTiedToChangedOp = false;
3347    // If it's a use that is tied with a previous def, it has no
3348    // reg class constraint.
3349    if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
3350      IsTiedToChangedOp = OpChanged[DefIdx];
3351
3352    if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
3353        && Kind != InlineAsm::Kind_RegDefEarlyClobber)
3354      continue;
3355
3356    unsigned RC;
3357    bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
3358    if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
3359        || NumRegs != 2)
3360      continue;
3361
3362    assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
3363    SDValue V0 = N->getOperand(i+1);
3364    SDValue V1 = N->getOperand(i+2);
3365    unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
3366    unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
3367    SDValue PairedReg;
3368    MachineRegisterInfo &MRI = MF->getRegInfo();
3369
3370    if (Kind == InlineAsm::Kind_RegDef ||
3371        Kind == InlineAsm::Kind_RegDefEarlyClobber) {
3372      // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
3373      // the original GPRs.
3374
3375      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
3376      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
3377      SDValue Chain = SDValue(N,0);
3378
3379      SDNode *GU = N->getGluedUser();
3380      SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
3381                                               Chain.getValue(1));
3382
3383      // Extract values from a GPRPair reg and copy to the original GPR reg.
3384      SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
3385                                                    RegCopy);
3386      SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
3387                                                    RegCopy);
3388      SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
3389                                        RegCopy.getValue(1));
3390      SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
3391
3392      // Update the original glue user.
3393      std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
3394      Ops.push_back(T1.getValue(1));
3395      CurDAG->UpdateNodeOperands(GU, Ops);
3396      GU = T1.getNode();
3397    }
3398    else {
3399      // For Kind  == InlineAsm::Kind_RegUse, we first copy two GPRs into a
3400      // GPRPair and then pass the GPRPair to the inline asm.
3401      SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
3402
3403      // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
3404      SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
3405                                          Chain.getValue(1));
3406      SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
3407                                          T0.getValue(1));
3408      SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
3409
3410      // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
3411      // i32 VRs of inline asm with it.
3412      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
3413      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
3414      Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
3415
3416      AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
3417      Glue = Chain.getValue(1);
3418    }
3419
3420    Changed = true;
3421
3422    if(PairedReg.getNode()) {
3423      OpChanged[OpChanged.size() -1 ] = true;
3424      Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
3425      if (IsTiedToChangedOp)
3426        Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
3427      else
3428        Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
3429      // Replace the current flag.
3430      AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
3431          Flag, MVT::i32);
3432      // Add the new register node and skip the original two GPRs.
3433      AsmNodeOperands.push_back(PairedReg);
3434      // Skip the next two GPRs.
3435      i += 2;
3436    }
3437  }
3438
3439  if (Glue.getNode())
3440    AsmNodeOperands.push_back(Glue);
3441  if (!Changed)
3442    return nullptr;
3443
3444  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
3445      CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
3446  New->setNodeId(-1);
3447  return New.getNode();
3448}
3449
3450
3451bool ARMDAGToDAGISel::
3452SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
3453                             std::vector<SDValue> &OutOps) {
3454  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
3455  // Require the address to be in a register.  That is safe for all ARM
3456  // variants and it is hard to do anything much smarter without knowing
3457  // how the operand is used.
3458  OutOps.push_back(Op);
3459  return false;
3460}
3461
3462/// createARMISelDag - This pass converts a legalized DAG into a
3463/// ARM-specific DAG, ready for instruction scheduling.
3464///
3465FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3466                                     CodeGenOpt::Level OptLevel) {
3467  return new ARMDAGToDAGISel(TM, OptLevel);
3468}
3469