1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMTargetMachine.h"
17#include "MCTargetDesc/ARMAddressingModes.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/SelectionDAG.h"
23#include "llvm/CodeGen/SelectionDAGISel.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Target/TargetLowering.h"
35#include "llvm/Target/TargetOptions.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE "arm-isel"
40
41static cl::opt<bool>
42DisableShifterOp("disable-shifter-op", cl::Hidden,
43  cl::desc("Disable isel of shifter-op"),
44  cl::init(false));
45
46static cl::opt<bool>
47CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
48  cl::desc("Check fp vmla / vmls hazard at isel time"),
49  cl::init(true));
50
51//===--------------------------------------------------------------------===//
52/// ARMDAGToDAGISel - ARM specific code to select ARM machine
53/// instructions for SelectionDAG operations.
54///
55namespace {
56
57enum AddrMode2Type {
58  AM2_BASE, // Simple AM2 (+-imm12)
59  AM2_SHOP  // Shifter-op AM2
60};
61
62class ARMDAGToDAGISel : public SelectionDAGISel {
63  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
64  /// make the right decision when generating code for different targets.
65  const ARMSubtarget *Subtarget;
66
67public:
68  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
69      : SelectionDAGISel(tm, OptLevel) {}
70
71  bool runOnMachineFunction(MachineFunction &MF) override {
72    // Reset the subtarget each time through.
73    Subtarget = &MF.getSubtarget<ARMSubtarget>();
74    SelectionDAGISel::runOnMachineFunction(MF);
75    return true;
76  }
77
78  const char *getPassName() const override {
79    return "ARM Instruction Selection";
80  }
81
82  void PreprocessISelDAG() override;
83
84  /// getI32Imm - Return a target constant of type i32 with the specified
85  /// value.
86  inline SDValue getI32Imm(unsigned Imm) {
87    return CurDAG->getTargetConstant(Imm, MVT::i32);
88  }
89
90  SDNode *Select(SDNode *N) override;
91
92
93  bool hasNoVMLxHazardUse(SDNode *N) const;
94  bool isShifterOpProfitable(const SDValue &Shift,
95                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
96  bool SelectRegShifterOperand(SDValue N, SDValue &A,
97                               SDValue &B, SDValue &C,
98                               bool CheckProfitability = true);
99  bool SelectImmShifterOperand(SDValue N, SDValue &A,
100                               SDValue &B, bool CheckProfitability = true);
101  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
102                                    SDValue &B, SDValue &C) {
103    // Don't apply the profitability check
104    return SelectRegShifterOperand(N, A, B, C, false);
105  }
106  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
107                                    SDValue &B) {
108    // Don't apply the profitability check
109    return SelectImmShifterOperand(N, A, B, false);
110  }
111
112  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
113  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
114
115  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
116                                      SDValue &Offset, SDValue &Opc);
117  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
118                           SDValue &Opc) {
119    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
120  }
121
122  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
123                           SDValue &Opc) {
124    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
125  }
126
127  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
128                       SDValue &Opc) {
129    SelectAddrMode2Worker(N, Base, Offset, Opc);
130//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
131    // This always matches one way or another.
132    return true;
133  }
134
135  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
136    const ConstantSDNode *CN = cast<ConstantSDNode>(N);
137    Pred = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
138    Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
139    return true;
140  }
141
142  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
143                             SDValue &Offset, SDValue &Opc);
144  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
145                             SDValue &Offset, SDValue &Opc);
146  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
147                             SDValue &Offset, SDValue &Opc);
148  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
149  bool SelectAddrMode3(SDValue N, SDValue &Base,
150                       SDValue &Offset, SDValue &Opc);
151  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
152                             SDValue &Offset, SDValue &Opc);
153  bool SelectAddrMode5(SDValue N, SDValue &Base,
154                       SDValue &Offset);
155  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
156  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
157
158  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
159
160  // Thumb Addressing Modes:
161  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
162  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
163                             unsigned Scale);
164  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
165  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
166  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
167  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
168                                SDValue &OffImm);
169  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
170                                 SDValue &OffImm);
171  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
172                                 SDValue &OffImm);
173  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
174                                 SDValue &OffImm);
175  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
176
177  // Thumb 2 Addressing Modes:
178  bool SelectT2ShifterOperandReg(SDValue N,
179                                 SDValue &BaseReg, SDValue &Opc);
180  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
181  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
182                            SDValue &OffImm);
183  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
184                                 SDValue &OffImm);
185  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
186                             SDValue &OffReg, SDValue &ShImm);
187  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
188
189  inline bool is_so_imm(unsigned Imm) const {
190    return ARM_AM::getSOImmVal(Imm) != -1;
191  }
192
193  inline bool is_so_imm_not(unsigned Imm) const {
194    return ARM_AM::getSOImmVal(~Imm) != -1;
195  }
196
197  inline bool is_t2_so_imm(unsigned Imm) const {
198    return ARM_AM::getT2SOImmVal(Imm) != -1;
199  }
200
201  inline bool is_t2_so_imm_not(unsigned Imm) const {
202    return ARM_AM::getT2SOImmVal(~Imm) != -1;
203  }
204
205  // Include the pieces autogenerated from the target description.
206#include "ARMGenDAGISel.inc"
207
208private:
209  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
210  /// ARM.
211  SDNode *SelectARMIndexedLoad(SDNode *N);
212  SDNode *SelectT2IndexedLoad(SDNode *N);
213
214  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
215  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
216  /// loads of D registers and even subregs and odd subregs of Q registers.
217  /// For NumVecs <= 2, QOpcodes1 is not used.
218  SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
219                    const uint16_t *DOpcodes,
220                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
221
222  /// SelectVST - Select NEON store intrinsics.  NumVecs should
223  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
224  /// stores of D registers and even subregs and odd subregs of Q registers.
225  /// For NumVecs <= 2, QOpcodes1 is not used.
226  SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
227                    const uint16_t *DOpcodes,
228                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
229
230  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
231  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
232  /// load/store of D registers and Q registers.
233  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
234                          bool isUpdating, unsigned NumVecs,
235                          const uint16_t *DOpcodes, const uint16_t *QOpcodes);
236
237  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
238  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
239  /// for loading D registers.  (Q registers are not supported.)
240  SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
241                       const uint16_t *Opcodes);
242
243  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
244  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
245  /// generated to force the table registers to be consecutive.
246  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
247
248  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
249  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
250
251  // Select special operations if node forms integer ABS pattern
252  SDNode *SelectABSOp(SDNode *N);
253
254  SDNode *SelectInlineAsm(SDNode *N);
255
256  SDNode *SelectConcatVector(SDNode *N);
257
258  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
259  /// inline asm expressions.
260  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
261                                    std::vector<SDValue> &OutOps) override;
262
263  // Form pairs of consecutive R, S, D, or Q registers.
264  SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
265  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
266  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
267  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
268
269  // Form sequences of 4 consecutive S, D, or Q registers.
270  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
271  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
272  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
273
274  // Get the alignment operand for a NEON VLD or VST instruction.
275  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
276};
277}
278
279/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
280/// operand. If so Imm will receive the 32-bit value.
281static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
282  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
283    Imm = cast<ConstantSDNode>(N)->getZExtValue();
284    return true;
285  }
286  return false;
287}
288
289// isInt32Immediate - This method tests to see if a constant operand.
290// If so Imm will receive the 32 bit value.
291static bool isInt32Immediate(SDValue N, unsigned &Imm) {
292  return isInt32Immediate(N.getNode(), Imm);
293}
294
295// isOpcWithIntImmediate - This method tests to see if the node is a specific
296// opcode and that it has a immediate integer right operand.
297// If so Imm will receive the 32 bit value.
298static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
299  return N->getOpcode() == Opc &&
300         isInt32Immediate(N->getOperand(1).getNode(), Imm);
301}
302
303/// \brief Check whether a particular node is a constant value representable as
304/// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
305///
306/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
307static bool isScaledConstantInRange(SDValue Node, int Scale,
308                                    int RangeMin, int RangeMax,
309                                    int &ScaledConstant) {
310  assert(Scale > 0 && "Invalid scale!");
311
312  // Check that this is a constant.
313  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
314  if (!C)
315    return false;
316
317  ScaledConstant = (int) C->getZExtValue();
318  if ((ScaledConstant % Scale) != 0)
319    return false;
320
321  ScaledConstant /= Scale;
322  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
323}
324
325void ARMDAGToDAGISel::PreprocessISelDAG() {
326  if (!Subtarget->hasV6T2Ops())
327    return;
328
329  bool isThumb2 = Subtarget->isThumb();
330  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
331       E = CurDAG->allnodes_end(); I != E; ) {
332    SDNode *N = I++;  // Preincrement iterator to avoid invalidation issues.
333
334    if (N->getOpcode() != ISD::ADD)
335      continue;
336
337    // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
338    // leading zeros, followed by consecutive set bits, followed by 1 or 2
339    // trailing zeros, e.g. 1020.
340    // Transform the expression to
341    // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
342    // of trailing zeros of c2. The left shift would be folded as an shifter
343    // operand of 'add' and the 'and' and 'srl' would become a bits extraction
344    // node (UBFX).
345
346    SDValue N0 = N->getOperand(0);
347    SDValue N1 = N->getOperand(1);
348    unsigned And_imm = 0;
349    if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
350      if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
351        std::swap(N0, N1);
352    }
353    if (!And_imm)
354      continue;
355
356    // Check if the AND mask is an immediate of the form: 000.....1111111100
357    unsigned TZ = countTrailingZeros(And_imm);
358    if (TZ != 1 && TZ != 2)
359      // Be conservative here. Shifter operands aren't always free. e.g. On
360      // Swift, left shifter operand of 1 / 2 for free but others are not.
361      // e.g.
362      //  ubfx   r3, r1, #16, #8
363      //  ldr.w  r3, [r0, r3, lsl #2]
364      // vs.
365      //  mov.w  r9, #1020
366      //  and.w  r2, r9, r1, lsr #14
367      //  ldr    r2, [r0, r2]
368      continue;
369    And_imm >>= TZ;
370    if (And_imm & (And_imm + 1))
371      continue;
372
373    // Look for (and (srl X, c1), c2).
374    SDValue Srl = N1.getOperand(0);
375    unsigned Srl_imm = 0;
376    if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
377        (Srl_imm <= 2))
378      continue;
379
380    // Make sure first operand is not a shifter operand which would prevent
381    // folding of the left shift.
382    SDValue CPTmp0;
383    SDValue CPTmp1;
384    SDValue CPTmp2;
385    if (isThumb2) {
386      if (SelectT2ShifterOperandReg(N0, CPTmp0, CPTmp1))
387        continue;
388    } else {
389      if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
390          SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
391        continue;
392    }
393
394    // Now make the transformation.
395    Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
396                          Srl.getOperand(0),
397                          CurDAG->getConstant(Srl_imm+TZ, MVT::i32));
398    N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
399                         Srl, CurDAG->getConstant(And_imm, MVT::i32));
400    N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
401                         N1, CurDAG->getConstant(TZ, MVT::i32));
402    CurDAG->UpdateNodeOperands(N, N0, N1);
403  }
404}
405
406/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
407/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
408/// least on current ARM implementations) which should be avoidded.
409bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
410  if (OptLevel == CodeGenOpt::None)
411    return true;
412
413  if (!CheckVMLxHazard)
414    return true;
415
416  if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() &&
417      !Subtarget->isCortexA9() && !Subtarget->isSwift())
418    return true;
419
420  if (!N->hasOneUse())
421    return false;
422
423  SDNode *Use = *N->use_begin();
424  if (Use->getOpcode() == ISD::CopyToReg)
425    return true;
426  if (Use->isMachineOpcode()) {
427    const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
428        CurDAG->getSubtarget().getInstrInfo());
429
430    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
431    if (MCID.mayStore())
432      return true;
433    unsigned Opcode = MCID.getOpcode();
434    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
435      return true;
436    // vmlx feeding into another vmlx. We actually want to unfold
437    // the use later in the MLxExpansion pass. e.g.
438    // vmla
439    // vmla (stall 8 cycles)
440    //
441    // vmul (5 cycles)
442    // vadd (5 cycles)
443    // vmla
444    // This adds up to about 18 - 19 cycles.
445    //
446    // vmla
447    // vmul (stall 4 cycles)
448    // vadd adds up to about 14 cycles.
449    return TII->isFpMLxInstruction(Opcode);
450  }
451
452  return false;
453}
454
455bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
456                                            ARM_AM::ShiftOpc ShOpcVal,
457                                            unsigned ShAmt) {
458  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
459    return true;
460  if (Shift.hasOneUse())
461    return true;
462  // R << 2 is free.
463  return ShOpcVal == ARM_AM::lsl &&
464         (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
465}
466
467bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
468                                              SDValue &BaseReg,
469                                              SDValue &Opc,
470                                              bool CheckProfitability) {
471  if (DisableShifterOp)
472    return false;
473
474  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
475
476  // Don't match base register only case. That is matched to a separate
477  // lower complexity pattern with explicit register operand.
478  if (ShOpcVal == ARM_AM::no_shift) return false;
479
480  BaseReg = N.getOperand(0);
481  unsigned ShImmVal = 0;
482  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
483  if (!RHS) return false;
484  ShImmVal = RHS->getZExtValue() & 31;
485  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
486                                  MVT::i32);
487  return true;
488}
489
490bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
491                                              SDValue &BaseReg,
492                                              SDValue &ShReg,
493                                              SDValue &Opc,
494                                              bool CheckProfitability) {
495  if (DisableShifterOp)
496    return false;
497
498  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
499
500  // Don't match base register only case. That is matched to a separate
501  // lower complexity pattern with explicit register operand.
502  if (ShOpcVal == ARM_AM::no_shift) return false;
503
504  BaseReg = N.getOperand(0);
505  unsigned ShImmVal = 0;
506  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
507  if (RHS) return false;
508
509  ShReg = N.getOperand(1);
510  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
511    return false;
512  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
513                                  MVT::i32);
514  return true;
515}
516
517
518bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
519                                          SDValue &Base,
520                                          SDValue &OffImm) {
521  // Match simple R + imm12 operands.
522
523  // Base only.
524  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
525      !CurDAG->isBaseWithConstantOffset(N)) {
526    if (N.getOpcode() == ISD::FrameIndex) {
527      // Match frame index.
528      int FI = cast<FrameIndexSDNode>(N)->getIndex();
529      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
530      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
531      return true;
532    }
533
534    if (N.getOpcode() == ARMISD::Wrapper &&
535        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
536      Base = N.getOperand(0);
537    } else
538      Base = N;
539    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
540    return true;
541  }
542
543  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
544    int RHSC = (int)RHS->getSExtValue();
545    if (N.getOpcode() == ISD::SUB)
546      RHSC = -RHSC;
547
548    if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
549      Base   = N.getOperand(0);
550      if (Base.getOpcode() == ISD::FrameIndex) {
551        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
552        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
553      }
554      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
555      return true;
556    }
557  }
558
559  // Base only.
560  Base = N;
561  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
562  return true;
563}
564
565
566
567bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
568                                      SDValue &Opc) {
569  if (N.getOpcode() == ISD::MUL &&
570      ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
571    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
572      // X * [3,5,9] -> X + X * [2,4,8] etc.
573      int RHSC = (int)RHS->getZExtValue();
574      if (RHSC & 1) {
575        RHSC = RHSC & ~1;
576        ARM_AM::AddrOpc AddSub = ARM_AM::add;
577        if (RHSC < 0) {
578          AddSub = ARM_AM::sub;
579          RHSC = - RHSC;
580        }
581        if (isPowerOf2_32(RHSC)) {
582          unsigned ShAmt = Log2_32(RHSC);
583          Base = Offset = N.getOperand(0);
584          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
585                                                            ARM_AM::lsl),
586                                          MVT::i32);
587          return true;
588        }
589      }
590    }
591  }
592
593  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
594      // ISD::OR that is equivalent to an ISD::ADD.
595      !CurDAG->isBaseWithConstantOffset(N))
596    return false;
597
598  // Leave simple R +/- imm12 operands for LDRi12
599  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
600    int RHSC;
601    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
602                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
603      return false;
604  }
605
606  // Otherwise this is R +/- [possibly shifted] R.
607  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
608  ARM_AM::ShiftOpc ShOpcVal =
609    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
610  unsigned ShAmt = 0;
611
612  Base   = N.getOperand(0);
613  Offset = N.getOperand(1);
614
615  if (ShOpcVal != ARM_AM::no_shift) {
616    // Check to see if the RHS of the shift is a constant, if not, we can't fold
617    // it.
618    if (ConstantSDNode *Sh =
619           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
620      ShAmt = Sh->getZExtValue();
621      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
622        Offset = N.getOperand(1).getOperand(0);
623      else {
624        ShAmt = 0;
625        ShOpcVal = ARM_AM::no_shift;
626      }
627    } else {
628      ShOpcVal = ARM_AM::no_shift;
629    }
630  }
631
632  // Try matching (R shl C) + (R).
633  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
634      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
635        N.getOperand(0).hasOneUse())) {
636    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
637    if (ShOpcVal != ARM_AM::no_shift) {
638      // Check to see if the RHS of the shift is a constant, if not, we can't
639      // fold it.
640      if (ConstantSDNode *Sh =
641          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
642        ShAmt = Sh->getZExtValue();
643        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
644          Offset = N.getOperand(0).getOperand(0);
645          Base = N.getOperand(1);
646        } else {
647          ShAmt = 0;
648          ShOpcVal = ARM_AM::no_shift;
649        }
650      } else {
651        ShOpcVal = ARM_AM::no_shift;
652      }
653    }
654  }
655
656  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
657                                  MVT::i32);
658  return true;
659}
660
661
662//-----
663
664AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
665                                                     SDValue &Base,
666                                                     SDValue &Offset,
667                                                     SDValue &Opc) {
668  if (N.getOpcode() == ISD::MUL &&
669      (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
670    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
671      // X * [3,5,9] -> X + X * [2,4,8] etc.
672      int RHSC = (int)RHS->getZExtValue();
673      if (RHSC & 1) {
674        RHSC = RHSC & ~1;
675        ARM_AM::AddrOpc AddSub = ARM_AM::add;
676        if (RHSC < 0) {
677          AddSub = ARM_AM::sub;
678          RHSC = - RHSC;
679        }
680        if (isPowerOf2_32(RHSC)) {
681          unsigned ShAmt = Log2_32(RHSC);
682          Base = Offset = N.getOperand(0);
683          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
684                                                            ARM_AM::lsl),
685                                          MVT::i32);
686          return AM2_SHOP;
687        }
688      }
689    }
690  }
691
692  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
693      // ISD::OR that is equivalent to an ADD.
694      !CurDAG->isBaseWithConstantOffset(N)) {
695    Base = N;
696    if (N.getOpcode() == ISD::FrameIndex) {
697      int FI = cast<FrameIndexSDNode>(N)->getIndex();
698      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
699    } else if (N.getOpcode() == ARMISD::Wrapper &&
700               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
701      Base = N.getOperand(0);
702    }
703    Offset = CurDAG->getRegister(0, MVT::i32);
704    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
705                                                      ARM_AM::no_shift),
706                                    MVT::i32);
707    return AM2_BASE;
708  }
709
710  // Match simple R +/- imm12 operands.
711  if (N.getOpcode() != ISD::SUB) {
712    int RHSC;
713    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
714                                -0x1000+1, 0x1000, RHSC)) { // 12 bits.
715      Base = N.getOperand(0);
716      if (Base.getOpcode() == ISD::FrameIndex) {
717        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
718        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
719      }
720      Offset = CurDAG->getRegister(0, MVT::i32);
721
722      ARM_AM::AddrOpc AddSub = ARM_AM::add;
723      if (RHSC < 0) {
724        AddSub = ARM_AM::sub;
725        RHSC = - RHSC;
726      }
727      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
728                                                        ARM_AM::no_shift),
729                                      MVT::i32);
730      return AM2_BASE;
731    }
732  }
733
734  if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) {
735    // Compute R +/- (R << N) and reuse it.
736    Base = N;
737    Offset = CurDAG->getRegister(0, MVT::i32);
738    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
739                                                      ARM_AM::no_shift),
740                                    MVT::i32);
741    return AM2_BASE;
742  }
743
744  // Otherwise this is R +/- [possibly shifted] R.
745  ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
746  ARM_AM::ShiftOpc ShOpcVal =
747    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
748  unsigned ShAmt = 0;
749
750  Base   = N.getOperand(0);
751  Offset = N.getOperand(1);
752
753  if (ShOpcVal != ARM_AM::no_shift) {
754    // Check to see if the RHS of the shift is a constant, if not, we can't fold
755    // it.
756    if (ConstantSDNode *Sh =
757           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
758      ShAmt = Sh->getZExtValue();
759      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
760        Offset = N.getOperand(1).getOperand(0);
761      else {
762        ShAmt = 0;
763        ShOpcVal = ARM_AM::no_shift;
764      }
765    } else {
766      ShOpcVal = ARM_AM::no_shift;
767    }
768  }
769
770  // Try matching (R shl C) + (R).
771  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
772      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
773        N.getOperand(0).hasOneUse())) {
774    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
775    if (ShOpcVal != ARM_AM::no_shift) {
776      // Check to see if the RHS of the shift is a constant, if not, we can't
777      // fold it.
778      if (ConstantSDNode *Sh =
779          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
780        ShAmt = Sh->getZExtValue();
781        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
782          Offset = N.getOperand(0).getOperand(0);
783          Base = N.getOperand(1);
784        } else {
785          ShAmt = 0;
786          ShOpcVal = ARM_AM::no_shift;
787        }
788      } else {
789        ShOpcVal = ARM_AM::no_shift;
790      }
791    }
792  }
793
794  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
795                                  MVT::i32);
796  return AM2_SHOP;
797}
798
799bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
800                                            SDValue &Offset, SDValue &Opc) {
801  unsigned Opcode = Op->getOpcode();
802  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
803    ? cast<LoadSDNode>(Op)->getAddressingMode()
804    : cast<StoreSDNode>(Op)->getAddressingMode();
805  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
806    ? ARM_AM::add : ARM_AM::sub;
807  int Val;
808  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
809    return false;
810
811  Offset = N;
812  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
813  unsigned ShAmt = 0;
814  if (ShOpcVal != ARM_AM::no_shift) {
815    // Check to see if the RHS of the shift is a constant, if not, we can't fold
816    // it.
817    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
818      ShAmt = Sh->getZExtValue();
819      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
820        Offset = N.getOperand(0);
821      else {
822        ShAmt = 0;
823        ShOpcVal = ARM_AM::no_shift;
824      }
825    } else {
826      ShOpcVal = ARM_AM::no_shift;
827    }
828  }
829
830  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
831                                  MVT::i32);
832  return true;
833}
834
835bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
836                                            SDValue &Offset, SDValue &Opc) {
837  unsigned Opcode = Op->getOpcode();
838  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
839    ? cast<LoadSDNode>(Op)->getAddressingMode()
840    : cast<StoreSDNode>(Op)->getAddressingMode();
841  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
842    ? ARM_AM::add : ARM_AM::sub;
843  int Val;
844  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
845    if (AddSub == ARM_AM::sub) Val *= -1;
846    Offset = CurDAG->getRegister(0, MVT::i32);
847    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
848    return true;
849  }
850
851  return false;
852}
853
854
855bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
856                                            SDValue &Offset, SDValue &Opc) {
857  unsigned Opcode = Op->getOpcode();
858  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
859    ? cast<LoadSDNode>(Op)->getAddressingMode()
860    : cast<StoreSDNode>(Op)->getAddressingMode();
861  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
862    ? ARM_AM::add : ARM_AM::sub;
863  int Val;
864  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
865    Offset = CurDAG->getRegister(0, MVT::i32);
866    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
867                                                      ARM_AM::no_shift),
868                                    MVT::i32);
869    return true;
870  }
871
872  return false;
873}
874
875bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
876  Base = N;
877  return true;
878}
879
880bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
881                                      SDValue &Base, SDValue &Offset,
882                                      SDValue &Opc) {
883  if (N.getOpcode() == ISD::SUB) {
884    // X - C  is canonicalize to X + -C, no need to handle it here.
885    Base = N.getOperand(0);
886    Offset = N.getOperand(1);
887    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
888    return true;
889  }
890
891  if (!CurDAG->isBaseWithConstantOffset(N)) {
892    Base = N;
893    if (N.getOpcode() == ISD::FrameIndex) {
894      int FI = cast<FrameIndexSDNode>(N)->getIndex();
895      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
896    }
897    Offset = CurDAG->getRegister(0, MVT::i32);
898    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
899    return true;
900  }
901
902  // If the RHS is +/- imm8, fold into addr mode.
903  int RHSC;
904  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
905                              -256 + 1, 256, RHSC)) { // 8 bits.
906    Base = N.getOperand(0);
907    if (Base.getOpcode() == ISD::FrameIndex) {
908      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
909      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
910    }
911    Offset = CurDAG->getRegister(0, MVT::i32);
912
913    ARM_AM::AddrOpc AddSub = ARM_AM::add;
914    if (RHSC < 0) {
915      AddSub = ARM_AM::sub;
916      RHSC = -RHSC;
917    }
918    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
919    return true;
920  }
921
922  Base = N.getOperand(0);
923  Offset = N.getOperand(1);
924  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
925  return true;
926}
927
928bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
929                                            SDValue &Offset, SDValue &Opc) {
930  unsigned Opcode = Op->getOpcode();
931  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
932    ? cast<LoadSDNode>(Op)->getAddressingMode()
933    : cast<StoreSDNode>(Op)->getAddressingMode();
934  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
935    ? ARM_AM::add : ARM_AM::sub;
936  int Val;
937  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
938    Offset = CurDAG->getRegister(0, MVT::i32);
939    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
940    return true;
941  }
942
943  Offset = N;
944  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
945  return true;
946}
947
948bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
949                                      SDValue &Base, SDValue &Offset) {
950  if (!CurDAG->isBaseWithConstantOffset(N)) {
951    Base = N;
952    if (N.getOpcode() == ISD::FrameIndex) {
953      int FI = cast<FrameIndexSDNode>(N)->getIndex();
954      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
955    } else if (N.getOpcode() == ARMISD::Wrapper &&
956               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
957      Base = N.getOperand(0);
958    }
959    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
960                                       MVT::i32);
961    return true;
962  }
963
964  // If the RHS is +/- imm8, fold into addr mode.
965  int RHSC;
966  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
967                              -256 + 1, 256, RHSC)) {
968    Base = N.getOperand(0);
969    if (Base.getOpcode() == ISD::FrameIndex) {
970      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
971      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
972    }
973
974    ARM_AM::AddrOpc AddSub = ARM_AM::add;
975    if (RHSC < 0) {
976      AddSub = ARM_AM::sub;
977      RHSC = -RHSC;
978    }
979    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
980                                       MVT::i32);
981    return true;
982  }
983
984  Base = N;
985  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
986                                     MVT::i32);
987  return true;
988}
989
990bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
991                                      SDValue &Align) {
992  Addr = N;
993
994  unsigned Alignment = 0;
995
996  MemSDNode *MemN = cast<MemSDNode>(Parent);
997
998  if (isa<LSBaseSDNode>(MemN) ||
999      ((MemN->getOpcode() == ARMISD::VST1_UPD ||
1000        MemN->getOpcode() == ARMISD::VLD1_UPD) &&
1001       MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) {
1002    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
1003    // The maximum alignment is equal to the memory size being referenced.
1004    unsigned MMOAlign = MemN->getAlignment();
1005    unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
1006    if (MMOAlign >= MemSize && MemSize > 1)
1007      Alignment = MemSize;
1008  } else {
1009    // All other uses of addrmode6 are for intrinsics.  For now just record
1010    // the raw alignment value; it will be refined later based on the legal
1011    // alignment operands for the intrinsic.
1012    Alignment = MemN->getAlignment();
1013  }
1014
1015  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1016  return true;
1017}
1018
1019bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1020                                            SDValue &Offset) {
1021  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1022  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
1023  if (AM != ISD::POST_INC)
1024    return false;
1025  Offset = N;
1026  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1027    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1028      Offset = CurDAG->getRegister(0, MVT::i32);
1029  }
1030  return true;
1031}
1032
1033bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1034                                       SDValue &Offset, SDValue &Label) {
1035  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1036    Offset = N.getOperand(0);
1037    SDValue N1 = N.getOperand(1);
1038    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1039                                      MVT::i32);
1040    return true;
1041  }
1042
1043  return false;
1044}
1045
1046
1047//===----------------------------------------------------------------------===//
1048//                         Thumb Addressing Modes
1049//===----------------------------------------------------------------------===//
1050
1051bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
1052                                            SDValue &Base, SDValue &Offset){
1053  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1054    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
1055    if (!NC || !NC->isNullValue())
1056      return false;
1057
1058    Base = Offset = N;
1059    return true;
1060  }
1061
1062  Base = N.getOperand(0);
1063  Offset = N.getOperand(1);
1064  return true;
1065}
1066
1067bool
1068ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
1069                                       SDValue &Offset, unsigned Scale) {
1070  if (Scale == 4) {
1071    SDValue TmpBase, TmpOffImm;
1072    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1073      return false;  // We want to select tLDRspi / tSTRspi instead.
1074
1075    if (N.getOpcode() == ARMISD::Wrapper &&
1076        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1077      return false;  // We want to select tLDRpci instead.
1078  }
1079
1080  if (!CurDAG->isBaseWithConstantOffset(N))
1081    return false;
1082
1083  // Thumb does not have [sp, r] address mode.
1084  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1085  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1086  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1087      (RHSR && RHSR->getReg() == ARM::SP))
1088    return false;
1089
1090  // FIXME: Why do we explicitly check for a match here and then return false?
1091  // Presumably to allow something else to match, but shouldn't this be
1092  // documented?
1093  int RHSC;
1094  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
1095    return false;
1096
1097  Base = N.getOperand(0);
1098  Offset = N.getOperand(1);
1099  return true;
1100}
1101
1102bool
1103ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
1104                                          SDValue &Base,
1105                                          SDValue &Offset) {
1106  return SelectThumbAddrModeRI(N, Base, Offset, 1);
1107}
1108
1109bool
1110ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
1111                                          SDValue &Base,
1112                                          SDValue &Offset) {
1113  return SelectThumbAddrModeRI(N, Base, Offset, 2);
1114}
1115
1116bool
1117ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
1118                                          SDValue &Base,
1119                                          SDValue &Offset) {
1120  return SelectThumbAddrModeRI(N, Base, Offset, 4);
1121}
1122
1123bool
1124ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1125                                          SDValue &Base, SDValue &OffImm) {
1126  if (Scale == 4) {
1127    SDValue TmpBase, TmpOffImm;
1128    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1129      return false;  // We want to select tLDRspi / tSTRspi instead.
1130
1131    if (N.getOpcode() == ARMISD::Wrapper &&
1132        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1133      return false;  // We want to select tLDRpci instead.
1134  }
1135
1136  if (!CurDAG->isBaseWithConstantOffset(N)) {
1137    if (N.getOpcode() == ARMISD::Wrapper &&
1138        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
1139      Base = N.getOperand(0);
1140    } else {
1141      Base = N;
1142    }
1143
1144    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1145    return true;
1146  }
1147
1148  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1149  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1150  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1151      (RHSR && RHSR->getReg() == ARM::SP)) {
1152    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1153    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1154    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1155    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1156
1157    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1158    if (LHSC != 0 || RHSC != 0) return false;
1159
1160    Base = N;
1161    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1162    return true;
1163  }
1164
1165  // If the RHS is + imm5 * scale, fold into addr mode.
1166  int RHSC;
1167  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1168    Base = N.getOperand(0);
1169    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1170    return true;
1171  }
1172
1173  Base = N.getOperand(0);
1174  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1175  return true;
1176}
1177
1178bool
1179ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1180                                           SDValue &OffImm) {
1181  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1182}
1183
1184bool
1185ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1186                                           SDValue &OffImm) {
1187  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1188}
1189
1190bool
1191ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1192                                           SDValue &OffImm) {
1193  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1194}
1195
1196bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1197                                            SDValue &Base, SDValue &OffImm) {
1198  if (N.getOpcode() == ISD::FrameIndex) {
1199    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1200    // Only multiples of 4 are allowed for the offset, so the frame object
1201    // alignment must be at least 4.
1202    MachineFrameInfo *MFI = MF->getFrameInfo();
1203    if (MFI->getObjectAlignment(FI) < 4)
1204      MFI->setObjectAlignment(FI, 4);
1205    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1206    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1207    return true;
1208  }
1209
1210  if (!CurDAG->isBaseWithConstantOffset(N))
1211    return false;
1212
1213  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1214  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1215      (LHSR && LHSR->getReg() == ARM::SP)) {
1216    // If the RHS is + imm8 * scale, fold into addr mode.
1217    int RHSC;
1218    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1219      Base = N.getOperand(0);
1220      if (Base.getOpcode() == ISD::FrameIndex) {
1221        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1222        // For LHS+RHS to result in an offset that's a multiple of 4 the object
1223        // indexed by the LHS must be 4-byte aligned.
1224        MachineFrameInfo *MFI = MF->getFrameInfo();
1225        if (MFI->getObjectAlignment(FI) < 4)
1226          MFI->setObjectAlignment(FI, 4);
1227        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1228      }
1229      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1230      return true;
1231    }
1232  }
1233
1234  return false;
1235}
1236
1237
1238//===----------------------------------------------------------------------===//
1239//                        Thumb 2 Addressing Modes
1240//===----------------------------------------------------------------------===//
1241
1242
1243bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1244                                                SDValue &Opc) {
1245  if (DisableShifterOp)
1246    return false;
1247
1248  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1249
1250  // Don't match base register only case. That is matched to a separate
1251  // lower complexity pattern with explicit register operand.
1252  if (ShOpcVal == ARM_AM::no_shift) return false;
1253
1254  BaseReg = N.getOperand(0);
1255  unsigned ShImmVal = 0;
1256  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1257    ShImmVal = RHS->getZExtValue() & 31;
1258    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1259    return true;
1260  }
1261
1262  return false;
1263}
1264
1265bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1266                                            SDValue &Base, SDValue &OffImm) {
1267  // Match simple R + imm12 operands.
1268
1269  // Base only.
1270  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1271      !CurDAG->isBaseWithConstantOffset(N)) {
1272    if (N.getOpcode() == ISD::FrameIndex) {
1273      // Match frame index.
1274      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1275      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1276      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1277      return true;
1278    }
1279
1280    if (N.getOpcode() == ARMISD::Wrapper &&
1281        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
1282      Base = N.getOperand(0);
1283      if (Base.getOpcode() == ISD::TargetConstantPool)
1284        return false;  // We want to select t2LDRpci instead.
1285    } else
1286      Base = N;
1287    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1288    return true;
1289  }
1290
1291  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1292    if (SelectT2AddrModeImm8(N, Base, OffImm))
1293      // Let t2LDRi8 handle (R - imm8).
1294      return false;
1295
1296    int RHSC = (int)RHS->getZExtValue();
1297    if (N.getOpcode() == ISD::SUB)
1298      RHSC = -RHSC;
1299
1300    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1301      Base   = N.getOperand(0);
1302      if (Base.getOpcode() == ISD::FrameIndex) {
1303        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1304        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1305      }
1306      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1307      return true;
1308    }
1309  }
1310
1311  // Base only.
1312  Base = N;
1313  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1314  return true;
1315}
1316
1317bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1318                                           SDValue &Base, SDValue &OffImm) {
1319  // Match simple R - imm8 operands.
1320  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1321      !CurDAG->isBaseWithConstantOffset(N))
1322    return false;
1323
1324  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1325    int RHSC = (int)RHS->getSExtValue();
1326    if (N.getOpcode() == ISD::SUB)
1327      RHSC = -RHSC;
1328
1329    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1330      Base = N.getOperand(0);
1331      if (Base.getOpcode() == ISD::FrameIndex) {
1332        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1333        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1334      }
1335      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1336      return true;
1337    }
1338  }
1339
1340  return false;
1341}
1342
1343bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1344                                                 SDValue &OffImm){
1345  unsigned Opcode = Op->getOpcode();
1346  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1347    ? cast<LoadSDNode>(Op)->getAddressingMode()
1348    : cast<StoreSDNode>(Op)->getAddressingMode();
1349  int RHSC;
1350  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1351    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1352      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1353      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1354    return true;
1355  }
1356
1357  return false;
1358}
1359
1360bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1361                                            SDValue &Base,
1362                                            SDValue &OffReg, SDValue &ShImm) {
1363  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1364  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1365    return false;
1366
1367  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1368  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1369    int RHSC = (int)RHS->getZExtValue();
1370    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1371      return false;
1372    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1373      return false;
1374  }
1375
1376  // Look for (R + R) or (R + (R << [1,2,3])).
1377  unsigned ShAmt = 0;
1378  Base   = N.getOperand(0);
1379  OffReg = N.getOperand(1);
1380
1381  // Swap if it is ((R << c) + R).
1382  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1383  if (ShOpcVal != ARM_AM::lsl) {
1384    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1385    if (ShOpcVal == ARM_AM::lsl)
1386      std::swap(Base, OffReg);
1387  }
1388
1389  if (ShOpcVal == ARM_AM::lsl) {
1390    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1391    // it.
1392    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1393      ShAmt = Sh->getZExtValue();
1394      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1395        OffReg = OffReg.getOperand(0);
1396      else {
1397        ShAmt = 0;
1398      }
1399    }
1400  }
1401
1402  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1403
1404  return true;
1405}
1406
1407bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1408                                                SDValue &OffImm) {
1409  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1410  // instructions.
1411  Base = N;
1412  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1413
1414  if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1415    return true;
1416
1417  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1418  if (!RHS)
1419    return true;
1420
1421  uint32_t RHSC = (int)RHS->getZExtValue();
1422  if (RHSC > 1020 || RHSC % 4 != 0)
1423    return true;
1424
1425  Base = N.getOperand(0);
1426  if (Base.getOpcode() == ISD::FrameIndex) {
1427    int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1428    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1429  }
1430
1431  OffImm = CurDAG->getTargetConstant(RHSC / 4, MVT::i32);
1432  return true;
1433}
1434
1435//===--------------------------------------------------------------------===//
1436
1437/// getAL - Returns a ARMCC::AL immediate node.
1438static inline SDValue getAL(SelectionDAG *CurDAG) {
1439  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1440}
1441
1442SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1443  LoadSDNode *LD = cast<LoadSDNode>(N);
1444  ISD::MemIndexedMode AM = LD->getAddressingMode();
1445  if (AM == ISD::UNINDEXED)
1446    return nullptr;
1447
1448  EVT LoadedVT = LD->getMemoryVT();
1449  SDValue Offset, AMOpc;
1450  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1451  unsigned Opcode = 0;
1452  bool Match = false;
1453  if (LoadedVT == MVT::i32 && isPre &&
1454      SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1455    Opcode = ARM::LDR_PRE_IMM;
1456    Match = true;
1457  } else if (LoadedVT == MVT::i32 && !isPre &&
1458      SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1459    Opcode = ARM::LDR_POST_IMM;
1460    Match = true;
1461  } else if (LoadedVT == MVT::i32 &&
1462      SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1463    Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1464    Match = true;
1465
1466  } else if (LoadedVT == MVT::i16 &&
1467             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1468    Match = true;
1469    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1470      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1471      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1472  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1473    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1474      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1475        Match = true;
1476        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1477      }
1478    } else {
1479      if (isPre &&
1480          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1481        Match = true;
1482        Opcode = ARM::LDRB_PRE_IMM;
1483      } else if (!isPre &&
1484                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1485        Match = true;
1486        Opcode = ARM::LDRB_POST_IMM;
1487      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1488        Match = true;
1489        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1490      }
1491    }
1492  }
1493
1494  if (Match) {
1495    if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1496      SDValue Chain = LD->getChain();
1497      SDValue Base = LD->getBasePtr();
1498      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
1499                       CurDAG->getRegister(0, MVT::i32), Chain };
1500      return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1501                                    MVT::i32, MVT::Other, Ops);
1502    } else {
1503      SDValue Chain = LD->getChain();
1504      SDValue Base = LD->getBasePtr();
1505      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1506                       CurDAG->getRegister(0, MVT::i32), Chain };
1507      return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1508                                    MVT::i32, MVT::Other, Ops);
1509    }
1510  }
1511
1512  return nullptr;
1513}
1514
1515SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1516  LoadSDNode *LD = cast<LoadSDNode>(N);
1517  ISD::MemIndexedMode AM = LD->getAddressingMode();
1518  if (AM == ISD::UNINDEXED)
1519    return nullptr;
1520
1521  EVT LoadedVT = LD->getMemoryVT();
1522  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1523  SDValue Offset;
1524  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1525  unsigned Opcode = 0;
1526  bool Match = false;
1527  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1528    switch (LoadedVT.getSimpleVT().SimpleTy) {
1529    case MVT::i32:
1530      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1531      break;
1532    case MVT::i16:
1533      if (isSExtLd)
1534        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1535      else
1536        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1537      break;
1538    case MVT::i8:
1539    case MVT::i1:
1540      if (isSExtLd)
1541        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1542      else
1543        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1544      break;
1545    default:
1546      return nullptr;
1547    }
1548    Match = true;
1549  }
1550
1551  if (Match) {
1552    SDValue Chain = LD->getChain();
1553    SDValue Base = LD->getBasePtr();
1554    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1555                     CurDAG->getRegister(0, MVT::i32), Chain };
1556    return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1557                                  MVT::Other, Ops);
1558  }
1559
1560  return nullptr;
1561}
1562
1563/// \brief Form a GPRPair pseudo register from a pair of GPR regs.
1564SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1565  SDLoc dl(V0.getNode());
1566  SDValue RegClass =
1567    CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32);
1568  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
1569  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
1570  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1571  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1572}
1573
1574/// \brief Form a D register from a pair of S registers.
1575SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1576  SDLoc dl(V0.getNode());
1577  SDValue RegClass =
1578    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1579  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1580  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1581  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1582  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1583}
1584
1585/// \brief Form a quad register from a pair of D registers.
1586SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1587  SDLoc dl(V0.getNode());
1588  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1589  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1590  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1591  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1592  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1593}
1594
1595/// \brief Form 4 consecutive D registers from a pair of Q registers.
1596SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1597  SDLoc dl(V0.getNode());
1598  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1599  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1600  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1601  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1602  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1603}
1604
1605/// \brief Form 4 consecutive S registers.
1606SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1607                                   SDValue V2, SDValue V3) {
1608  SDLoc dl(V0.getNode());
1609  SDValue RegClass =
1610    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1611  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1612  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1613  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1614  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1615  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1616                                    V2, SubReg2, V3, SubReg3 };
1617  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1618}
1619
1620/// \brief Form 4 consecutive D registers.
1621SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1622                                   SDValue V2, SDValue V3) {
1623  SDLoc dl(V0.getNode());
1624  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1625  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1626  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1627  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1628  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1629  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1630                                    V2, SubReg2, V3, SubReg3 };
1631  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1632}
1633
1634/// \brief Form 4 consecutive Q registers.
1635SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1636                                   SDValue V2, SDValue V3) {
1637  SDLoc dl(V0.getNode());
1638  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1639  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1640  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1641  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1642  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1643  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1644                                    V2, SubReg2, V3, SubReg3 };
1645  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1646}
1647
1648/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1649/// of a NEON VLD or VST instruction.  The supported values depend on the
1650/// number of registers being loaded.
1651SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1652                                       bool is64BitVector) {
1653  unsigned NumRegs = NumVecs;
1654  if (!is64BitVector && NumVecs < 3)
1655    NumRegs *= 2;
1656
1657  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1658  if (Alignment >= 32 && NumRegs == 4)
1659    Alignment = 32;
1660  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1661    Alignment = 16;
1662  else if (Alignment >= 8)
1663    Alignment = 8;
1664  else
1665    Alignment = 0;
1666
1667  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1668}
1669
1670static bool isVLDfixed(unsigned Opc)
1671{
1672  switch (Opc) {
1673  default: return false;
1674  case ARM::VLD1d8wb_fixed : return true;
1675  case ARM::VLD1d16wb_fixed : return true;
1676  case ARM::VLD1d64Qwb_fixed : return true;
1677  case ARM::VLD1d32wb_fixed : return true;
1678  case ARM::VLD1d64wb_fixed : return true;
1679  case ARM::VLD1d64TPseudoWB_fixed : return true;
1680  case ARM::VLD1d64QPseudoWB_fixed : return true;
1681  case ARM::VLD1q8wb_fixed : return true;
1682  case ARM::VLD1q16wb_fixed : return true;
1683  case ARM::VLD1q32wb_fixed : return true;
1684  case ARM::VLD1q64wb_fixed : return true;
1685  case ARM::VLD2d8wb_fixed : return true;
1686  case ARM::VLD2d16wb_fixed : return true;
1687  case ARM::VLD2d32wb_fixed : return true;
1688  case ARM::VLD2q8PseudoWB_fixed : return true;
1689  case ARM::VLD2q16PseudoWB_fixed : return true;
1690  case ARM::VLD2q32PseudoWB_fixed : return true;
1691  case ARM::VLD2DUPd8wb_fixed : return true;
1692  case ARM::VLD2DUPd16wb_fixed : return true;
1693  case ARM::VLD2DUPd32wb_fixed : return true;
1694  }
1695}
1696
1697static bool isVSTfixed(unsigned Opc)
1698{
1699  switch (Opc) {
1700  default: return false;
1701  case ARM::VST1d8wb_fixed : return true;
1702  case ARM::VST1d16wb_fixed : return true;
1703  case ARM::VST1d32wb_fixed : return true;
1704  case ARM::VST1d64wb_fixed : return true;
1705  case ARM::VST1q8wb_fixed : return true;
1706  case ARM::VST1q16wb_fixed : return true;
1707  case ARM::VST1q32wb_fixed : return true;
1708  case ARM::VST1q64wb_fixed : return true;
1709  case ARM::VST1d64TPseudoWB_fixed : return true;
1710  case ARM::VST1d64QPseudoWB_fixed : return true;
1711  case ARM::VST2d8wb_fixed : return true;
1712  case ARM::VST2d16wb_fixed : return true;
1713  case ARM::VST2d32wb_fixed : return true;
1714  case ARM::VST2q8PseudoWB_fixed : return true;
1715  case ARM::VST2q16PseudoWB_fixed : return true;
1716  case ARM::VST2q32PseudoWB_fixed : return true;
1717  }
1718}
1719
1720// Get the register stride update opcode of a VLD/VST instruction that
1721// is otherwise equivalent to the given fixed stride updating instruction.
1722static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1723  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1724    && "Incorrect fixed stride updating instruction.");
1725  switch (Opc) {
1726  default: break;
1727  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1728  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1729  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1730  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1731  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1732  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1733  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1734  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1735  case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
1736  case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
1737  case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
1738  case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
1739
1740  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1741  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1742  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1743  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1744  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1745  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1746  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1747  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1748  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1749  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1750
1751  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1752  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1753  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1754  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1755  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1756  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1757
1758  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1759  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1760  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1761  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1762  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1763  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1764
1765  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1766  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1767  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1768  }
1769  return Opc; // If not one we handle, return it unchanged.
1770}
1771
1772SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1773                                   const uint16_t *DOpcodes,
1774                                   const uint16_t *QOpcodes0,
1775                                   const uint16_t *QOpcodes1) {
1776  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1777  SDLoc dl(N);
1778
1779  SDValue MemAddr, Align;
1780  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1781  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1782    return nullptr;
1783
1784  SDValue Chain = N->getOperand(0);
1785  EVT VT = N->getValueType(0);
1786  bool is64BitVector = VT.is64BitVector();
1787  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1788
1789  unsigned OpcodeIndex;
1790  switch (VT.getSimpleVT().SimpleTy) {
1791  default: llvm_unreachable("unhandled vld type");
1792    // Double-register operations:
1793  case MVT::v8i8:  OpcodeIndex = 0; break;
1794  case MVT::v4i16: OpcodeIndex = 1; break;
1795  case MVT::v2f32:
1796  case MVT::v2i32: OpcodeIndex = 2; break;
1797  case MVT::v1i64: OpcodeIndex = 3; break;
1798    // Quad-register operations:
1799  case MVT::v16i8: OpcodeIndex = 0; break;
1800  case MVT::v8i16: OpcodeIndex = 1; break;
1801  case MVT::v4f32:
1802  case MVT::v4i32: OpcodeIndex = 2; break;
1803  case MVT::v2f64:
1804  case MVT::v2i64: OpcodeIndex = 3;
1805    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1806    break;
1807  }
1808
1809  EVT ResTy;
1810  if (NumVecs == 1)
1811    ResTy = VT;
1812  else {
1813    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1814    if (!is64BitVector)
1815      ResTyElts *= 2;
1816    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1817  }
1818  std::vector<EVT> ResTys;
1819  ResTys.push_back(ResTy);
1820  if (isUpdating)
1821    ResTys.push_back(MVT::i32);
1822  ResTys.push_back(MVT::Other);
1823
1824  SDValue Pred = getAL(CurDAG);
1825  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1826  SDNode *VLd;
1827  SmallVector<SDValue, 7> Ops;
1828
1829  // Double registers and VLD1/VLD2 quad registers are directly supported.
1830  if (is64BitVector || NumVecs <= 2) {
1831    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1832                    QOpcodes0[OpcodeIndex]);
1833    Ops.push_back(MemAddr);
1834    Ops.push_back(Align);
1835    if (isUpdating) {
1836      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1837      // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
1838      // case entirely when the rest are updated to that form, too.
1839      if ((NumVecs <= 2) && !isa<ConstantSDNode>(Inc.getNode()))
1840        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1841      // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1842      // check for that explicitly too. Horribly hacky, but temporary.
1843      if ((NumVecs > 2 && !isVLDfixed(Opc)) ||
1844          !isa<ConstantSDNode>(Inc.getNode()))
1845        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1846    }
1847    Ops.push_back(Pred);
1848    Ops.push_back(Reg0);
1849    Ops.push_back(Chain);
1850    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1851
1852  } else {
1853    // Otherwise, quad registers are loaded with two separate instructions,
1854    // where one loads the even registers and the other loads the odd registers.
1855    EVT AddrTy = MemAddr.getValueType();
1856
1857    // Load the even subregs.  This is always an updating load, so that it
1858    // provides the address to the second load for the odd subregs.
1859    SDValue ImplDef =
1860      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1861    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1862    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1863                                          ResTy, AddrTy, MVT::Other, OpsA);
1864    Chain = SDValue(VLdA, 2);
1865
1866    // Load the odd subregs.
1867    Ops.push_back(SDValue(VLdA, 1));
1868    Ops.push_back(Align);
1869    if (isUpdating) {
1870      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1871      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1872             "only constant post-increment update allowed for VLD3/4");
1873      (void)Inc;
1874      Ops.push_back(Reg0);
1875    }
1876    Ops.push_back(SDValue(VLdA, 0));
1877    Ops.push_back(Pred);
1878    Ops.push_back(Reg0);
1879    Ops.push_back(Chain);
1880    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1881  }
1882
1883  // Transfer memoperands.
1884  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1885  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1886  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1887
1888  if (NumVecs == 1)
1889    return VLd;
1890
1891  // Extract out the subregisters.
1892  SDValue SuperReg = SDValue(VLd, 0);
1893  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1894         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1895  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1896  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1897    ReplaceUses(SDValue(N, Vec),
1898                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1899  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1900  if (isUpdating)
1901    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1902  return nullptr;
1903}
1904
1905SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1906                                   const uint16_t *DOpcodes,
1907                                   const uint16_t *QOpcodes0,
1908                                   const uint16_t *QOpcodes1) {
1909  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1910  SDLoc dl(N);
1911
1912  SDValue MemAddr, Align;
1913  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1914  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1915  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1916    return nullptr;
1917
1918  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1919  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1920
1921  SDValue Chain = N->getOperand(0);
1922  EVT VT = N->getOperand(Vec0Idx).getValueType();
1923  bool is64BitVector = VT.is64BitVector();
1924  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1925
1926  unsigned OpcodeIndex;
1927  switch (VT.getSimpleVT().SimpleTy) {
1928  default: llvm_unreachable("unhandled vst type");
1929    // Double-register operations:
1930  case MVT::v8i8:  OpcodeIndex = 0; break;
1931  case MVT::v4i16: OpcodeIndex = 1; break;
1932  case MVT::v2f32:
1933  case MVT::v2i32: OpcodeIndex = 2; break;
1934  case MVT::v1i64: OpcodeIndex = 3; break;
1935    // Quad-register operations:
1936  case MVT::v16i8: OpcodeIndex = 0; break;
1937  case MVT::v8i16: OpcodeIndex = 1; break;
1938  case MVT::v4f32:
1939  case MVT::v4i32: OpcodeIndex = 2; break;
1940  case MVT::v2f64:
1941  case MVT::v2i64: OpcodeIndex = 3;
1942    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1943    break;
1944  }
1945
1946  std::vector<EVT> ResTys;
1947  if (isUpdating)
1948    ResTys.push_back(MVT::i32);
1949  ResTys.push_back(MVT::Other);
1950
1951  SDValue Pred = getAL(CurDAG);
1952  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1953  SmallVector<SDValue, 7> Ops;
1954
1955  // Double registers and VST1/VST2 quad registers are directly supported.
1956  if (is64BitVector || NumVecs <= 2) {
1957    SDValue SrcReg;
1958    if (NumVecs == 1) {
1959      SrcReg = N->getOperand(Vec0Idx);
1960    } else if (is64BitVector) {
1961      // Form a REG_SEQUENCE to force register allocation.
1962      SDValue V0 = N->getOperand(Vec0Idx + 0);
1963      SDValue V1 = N->getOperand(Vec0Idx + 1);
1964      if (NumVecs == 2)
1965        SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
1966      else {
1967        SDValue V2 = N->getOperand(Vec0Idx + 2);
1968        // If it's a vst3, form a quad D-register and leave the last part as
1969        // an undef.
1970        SDValue V3 = (NumVecs == 3)
1971          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1972          : N->getOperand(Vec0Idx + 3);
1973        SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
1974      }
1975    } else {
1976      // Form a QQ register.
1977      SDValue Q0 = N->getOperand(Vec0Idx);
1978      SDValue Q1 = N->getOperand(Vec0Idx + 1);
1979      SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
1980    }
1981
1982    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1983                    QOpcodes0[OpcodeIndex]);
1984    Ops.push_back(MemAddr);
1985    Ops.push_back(Align);
1986    if (isUpdating) {
1987      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1988      // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
1989      // case entirely when the rest are updated to that form, too.
1990      if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
1991        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1992      // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
1993      // check for that explicitly too. Horribly hacky, but temporary.
1994      if  (!isa<ConstantSDNode>(Inc.getNode()))
1995        Ops.push_back(Inc);
1996      else if (NumVecs > 2 && !isVSTfixed(Opc))
1997        Ops.push_back(Reg0);
1998    }
1999    Ops.push_back(SrcReg);
2000    Ops.push_back(Pred);
2001    Ops.push_back(Reg0);
2002    Ops.push_back(Chain);
2003    SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2004
2005    // Transfer memoperands.
2006    cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
2007
2008    return VSt;
2009  }
2010
2011  // Otherwise, quad registers are stored with two separate instructions,
2012  // where one stores the even registers and the other stores the odd registers.
2013
2014  // Form the QQQQ REG_SEQUENCE.
2015  SDValue V0 = N->getOperand(Vec0Idx + 0);
2016  SDValue V1 = N->getOperand(Vec0Idx + 1);
2017  SDValue V2 = N->getOperand(Vec0Idx + 2);
2018  SDValue V3 = (NumVecs == 3)
2019    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2020    : N->getOperand(Vec0Idx + 3);
2021  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2022
2023  // Store the even D registers.  This is always an updating store, so that it
2024  // provides the address to the second store for the odd subregs.
2025  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2026  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2027                                        MemAddr.getValueType(),
2028                                        MVT::Other, OpsA);
2029  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
2030  Chain = SDValue(VStA, 1);
2031
2032  // Store the odd D registers.
2033  Ops.push_back(SDValue(VStA, 0));
2034  Ops.push_back(Align);
2035  if (isUpdating) {
2036    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2037    assert(isa<ConstantSDNode>(Inc.getNode()) &&
2038           "only constant post-increment update allowed for VST3/4");
2039    (void)Inc;
2040    Ops.push_back(Reg0);
2041  }
2042  Ops.push_back(RegSeq);
2043  Ops.push_back(Pred);
2044  Ops.push_back(Reg0);
2045  Ops.push_back(Chain);
2046  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2047                                        Ops);
2048  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
2049  return VStB;
2050}
2051
2052SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
2053                                         bool isUpdating, unsigned NumVecs,
2054                                         const uint16_t *DOpcodes,
2055                                         const uint16_t *QOpcodes) {
2056  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2057  SDLoc dl(N);
2058
2059  SDValue MemAddr, Align;
2060  unsigned AddrOpIdx = isUpdating ? 1 : 2;
2061  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2062  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2063    return nullptr;
2064
2065  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2066  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2067
2068  SDValue Chain = N->getOperand(0);
2069  unsigned Lane =
2070    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2071  EVT VT = N->getOperand(Vec0Idx).getValueType();
2072  bool is64BitVector = VT.is64BitVector();
2073
2074  unsigned Alignment = 0;
2075  if (NumVecs != 3) {
2076    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2077    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2078    if (Alignment > NumBytes)
2079      Alignment = NumBytes;
2080    if (Alignment < 8 && Alignment < NumBytes)
2081      Alignment = 0;
2082    // Alignment must be a power of two; make sure of that.
2083    Alignment = (Alignment & -Alignment);
2084    if (Alignment == 1)
2085      Alignment = 0;
2086  }
2087  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2088
2089  unsigned OpcodeIndex;
2090  switch (VT.getSimpleVT().SimpleTy) {
2091  default: llvm_unreachable("unhandled vld/vst lane type");
2092    // Double-register operations:
2093  case MVT::v8i8:  OpcodeIndex = 0; break;
2094  case MVT::v4i16: OpcodeIndex = 1; break;
2095  case MVT::v2f32:
2096  case MVT::v2i32: OpcodeIndex = 2; break;
2097    // Quad-register operations:
2098  case MVT::v8i16: OpcodeIndex = 0; break;
2099  case MVT::v4f32:
2100  case MVT::v4i32: OpcodeIndex = 1; break;
2101  }
2102
2103  std::vector<EVT> ResTys;
2104  if (IsLoad) {
2105    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2106    if (!is64BitVector)
2107      ResTyElts *= 2;
2108    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2109                                      MVT::i64, ResTyElts));
2110  }
2111  if (isUpdating)
2112    ResTys.push_back(MVT::i32);
2113  ResTys.push_back(MVT::Other);
2114
2115  SDValue Pred = getAL(CurDAG);
2116  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2117
2118  SmallVector<SDValue, 8> Ops;
2119  Ops.push_back(MemAddr);
2120  Ops.push_back(Align);
2121  if (isUpdating) {
2122    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2123    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
2124  }
2125
2126  SDValue SuperReg;
2127  SDValue V0 = N->getOperand(Vec0Idx + 0);
2128  SDValue V1 = N->getOperand(Vec0Idx + 1);
2129  if (NumVecs == 2) {
2130    if (is64BitVector)
2131      SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2132    else
2133      SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2134  } else {
2135    SDValue V2 = N->getOperand(Vec0Idx + 2);
2136    SDValue V3 = (NumVecs == 3)
2137      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2138      : N->getOperand(Vec0Idx + 3);
2139    if (is64BitVector)
2140      SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2141    else
2142      SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2143  }
2144  Ops.push_back(SuperReg);
2145  Ops.push_back(getI32Imm(Lane));
2146  Ops.push_back(Pred);
2147  Ops.push_back(Reg0);
2148  Ops.push_back(Chain);
2149
2150  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2151                                  QOpcodes[OpcodeIndex]);
2152  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2153  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
2154  if (!IsLoad)
2155    return VLdLn;
2156
2157  // Extract the subregisters.
2158  SuperReg = SDValue(VLdLn, 0);
2159  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
2160         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
2161  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2162  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2163    ReplaceUses(SDValue(N, Vec),
2164                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2165  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2166  if (isUpdating)
2167    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2168  return nullptr;
2169}
2170
2171SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
2172                                      unsigned NumVecs,
2173                                      const uint16_t *Opcodes) {
2174  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2175  SDLoc dl(N);
2176
2177  SDValue MemAddr, Align;
2178  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2179    return nullptr;
2180
2181  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2182  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2183
2184  SDValue Chain = N->getOperand(0);
2185  EVT VT = N->getValueType(0);
2186
2187  unsigned Alignment = 0;
2188  if (NumVecs != 3) {
2189    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2190    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2191    if (Alignment > NumBytes)
2192      Alignment = NumBytes;
2193    if (Alignment < 8 && Alignment < NumBytes)
2194      Alignment = 0;
2195    // Alignment must be a power of two; make sure of that.
2196    Alignment = (Alignment & -Alignment);
2197    if (Alignment == 1)
2198      Alignment = 0;
2199  }
2200  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2201
2202  unsigned OpcodeIndex;
2203  switch (VT.getSimpleVT().SimpleTy) {
2204  default: llvm_unreachable("unhandled vld-dup type");
2205  case MVT::v8i8:  OpcodeIndex = 0; break;
2206  case MVT::v4i16: OpcodeIndex = 1; break;
2207  case MVT::v2f32:
2208  case MVT::v2i32: OpcodeIndex = 2; break;
2209  }
2210
2211  SDValue Pred = getAL(CurDAG);
2212  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2213  SDValue SuperReg;
2214  unsigned Opc = Opcodes[OpcodeIndex];
2215  SmallVector<SDValue, 6> Ops;
2216  Ops.push_back(MemAddr);
2217  Ops.push_back(Align);
2218  if (isUpdating) {
2219    // fixed-stride update instructions don't have an explicit writeback
2220    // operand. It's implicit in the opcode itself.
2221    SDValue Inc = N->getOperand(2);
2222    if (!isa<ConstantSDNode>(Inc.getNode()))
2223      Ops.push_back(Inc);
2224    // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2225    else if (NumVecs > 2)
2226      Ops.push_back(Reg0);
2227  }
2228  Ops.push_back(Pred);
2229  Ops.push_back(Reg0);
2230  Ops.push_back(Chain);
2231
2232  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2233  std::vector<EVT> ResTys;
2234  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2235  if (isUpdating)
2236    ResTys.push_back(MVT::i32);
2237  ResTys.push_back(MVT::Other);
2238  SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2239  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2240  SuperReg = SDValue(VLdDup, 0);
2241
2242  // Extract the subregisters.
2243  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
2244  unsigned SubIdx = ARM::dsub_0;
2245  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2246    ReplaceUses(SDValue(N, Vec),
2247                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2248  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2249  if (isUpdating)
2250    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2251  return nullptr;
2252}
2253
2254SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
2255                                    unsigned Opc) {
2256  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
2257  SDLoc dl(N);
2258  EVT VT = N->getValueType(0);
2259  unsigned FirstTblReg = IsExt ? 2 : 1;
2260
2261  // Form a REG_SEQUENCE to force register allocation.
2262  SDValue RegSeq;
2263  SDValue V0 = N->getOperand(FirstTblReg + 0);
2264  SDValue V1 = N->getOperand(FirstTblReg + 1);
2265  if (NumVecs == 2)
2266    RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0);
2267  else {
2268    SDValue V2 = N->getOperand(FirstTblReg + 2);
2269    // If it's a vtbl3, form a quad D-register and leave the last part as
2270    // an undef.
2271    SDValue V3 = (NumVecs == 3)
2272      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2273      : N->getOperand(FirstTblReg + 3);
2274    RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2275  }
2276
2277  SmallVector<SDValue, 6> Ops;
2278  if (IsExt)
2279    Ops.push_back(N->getOperand(1));
2280  Ops.push_back(RegSeq);
2281  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
2282  Ops.push_back(getAL(CurDAG)); // predicate
2283  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
2284  return CurDAG->getMachineNode(Opc, dl, VT, Ops);
2285}
2286
2287SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
2288                                                     bool isSigned) {
2289  if (!Subtarget->hasV6T2Ops())
2290    return nullptr;
2291
2292  unsigned Opc = isSigned
2293    ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2294    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2295
2296  // For unsigned extracts, check for a shift right and mask
2297  unsigned And_imm = 0;
2298  if (N->getOpcode() == ISD::AND) {
2299    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2300
2301      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2302      if (And_imm & (And_imm + 1))
2303        return nullptr;
2304
2305      unsigned Srl_imm = 0;
2306      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2307                                Srl_imm)) {
2308        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2309
2310        // Note: The width operand is encoded as width-1.
2311        unsigned Width = countTrailingOnes(And_imm) - 1;
2312        unsigned LSB = Srl_imm;
2313
2314        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2315
2316        if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2317          // It's cheaper to use a right shift to extract the top bits.
2318          if (Subtarget->isThumb()) {
2319            Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2320            SDValue Ops[] = { N->getOperand(0).getOperand(0),
2321                              CurDAG->getTargetConstant(LSB, MVT::i32),
2322                              getAL(CurDAG), Reg0, Reg0 };
2323            return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2324          }
2325
2326          // ARM models shift instructions as MOVsi with shifter operand.
2327          ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL);
2328          SDValue ShOpc =
2329            CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB),
2330                                      MVT::i32);
2331          SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2332                            getAL(CurDAG), Reg0, Reg0 };
2333          return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2334        }
2335
2336        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2337                          CurDAG->getTargetConstant(LSB, MVT::i32),
2338                          CurDAG->getTargetConstant(Width, MVT::i32),
2339                          getAL(CurDAG), Reg0 };
2340        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2341      }
2342    }
2343    return nullptr;
2344  }
2345
2346  // Otherwise, we're looking for a shift of a shift
2347  unsigned Shl_imm = 0;
2348  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2349    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2350    unsigned Srl_imm = 0;
2351    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2352      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2353      // Note: The width operand is encoded as width-1.
2354      unsigned Width = 32 - Srl_imm - 1;
2355      int LSB = Srl_imm - Shl_imm;
2356      if (LSB < 0)
2357        return nullptr;
2358      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2359      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2360                        CurDAG->getTargetConstant(LSB, MVT::i32),
2361                        CurDAG->getTargetConstant(Width, MVT::i32),
2362                        getAL(CurDAG), Reg0 };
2363      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2364    }
2365  }
2366
2367  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
2368    unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2369    unsigned LSB = 0;
2370    if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
2371        !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB))
2372      return nullptr;
2373
2374    if (LSB + Width > 32)
2375      return nullptr;
2376
2377    SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2378    SDValue Ops[] = { N->getOperand(0).getOperand(0),
2379                      CurDAG->getTargetConstant(LSB, MVT::i32),
2380                      CurDAG->getTargetConstant(Width - 1, MVT::i32),
2381                      getAL(CurDAG), Reg0 };
2382    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2383  }
2384
2385  return nullptr;
2386}
2387
2388/// Target-specific DAG combining for ISD::XOR.
2389/// Target-independent combining lowers SELECT_CC nodes of the form
2390/// select_cc setg[ge] X,  0,  X, -X
2391/// select_cc setgt    X, -1,  X, -X
2392/// select_cc setl[te] X,  0, -X,  X
2393/// select_cc setlt    X,  1, -X,  X
2394/// which represent Integer ABS into:
2395/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2396/// ARM instruction selection detects the latter and matches it to
2397/// ARM::ABS or ARM::t2ABS machine node.
2398SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
2399  SDValue XORSrc0 = N->getOperand(0);
2400  SDValue XORSrc1 = N->getOperand(1);
2401  EVT VT = N->getValueType(0);
2402
2403  if (Subtarget->isThumb1Only())
2404    return nullptr;
2405
2406  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2407    return nullptr;
2408
2409  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2410  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2411  SDValue SRASrc0 = XORSrc1.getOperand(0);
2412  SDValue SRASrc1 = XORSrc1.getOperand(1);
2413  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
2414  EVT XType = SRASrc0.getValueType();
2415  unsigned Size = XType.getSizeInBits() - 1;
2416
2417  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2418      XType.isInteger() && SRAConstant != nullptr &&
2419      Size == SRAConstant->getZExtValue()) {
2420    unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2421    return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2422  }
2423
2424  return nullptr;
2425}
2426
2427SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2428  // The only time a CONCAT_VECTORS operation can have legal types is when
2429  // two 64-bit vectors are concatenated to a 128-bit vector.
2430  EVT VT = N->getValueType(0);
2431  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2432    llvm_unreachable("unexpected CONCAT_VECTORS");
2433  return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1));
2434}
2435
2436SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2437  SDLoc dl(N);
2438
2439  if (N->isMachineOpcode()) {
2440    N->setNodeId(-1);
2441    return nullptr;   // Already selected.
2442  }
2443
2444  switch (N->getOpcode()) {
2445  default: break;
2446  case ISD::INLINEASM: {
2447    SDNode *ResNode = SelectInlineAsm(N);
2448    if (ResNode)
2449      return ResNode;
2450    break;
2451  }
2452  case ISD::XOR: {
2453    // Select special operations if XOR node forms integer ABS pattern
2454    SDNode *ResNode = SelectABSOp(N);
2455    if (ResNode)
2456      return ResNode;
2457    // Other cases are autogenerated.
2458    break;
2459  }
2460  case ISD::Constant: {
2461    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2462    bool UseCP = true;
2463    if (Subtarget->useMovt(*MF))
2464      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2465      // be done with MOV + MOVT, at worst.
2466      UseCP = false;
2467    else {
2468      if (Subtarget->isThumb()) {
2469        UseCP = (Val > 255 &&                                  // MOV
2470                 ~Val > 255 &&                                 // MOV + MVN
2471                 !ARM_AM::isThumbImmShiftedVal(Val) &&         // MOV + LSL
2472                 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW
2473      } else
2474        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&             // MOV
2475                 ARM_AM::getSOImmVal(~Val) == -1 &&            // MVN
2476                 !ARM_AM::isSOImmTwoPartVal(Val) &&            // two instrs.
2477                 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW
2478    }
2479
2480    if (UseCP) {
2481      SDValue CPIdx = CurDAG->getTargetConstantPool(
2482          ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
2483          TLI->getPointerTy());
2484
2485      SDNode *ResNode;
2486      if (Subtarget->isThumb()) {
2487        SDValue Pred = getAL(CurDAG);
2488        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2489        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2490        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2491                                         Ops);
2492      } else {
2493        SDValue Ops[] = {
2494          CPIdx,
2495          CurDAG->getTargetConstant(0, MVT::i32),
2496          getAL(CurDAG),
2497          CurDAG->getRegister(0, MVT::i32),
2498          CurDAG->getEntryNode()
2499        };
2500        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2501                                       Ops);
2502      }
2503      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2504      return nullptr;
2505    }
2506
2507    // Other cases are autogenerated.
2508    break;
2509  }
2510  case ISD::FrameIndex: {
2511    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2512    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2513    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
2514    if (Subtarget->isThumb1Only()) {
2515      // Set the alignment of the frame object to 4, to avoid having to generate
2516      // more than one ADD
2517      MachineFrameInfo *MFI = MF->getFrameInfo();
2518      if (MFI->getObjectAlignment(FI) < 4)
2519        MFI->setObjectAlignment(FI, 4);
2520      return CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
2521                                  CurDAG->getTargetConstant(0, MVT::i32));
2522    } else {
2523      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2524                      ARM::t2ADDri : ARM::ADDri);
2525      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2526                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2527                        CurDAG->getRegister(0, MVT::i32) };
2528      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2529    }
2530  }
2531  case ISD::SRL:
2532    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2533      return I;
2534    break;
2535  case ISD::SIGN_EXTEND_INREG:
2536  case ISD::SRA:
2537    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2538      return I;
2539    break;
2540  case ISD::MUL:
2541    if (Subtarget->isThumb1Only())
2542      break;
2543    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2544      unsigned RHSV = C->getZExtValue();
2545      if (!RHSV) break;
2546      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2547        unsigned ShImm = Log2_32(RHSV-1);
2548        if (ShImm >= 32)
2549          break;
2550        SDValue V = N->getOperand(0);
2551        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2552        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2553        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2554        if (Subtarget->isThumb()) {
2555          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2556          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2557        } else {
2558          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2559          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2560        }
2561      }
2562      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2563        unsigned ShImm = Log2_32(RHSV+1);
2564        if (ShImm >= 32)
2565          break;
2566        SDValue V = N->getOperand(0);
2567        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2568        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2569        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2570        if (Subtarget->isThumb()) {
2571          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2572          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2573        } else {
2574          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2575          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2576        }
2577      }
2578    }
2579    break;
2580  case ISD::AND: {
2581    // Check for unsigned bitfield extract
2582    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2583      return I;
2584
2585    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2586    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2587    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2588    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2589    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2590    EVT VT = N->getValueType(0);
2591    if (VT != MVT::i32)
2592      break;
2593    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2594      ? ARM::t2MOVTi16
2595      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2596    if (!Opc)
2597      break;
2598    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2599    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2600    if (!N1C)
2601      break;
2602    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2603      SDValue N2 = N0.getOperand(1);
2604      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2605      if (!N2C)
2606        break;
2607      unsigned N1CVal = N1C->getZExtValue();
2608      unsigned N2CVal = N2C->getZExtValue();
2609      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2610          (N1CVal & 0xffffU) == 0xffffU &&
2611          (N2CVal & 0xffffU) == 0x0U) {
2612        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2613                                                  MVT::i32);
2614        SDValue Ops[] = { N0.getOperand(0), Imm16,
2615                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2616        return CurDAG->getMachineNode(Opc, dl, VT, Ops);
2617      }
2618    }
2619    break;
2620  }
2621  case ARMISD::VMOVRRD:
2622    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2623                                  N->getOperand(0), getAL(CurDAG),
2624                                  CurDAG->getRegister(0, MVT::i32));
2625  case ISD::UMUL_LOHI: {
2626    if (Subtarget->isThumb1Only())
2627      break;
2628    if (Subtarget->isThumb()) {
2629      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2630                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2631      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops);
2632    } else {
2633      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2634                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2635                        CurDAG->getRegister(0, MVT::i32) };
2636      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2637                                    ARM::UMULL : ARM::UMULLv5,
2638                                    dl, MVT::i32, MVT::i32, Ops);
2639    }
2640  }
2641  case ISD::SMUL_LOHI: {
2642    if (Subtarget->isThumb1Only())
2643      break;
2644    if (Subtarget->isThumb()) {
2645      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2646                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2647      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops);
2648    } else {
2649      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2650                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2651                        CurDAG->getRegister(0, MVT::i32) };
2652      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2653                                    ARM::SMULL : ARM::SMULLv5,
2654                                    dl, MVT::i32, MVT::i32, Ops);
2655    }
2656  }
2657  case ARMISD::UMLAL:{
2658    if (Subtarget->isThumb()) {
2659      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2660                        N->getOperand(3), getAL(CurDAG),
2661                        CurDAG->getRegister(0, MVT::i32)};
2662      return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops);
2663    }else{
2664      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2665                        N->getOperand(3), getAL(CurDAG),
2666                        CurDAG->getRegister(0, MVT::i32),
2667                        CurDAG->getRegister(0, MVT::i32) };
2668      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2669                                      ARM::UMLAL : ARM::UMLALv5,
2670                                      dl, MVT::i32, MVT::i32, Ops);
2671    }
2672  }
2673  case ARMISD::SMLAL:{
2674    if (Subtarget->isThumb()) {
2675      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2676                        N->getOperand(3), getAL(CurDAG),
2677                        CurDAG->getRegister(0, MVT::i32)};
2678      return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops);
2679    }else{
2680      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2681                        N->getOperand(3), getAL(CurDAG),
2682                        CurDAG->getRegister(0, MVT::i32),
2683                        CurDAG->getRegister(0, MVT::i32) };
2684      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2685                                      ARM::SMLAL : ARM::SMLALv5,
2686                                      dl, MVT::i32, MVT::i32, Ops);
2687    }
2688  }
2689  case ISD::LOAD: {
2690    SDNode *ResNode = nullptr;
2691    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2692      ResNode = SelectT2IndexedLoad(N);
2693    else
2694      ResNode = SelectARMIndexedLoad(N);
2695    if (ResNode)
2696      return ResNode;
2697    // Other cases are autogenerated.
2698    break;
2699  }
2700  case ARMISD::BRCOND: {
2701    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2702    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2703    // Pattern complexity = 6  cost = 1  size = 0
2704
2705    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2706    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2707    // Pattern complexity = 6  cost = 1  size = 0
2708
2709    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2710    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2711    // Pattern complexity = 6  cost = 1  size = 0
2712
2713    unsigned Opc = Subtarget->isThumb() ?
2714      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2715    SDValue Chain = N->getOperand(0);
2716    SDValue N1 = N->getOperand(1);
2717    SDValue N2 = N->getOperand(2);
2718    SDValue N3 = N->getOperand(3);
2719    SDValue InFlag = N->getOperand(4);
2720    assert(N1.getOpcode() == ISD::BasicBlock);
2721    assert(N2.getOpcode() == ISD::Constant);
2722    assert(N3.getOpcode() == ISD::Register);
2723
2724    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2725                               cast<ConstantSDNode>(N2)->getZExtValue()),
2726                               MVT::i32);
2727    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2728    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2729                                             MVT::Glue, Ops);
2730    Chain = SDValue(ResNode, 0);
2731    if (N->getNumValues() == 2) {
2732      InFlag = SDValue(ResNode, 1);
2733      ReplaceUses(SDValue(N, 1), InFlag);
2734    }
2735    ReplaceUses(SDValue(N, 0),
2736                SDValue(Chain.getNode(), Chain.getResNo()));
2737    return nullptr;
2738  }
2739  case ARMISD::VZIP: {
2740    unsigned Opc = 0;
2741    EVT VT = N->getValueType(0);
2742    switch (VT.getSimpleVT().SimpleTy) {
2743    default: return nullptr;
2744    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2745    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2746    case MVT::v2f32:
2747    // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2748    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2749    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2750    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2751    case MVT::v4f32:
2752    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2753    }
2754    SDValue Pred = getAL(CurDAG);
2755    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2756    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2757    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2758  }
2759  case ARMISD::VUZP: {
2760    unsigned Opc = 0;
2761    EVT VT = N->getValueType(0);
2762    switch (VT.getSimpleVT().SimpleTy) {
2763    default: return nullptr;
2764    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2765    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2766    case MVT::v2f32:
2767    // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2768    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2769    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2770    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2771    case MVT::v4f32:
2772    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2773    }
2774    SDValue Pred = getAL(CurDAG);
2775    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2776    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2777    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2778  }
2779  case ARMISD::VTRN: {
2780    unsigned Opc = 0;
2781    EVT VT = N->getValueType(0);
2782    switch (VT.getSimpleVT().SimpleTy) {
2783    default: return nullptr;
2784    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2785    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2786    case MVT::v2f32:
2787    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2788    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2789    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2790    case MVT::v4f32:
2791    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2792    }
2793    SDValue Pred = getAL(CurDAG);
2794    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2795    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2796    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2797  }
2798  case ARMISD::BUILD_VECTOR: {
2799    EVT VecVT = N->getValueType(0);
2800    EVT EltVT = VecVT.getVectorElementType();
2801    unsigned NumElts = VecVT.getVectorNumElements();
2802    if (EltVT == MVT::f64) {
2803      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2804      return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1));
2805    }
2806    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2807    if (NumElts == 2)
2808      return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1));
2809    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2810    return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
2811                     N->getOperand(2), N->getOperand(3));
2812  }
2813
2814  case ARMISD::VLD2DUP: {
2815    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
2816                                        ARM::VLD2DUPd32 };
2817    return SelectVLDDup(N, false, 2, Opcodes);
2818  }
2819
2820  case ARMISD::VLD3DUP: {
2821    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
2822                                        ARM::VLD3DUPd16Pseudo,
2823                                        ARM::VLD3DUPd32Pseudo };
2824    return SelectVLDDup(N, false, 3, Opcodes);
2825  }
2826
2827  case ARMISD::VLD4DUP: {
2828    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
2829                                        ARM::VLD4DUPd16Pseudo,
2830                                        ARM::VLD4DUPd32Pseudo };
2831    return SelectVLDDup(N, false, 4, Opcodes);
2832  }
2833
2834  case ARMISD::VLD2DUP_UPD: {
2835    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
2836                                        ARM::VLD2DUPd16wb_fixed,
2837                                        ARM::VLD2DUPd32wb_fixed };
2838    return SelectVLDDup(N, true, 2, Opcodes);
2839  }
2840
2841  case ARMISD::VLD3DUP_UPD: {
2842    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
2843                                        ARM::VLD3DUPd16Pseudo_UPD,
2844                                        ARM::VLD3DUPd32Pseudo_UPD };
2845    return SelectVLDDup(N, true, 3, Opcodes);
2846  }
2847
2848  case ARMISD::VLD4DUP_UPD: {
2849    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
2850                                        ARM::VLD4DUPd16Pseudo_UPD,
2851                                        ARM::VLD4DUPd32Pseudo_UPD };
2852    return SelectVLDDup(N, true, 4, Opcodes);
2853  }
2854
2855  case ARMISD::VLD1_UPD: {
2856    static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
2857                                         ARM::VLD1d16wb_fixed,
2858                                         ARM::VLD1d32wb_fixed,
2859                                         ARM::VLD1d64wb_fixed };
2860    static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
2861                                         ARM::VLD1q16wb_fixed,
2862                                         ARM::VLD1q32wb_fixed,
2863                                         ARM::VLD1q64wb_fixed };
2864    return SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
2865  }
2866
2867  case ARMISD::VLD2_UPD: {
2868    static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
2869                                         ARM::VLD2d16wb_fixed,
2870                                         ARM::VLD2d32wb_fixed,
2871                                         ARM::VLD1q64wb_fixed};
2872    static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
2873                                         ARM::VLD2q16PseudoWB_fixed,
2874                                         ARM::VLD2q32PseudoWB_fixed };
2875    return SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
2876  }
2877
2878  case ARMISD::VLD3_UPD: {
2879    static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
2880                                         ARM::VLD3d16Pseudo_UPD,
2881                                         ARM::VLD3d32Pseudo_UPD,
2882                                         ARM::VLD1d64TPseudoWB_fixed};
2883    static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2884                                          ARM::VLD3q16Pseudo_UPD,
2885                                          ARM::VLD3q32Pseudo_UPD };
2886    static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2887                                          ARM::VLD3q16oddPseudo_UPD,
2888                                          ARM::VLD3q32oddPseudo_UPD };
2889    return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2890  }
2891
2892  case ARMISD::VLD4_UPD: {
2893    static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
2894                                         ARM::VLD4d16Pseudo_UPD,
2895                                         ARM::VLD4d32Pseudo_UPD,
2896                                         ARM::VLD1d64QPseudoWB_fixed};
2897    static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2898                                          ARM::VLD4q16Pseudo_UPD,
2899                                          ARM::VLD4q32Pseudo_UPD };
2900    static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2901                                          ARM::VLD4q16oddPseudo_UPD,
2902                                          ARM::VLD4q32oddPseudo_UPD };
2903    return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2904  }
2905
2906  case ARMISD::VLD2LN_UPD: {
2907    static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
2908                                         ARM::VLD2LNd16Pseudo_UPD,
2909                                         ARM::VLD2LNd32Pseudo_UPD };
2910    static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2911                                         ARM::VLD2LNq32Pseudo_UPD };
2912    return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2913  }
2914
2915  case ARMISD::VLD3LN_UPD: {
2916    static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
2917                                         ARM::VLD3LNd16Pseudo_UPD,
2918                                         ARM::VLD3LNd32Pseudo_UPD };
2919    static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2920                                         ARM::VLD3LNq32Pseudo_UPD };
2921    return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2922  }
2923
2924  case ARMISD::VLD4LN_UPD: {
2925    static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
2926                                         ARM::VLD4LNd16Pseudo_UPD,
2927                                         ARM::VLD4LNd32Pseudo_UPD };
2928    static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
2929                                         ARM::VLD4LNq32Pseudo_UPD };
2930    return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
2931  }
2932
2933  case ARMISD::VST1_UPD: {
2934    static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
2935                                         ARM::VST1d16wb_fixed,
2936                                         ARM::VST1d32wb_fixed,
2937                                         ARM::VST1d64wb_fixed };
2938    static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
2939                                         ARM::VST1q16wb_fixed,
2940                                         ARM::VST1q32wb_fixed,
2941                                         ARM::VST1q64wb_fixed };
2942    return SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
2943  }
2944
2945  case ARMISD::VST2_UPD: {
2946    static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
2947                                         ARM::VST2d16wb_fixed,
2948                                         ARM::VST2d32wb_fixed,
2949                                         ARM::VST1q64wb_fixed};
2950    static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
2951                                         ARM::VST2q16PseudoWB_fixed,
2952                                         ARM::VST2q32PseudoWB_fixed };
2953    return SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
2954  }
2955
2956  case ARMISD::VST3_UPD: {
2957    static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
2958                                         ARM::VST3d16Pseudo_UPD,
2959                                         ARM::VST3d32Pseudo_UPD,
2960                                         ARM::VST1d64TPseudoWB_fixed};
2961    static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2962                                          ARM::VST3q16Pseudo_UPD,
2963                                          ARM::VST3q32Pseudo_UPD };
2964    static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2965                                          ARM::VST3q16oddPseudo_UPD,
2966                                          ARM::VST3q32oddPseudo_UPD };
2967    return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2968  }
2969
2970  case ARMISD::VST4_UPD: {
2971    static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
2972                                         ARM::VST4d16Pseudo_UPD,
2973                                         ARM::VST4d32Pseudo_UPD,
2974                                         ARM::VST1d64QPseudoWB_fixed};
2975    static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2976                                          ARM::VST4q16Pseudo_UPD,
2977                                          ARM::VST4q32Pseudo_UPD };
2978    static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2979                                          ARM::VST4q16oddPseudo_UPD,
2980                                          ARM::VST4q32oddPseudo_UPD };
2981    return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2982  }
2983
2984  case ARMISD::VST2LN_UPD: {
2985    static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
2986                                         ARM::VST2LNd16Pseudo_UPD,
2987                                         ARM::VST2LNd32Pseudo_UPD };
2988    static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
2989                                         ARM::VST2LNq32Pseudo_UPD };
2990    return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
2991  }
2992
2993  case ARMISD::VST3LN_UPD: {
2994    static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
2995                                         ARM::VST3LNd16Pseudo_UPD,
2996                                         ARM::VST3LNd32Pseudo_UPD };
2997    static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
2998                                         ARM::VST3LNq32Pseudo_UPD };
2999    return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3000  }
3001
3002  case ARMISD::VST4LN_UPD: {
3003    static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
3004                                         ARM::VST4LNd16Pseudo_UPD,
3005                                         ARM::VST4LNd32Pseudo_UPD };
3006    static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3007                                         ARM::VST4LNq32Pseudo_UPD };
3008    return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3009  }
3010
3011  case ISD::INTRINSIC_VOID:
3012  case ISD::INTRINSIC_W_CHAIN: {
3013    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3014    switch (IntNo) {
3015    default:
3016      break;
3017
3018    case Intrinsic::arm_ldaexd:
3019    case Intrinsic::arm_ldrexd: {
3020      SDLoc dl(N);
3021      SDValue Chain = N->getOperand(0);
3022      SDValue MemAddr = N->getOperand(2);
3023      bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3024
3025      bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3026      unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
3027                                : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
3028
3029      // arm_ldrexd returns a i64 value in {i32, i32}
3030      std::vector<EVT> ResTys;
3031      if (isThumb) {
3032        ResTys.push_back(MVT::i32);
3033        ResTys.push_back(MVT::i32);
3034      } else
3035        ResTys.push_back(MVT::Untyped);
3036      ResTys.push_back(MVT::Other);
3037
3038      // Place arguments in the right order.
3039      SmallVector<SDValue, 7> Ops;
3040      Ops.push_back(MemAddr);
3041      Ops.push_back(getAL(CurDAG));
3042      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3043      Ops.push_back(Chain);
3044      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3045      // Transfer memoperands.
3046      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3047      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3048      cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3049
3050      // Remap uses.
3051      SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3052      if (!SDValue(N, 0).use_empty()) {
3053        SDValue Result;
3054        if (isThumb)
3055          Result = SDValue(Ld, 0);
3056        else {
3057          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
3058          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3059              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3060          Result = SDValue(ResNode,0);
3061        }
3062        ReplaceUses(SDValue(N, 0), Result);
3063      }
3064      if (!SDValue(N, 1).use_empty()) {
3065        SDValue Result;
3066        if (isThumb)
3067          Result = SDValue(Ld, 1);
3068        else {
3069          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
3070          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3071              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3072          Result = SDValue(ResNode,0);
3073        }
3074        ReplaceUses(SDValue(N, 1), Result);
3075      }
3076      ReplaceUses(SDValue(N, 2), OutChain);
3077      return nullptr;
3078    }
3079    case Intrinsic::arm_stlexd:
3080    case Intrinsic::arm_strexd: {
3081      SDLoc dl(N);
3082      SDValue Chain = N->getOperand(0);
3083      SDValue Val0 = N->getOperand(2);
3084      SDValue Val1 = N->getOperand(3);
3085      SDValue MemAddr = N->getOperand(4);
3086
3087      // Store exclusive double return a i32 value which is the return status
3088      // of the issued store.
3089      const EVT ResTys[] = {MVT::i32, MVT::Other};
3090
3091      bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3092      // Place arguments in the right order.
3093      SmallVector<SDValue, 7> Ops;
3094      if (isThumb) {
3095        Ops.push_back(Val0);
3096        Ops.push_back(Val1);
3097      } else
3098        // arm_strexd uses GPRPair.
3099        Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3100      Ops.push_back(MemAddr);
3101      Ops.push_back(getAL(CurDAG));
3102      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3103      Ops.push_back(Chain);
3104
3105      bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3106      unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
3107                                : (IsRelease ? ARM::STLEXD : ARM::STREXD);
3108
3109      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3110      // Transfer memoperands.
3111      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3112      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3113      cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3114
3115      return St;
3116    }
3117
3118    case Intrinsic::arm_neon_vld1: {
3119      static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3120                                           ARM::VLD1d32, ARM::VLD1d64 };
3121      static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3122                                           ARM::VLD1q32, ARM::VLD1q64};
3123      return SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3124    }
3125
3126    case Intrinsic::arm_neon_vld2: {
3127      static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3128                                           ARM::VLD2d32, ARM::VLD1q64 };
3129      static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3130                                           ARM::VLD2q32Pseudo };
3131      return SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3132    }
3133
3134    case Intrinsic::arm_neon_vld3: {
3135      static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3136                                           ARM::VLD3d16Pseudo,
3137                                           ARM::VLD3d32Pseudo,
3138                                           ARM::VLD1d64TPseudo };
3139      static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3140                                            ARM::VLD3q16Pseudo_UPD,
3141                                            ARM::VLD3q32Pseudo_UPD };
3142      static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3143                                            ARM::VLD3q16oddPseudo,
3144                                            ARM::VLD3q32oddPseudo };
3145      return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3146    }
3147
3148    case Intrinsic::arm_neon_vld4: {
3149      static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3150                                           ARM::VLD4d16Pseudo,
3151                                           ARM::VLD4d32Pseudo,
3152                                           ARM::VLD1d64QPseudo };
3153      static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3154                                            ARM::VLD4q16Pseudo_UPD,
3155                                            ARM::VLD4q32Pseudo_UPD };
3156      static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3157                                            ARM::VLD4q16oddPseudo,
3158                                            ARM::VLD4q32oddPseudo };
3159      return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3160    }
3161
3162    case Intrinsic::arm_neon_vld2lane: {
3163      static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3164                                           ARM::VLD2LNd16Pseudo,
3165                                           ARM::VLD2LNd32Pseudo };
3166      static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3167                                           ARM::VLD2LNq32Pseudo };
3168      return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3169    }
3170
3171    case Intrinsic::arm_neon_vld3lane: {
3172      static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3173                                           ARM::VLD3LNd16Pseudo,
3174                                           ARM::VLD3LNd32Pseudo };
3175      static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3176                                           ARM::VLD3LNq32Pseudo };
3177      return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3178    }
3179
3180    case Intrinsic::arm_neon_vld4lane: {
3181      static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3182                                           ARM::VLD4LNd16Pseudo,
3183                                           ARM::VLD4LNd32Pseudo };
3184      static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3185                                           ARM::VLD4LNq32Pseudo };
3186      return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3187    }
3188
3189    case Intrinsic::arm_neon_vst1: {
3190      static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3191                                           ARM::VST1d32, ARM::VST1d64 };
3192      static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3193                                           ARM::VST1q32, ARM::VST1q64 };
3194      return SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3195    }
3196
3197    case Intrinsic::arm_neon_vst2: {
3198      static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3199                                           ARM::VST2d32, ARM::VST1q64 };
3200      static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3201                                     ARM::VST2q32Pseudo };
3202      return SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3203    }
3204
3205    case Intrinsic::arm_neon_vst3: {
3206      static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3207                                           ARM::VST3d16Pseudo,
3208                                           ARM::VST3d32Pseudo,
3209                                           ARM::VST1d64TPseudo };
3210      static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3211                                            ARM::VST3q16Pseudo_UPD,
3212                                            ARM::VST3q32Pseudo_UPD };
3213      static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3214                                            ARM::VST3q16oddPseudo,
3215                                            ARM::VST3q32oddPseudo };
3216      return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3217    }
3218
3219    case Intrinsic::arm_neon_vst4: {
3220      static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3221                                           ARM::VST4d16Pseudo,
3222                                           ARM::VST4d32Pseudo,
3223                                           ARM::VST1d64QPseudo };
3224      static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3225                                            ARM::VST4q16Pseudo_UPD,
3226                                            ARM::VST4q32Pseudo_UPD };
3227      static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3228                                            ARM::VST4q16oddPseudo,
3229                                            ARM::VST4q32oddPseudo };
3230      return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3231    }
3232
3233    case Intrinsic::arm_neon_vst2lane: {
3234      static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3235                                           ARM::VST2LNd16Pseudo,
3236                                           ARM::VST2LNd32Pseudo };
3237      static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3238                                           ARM::VST2LNq32Pseudo };
3239      return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3240    }
3241
3242    case Intrinsic::arm_neon_vst3lane: {
3243      static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3244                                           ARM::VST3LNd16Pseudo,
3245                                           ARM::VST3LNd32Pseudo };
3246      static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3247                                           ARM::VST3LNq32Pseudo };
3248      return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3249    }
3250
3251    case Intrinsic::arm_neon_vst4lane: {
3252      static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3253                                           ARM::VST4LNd16Pseudo,
3254                                           ARM::VST4LNd32Pseudo };
3255      static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3256                                           ARM::VST4LNq32Pseudo };
3257      return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3258    }
3259    }
3260    break;
3261  }
3262
3263  case ISD::INTRINSIC_WO_CHAIN: {
3264    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3265    switch (IntNo) {
3266    default:
3267      break;
3268
3269    case Intrinsic::arm_neon_vtbl2:
3270      return SelectVTBL(N, false, 2, ARM::VTBL2);
3271    case Intrinsic::arm_neon_vtbl3:
3272      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
3273    case Intrinsic::arm_neon_vtbl4:
3274      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
3275
3276    case Intrinsic::arm_neon_vtbx2:
3277      return SelectVTBL(N, true, 2, ARM::VTBX2);
3278    case Intrinsic::arm_neon_vtbx3:
3279      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
3280    case Intrinsic::arm_neon_vtbx4:
3281      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
3282    }
3283    break;
3284  }
3285
3286  case ARMISD::VTBL1: {
3287    SDLoc dl(N);
3288    EVT VT = N->getValueType(0);
3289    SmallVector<SDValue, 6> Ops;
3290
3291    Ops.push_back(N->getOperand(0));
3292    Ops.push_back(N->getOperand(1));
3293    Ops.push_back(getAL(CurDAG));                    // Predicate
3294    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3295    return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops);
3296  }
3297  case ARMISD::VTBL2: {
3298    SDLoc dl(N);
3299    EVT VT = N->getValueType(0);
3300
3301    // Form a REG_SEQUENCE to force register allocation.
3302    SDValue V0 = N->getOperand(0);
3303    SDValue V1 = N->getOperand(1);
3304    SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0);
3305
3306    SmallVector<SDValue, 6> Ops;
3307    Ops.push_back(RegSeq);
3308    Ops.push_back(N->getOperand(2));
3309    Ops.push_back(getAL(CurDAG));                    // Predicate
3310    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3311    return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops);
3312  }
3313
3314  case ISD::CONCAT_VECTORS:
3315    return SelectConcatVector(N);
3316  }
3317
3318  return SelectCode(N);
3319}
3320
3321SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){
3322  std::vector<SDValue> AsmNodeOperands;
3323  unsigned Flag, Kind;
3324  bool Changed = false;
3325  unsigned NumOps = N->getNumOperands();
3326
3327  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
3328  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
3329  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
3330  // respectively. Since there is no constraint to explicitly specify a
3331  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
3332  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
3333  // them into a GPRPair.
3334
3335  SDLoc dl(N);
3336  SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
3337                                   : SDValue(nullptr,0);
3338
3339  SmallVector<bool, 8> OpChanged;
3340  // Glue node will be appended late.
3341  for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
3342    SDValue op = N->getOperand(i);
3343    AsmNodeOperands.push_back(op);
3344
3345    if (i < InlineAsm::Op_FirstOperand)
3346      continue;
3347
3348    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
3349      Flag = C->getZExtValue();
3350      Kind = InlineAsm::getKind(Flag);
3351    }
3352    else
3353      continue;
3354
3355    // Immediate operands to inline asm in the SelectionDAG are modeled with
3356    // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
3357    // the second is a constant with the value of the immediate. If we get here
3358    // and we have a Kind_Imm, skip the next operand, and continue.
3359    if (Kind == InlineAsm::Kind_Imm) {
3360      SDValue op = N->getOperand(++i);
3361      AsmNodeOperands.push_back(op);
3362      continue;
3363    }
3364
3365    unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
3366    if (NumRegs)
3367      OpChanged.push_back(false);
3368
3369    unsigned DefIdx = 0;
3370    bool IsTiedToChangedOp = false;
3371    // If it's a use that is tied with a previous def, it has no
3372    // reg class constraint.
3373    if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
3374      IsTiedToChangedOp = OpChanged[DefIdx];
3375
3376    if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
3377        && Kind != InlineAsm::Kind_RegDefEarlyClobber)
3378      continue;
3379
3380    unsigned RC;
3381    bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
3382    if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
3383        || NumRegs != 2)
3384      continue;
3385
3386    assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
3387    SDValue V0 = N->getOperand(i+1);
3388    SDValue V1 = N->getOperand(i+2);
3389    unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
3390    unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
3391    SDValue PairedReg;
3392    MachineRegisterInfo &MRI = MF->getRegInfo();
3393
3394    if (Kind == InlineAsm::Kind_RegDef ||
3395        Kind == InlineAsm::Kind_RegDefEarlyClobber) {
3396      // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
3397      // the original GPRs.
3398
3399      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
3400      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
3401      SDValue Chain = SDValue(N,0);
3402
3403      SDNode *GU = N->getGluedUser();
3404      SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
3405                                               Chain.getValue(1));
3406
3407      // Extract values from a GPRPair reg and copy to the original GPR reg.
3408      SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
3409                                                    RegCopy);
3410      SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
3411                                                    RegCopy);
3412      SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
3413                                        RegCopy.getValue(1));
3414      SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
3415
3416      // Update the original glue user.
3417      std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
3418      Ops.push_back(T1.getValue(1));
3419      CurDAG->UpdateNodeOperands(GU, Ops);
3420    }
3421    else {
3422      // For Kind  == InlineAsm::Kind_RegUse, we first copy two GPRs into a
3423      // GPRPair and then pass the GPRPair to the inline asm.
3424      SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
3425
3426      // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
3427      SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
3428                                          Chain.getValue(1));
3429      SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
3430                                          T0.getValue(1));
3431      SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
3432
3433      // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
3434      // i32 VRs of inline asm with it.
3435      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
3436      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
3437      Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
3438
3439      AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
3440      Glue = Chain.getValue(1);
3441    }
3442
3443    Changed = true;
3444
3445    if(PairedReg.getNode()) {
3446      OpChanged[OpChanged.size() -1 ] = true;
3447      Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
3448      if (IsTiedToChangedOp)
3449        Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
3450      else
3451        Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
3452      // Replace the current flag.
3453      AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
3454          Flag, MVT::i32);
3455      // Add the new register node and skip the original two GPRs.
3456      AsmNodeOperands.push_back(PairedReg);
3457      // Skip the next two GPRs.
3458      i += 2;
3459    }
3460  }
3461
3462  if (Glue.getNode())
3463    AsmNodeOperands.push_back(Glue);
3464  if (!Changed)
3465    return nullptr;
3466
3467  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
3468      CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
3469  New->setNodeId(-1);
3470  return New.getNode();
3471}
3472
3473
3474bool ARMDAGToDAGISel::
3475SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
3476                             std::vector<SDValue> &OutOps) {
3477  assert(ConstraintID == InlineAsm::Constraint_m &&
3478         "unexpected asm memory constraint");
3479  // Require the address to be in a register.  That is safe for all ARM
3480  // variants and it is hard to do anything much smarter without knowing
3481  // how the operand is used.
3482  OutOps.push_back(Op);
3483  return false;
3484}
3485
3486/// createARMISelDag - This pass converts a legalized DAG into a
3487/// ARM-specific DAG, ready for instruction scheduling.
3488///
3489FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3490                                     CodeGenOpt::Level OptLevel) {
3491  return new ARMDAGToDAGISel(TM, OptLevel);
3492}
3493