ARMISelDAGToDAG.cpp revision 2769028ab4732ce6083a71c67dc19b8901f55d5a 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMBaseInstrInfo.h"
17#include "ARMTargetMachine.h"
18#include "MCTargetDesc/ARMAddressingModes.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
37
38using namespace llvm;
39
40static cl::opt<bool>
41DisableShifterOp("disable-shifter-op", cl::Hidden,
42  cl::desc("Disable isel of shifter-op"),
43  cl::init(false));
44
45static cl::opt<bool>
46CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
47  cl::desc("Check fp vmla / vmls hazard at isel time"),
48  cl::init(true));
49
50static cl::opt<bool>
51DisableARMIntABS("disable-arm-int-abs", cl::Hidden,
52  cl::desc("Enable / disable ARM integer abs transform"),
53  cl::init(false));
54
55//===--------------------------------------------------------------------===//
56/// ARMDAGToDAGISel - ARM specific code to select ARM machine
57/// instructions for SelectionDAG operations.
58///
59namespace {
60
61enum AddrMode2Type {
62  AM2_BASE, // Simple AM2 (+-imm12)
63  AM2_SHOP  // Shifter-op AM2
64};
65
66class ARMDAGToDAGISel : public SelectionDAGISel {
67  ARMBaseTargetMachine &TM;
68  const ARMBaseInstrInfo *TII;
69
70  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
71  /// make the right decision when generating code for different targets.
72  const ARMSubtarget *Subtarget;
73
74public:
75  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
76                           CodeGenOpt::Level OptLevel)
77    : SelectionDAGISel(tm, OptLevel), TM(tm),
78      TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())),
79      Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
80  }
81
82  virtual const char *getPassName() const {
83    return "ARM Instruction Selection";
84  }
85
86  /// getI32Imm - Return a target constant of type i32 with the specified
87  /// value.
88  inline SDValue getI32Imm(unsigned Imm) {
89    return CurDAG->getTargetConstant(Imm, MVT::i32);
90  }
91
92  SDNode *Select(SDNode *N);
93
94
95  bool hasNoVMLxHazardUse(SDNode *N) const;
96  bool isShifterOpProfitable(const SDValue &Shift,
97                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
98  bool SelectRegShifterOperand(SDValue N, SDValue &A,
99                               SDValue &B, SDValue &C,
100                               bool CheckProfitability = true);
101  bool SelectImmShifterOperand(SDValue N, SDValue &A,
102                               SDValue &B, bool CheckProfitability = true);
103  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
104                                    SDValue &B, SDValue &C) {
105    // Don't apply the profitability check
106    return SelectRegShifterOperand(N, A, B, C, false);
107  }
108  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
109                                    SDValue &B) {
110    // Don't apply the profitability check
111    return SelectImmShifterOperand(N, A, B, false);
112  }
113
114  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
115  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
116
117  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
118                                      SDValue &Offset, SDValue &Opc);
119  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
120                           SDValue &Opc) {
121    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
122  }
123
124  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
125                           SDValue &Opc) {
126    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
127  }
128
129  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
130                       SDValue &Opc) {
131    SelectAddrMode2Worker(N, Base, Offset, Opc);
132//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
133    // This always matches one way or another.
134    return true;
135  }
136
137  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
138                             SDValue &Offset, SDValue &Opc);
139  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
140                             SDValue &Offset, SDValue &Opc);
141  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
142                             SDValue &Offset, SDValue &Opc);
143  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
144  bool SelectAddrMode3(SDValue N, SDValue &Base,
145                       SDValue &Offset, SDValue &Opc);
146  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
147                             SDValue &Offset, SDValue &Opc);
148  bool SelectAddrMode5(SDValue N, SDValue &Base,
149                       SDValue &Offset);
150  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
151  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
152
153  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
154
155  // Thumb Addressing Modes:
156  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
157  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
158                             unsigned Scale);
159  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
160  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
161  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
162  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
163                                SDValue &OffImm);
164  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
165                                 SDValue &OffImm);
166  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
167                                 SDValue &OffImm);
168  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
169                                 SDValue &OffImm);
170  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
171
172  // Thumb 2 Addressing Modes:
173  bool SelectT2ShifterOperandReg(SDValue N,
174                                 SDValue &BaseReg, SDValue &Opc);
175  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
176  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
177                            SDValue &OffImm);
178  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
179                                 SDValue &OffImm);
180  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
181                             SDValue &OffReg, SDValue &ShImm);
182
183  inline bool is_so_imm(unsigned Imm) const {
184    return ARM_AM::getSOImmVal(Imm) != -1;
185  }
186
187  inline bool is_so_imm_not(unsigned Imm) const {
188    return ARM_AM::getSOImmVal(~Imm) != -1;
189  }
190
191  inline bool is_t2_so_imm(unsigned Imm) const {
192    return ARM_AM::getT2SOImmVal(Imm) != -1;
193  }
194
195  inline bool is_t2_so_imm_not(unsigned Imm) const {
196    return ARM_AM::getT2SOImmVal(~Imm) != -1;
197  }
198
199  // Include the pieces autogenerated from the target description.
200#include "ARMGenDAGISel.inc"
201
202private:
203  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
204  /// ARM.
205  SDNode *SelectARMIndexedLoad(SDNode *N);
206  SDNode *SelectT2IndexedLoad(SDNode *N);
207
208  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
209  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
210  /// loads of D registers and even subregs and odd subregs of Q registers.
211  /// For NumVecs <= 2, QOpcodes1 is not used.
212  SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
213                    const uint16_t *DOpcodes,
214                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
215
216  /// SelectVST - Select NEON store intrinsics.  NumVecs should
217  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
218  /// stores of D registers and even subregs and odd subregs of Q registers.
219  /// For NumVecs <= 2, QOpcodes1 is not used.
220  SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
221                    const uint16_t *DOpcodes,
222                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
223
224  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
225  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
226  /// load/store of D registers and Q registers.
227  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
228                          bool isUpdating, unsigned NumVecs,
229                          const uint16_t *DOpcodes, const uint16_t *QOpcodes);
230
231  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
232  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
233  /// for loading D registers.  (Q registers are not supported.)
234  SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
235                       const uint16_t *Opcodes);
236
237  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
238  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
239  /// generated to force the table registers to be consecutive.
240  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
241
242  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
243  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
244
245  /// SelectCMOVOp - Select CMOV instructions for ARM.
246  SDNode *SelectCMOVOp(SDNode *N);
247  SDNode *SelectConditionalOp(SDNode *N);
248  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
249                              ARMCC::CondCodes CCVal, SDValue CCR,
250                              SDValue InFlag);
251  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
252                               ARMCC::CondCodes CCVal, SDValue CCR,
253                               SDValue InFlag);
254  SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
255                              ARMCC::CondCodes CCVal, SDValue CCR,
256                              SDValue InFlag);
257  SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
258                               ARMCC::CondCodes CCVal, SDValue CCR,
259                               SDValue InFlag);
260
261  // Select special operations if node forms integer ABS pattern
262  SDNode *SelectABSOp(SDNode *N);
263
264  SDNode *SelectConcatVector(SDNode *N);
265
266  SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
267
268  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
269  /// inline asm expressions.
270  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
271                                            char ConstraintCode,
272                                            std::vector<SDValue> &OutOps);
273
274  // Form pairs of consecutive S, D, or Q registers.
275  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
276  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
277  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
278
279  // Form sequences of 4 consecutive S, D, or Q registers.
280  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
281  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
282  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
283
284  // Get the alignment operand for a NEON VLD or VST instruction.
285  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
286};
287}
288
289/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
290/// operand. If so Imm will receive the 32-bit value.
291static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
292  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
293    Imm = cast<ConstantSDNode>(N)->getZExtValue();
294    return true;
295  }
296  return false;
297}
298
299// isInt32Immediate - This method tests to see if a constant operand.
300// If so Imm will receive the 32 bit value.
301static bool isInt32Immediate(SDValue N, unsigned &Imm) {
302  return isInt32Immediate(N.getNode(), Imm);
303}
304
305// isOpcWithIntImmediate - This method tests to see if the node is a specific
306// opcode and that it has a immediate integer right operand.
307// If so Imm will receive the 32 bit value.
308static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
309  return N->getOpcode() == Opc &&
310         isInt32Immediate(N->getOperand(1).getNode(), Imm);
311}
312
313/// \brief Check whether a particular node is a constant value representable as
314/// (N * Scale) where (N in [\arg RangeMin, \arg RangeMax).
315///
316/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
317static bool isScaledConstantInRange(SDValue Node, int Scale,
318                                    int RangeMin, int RangeMax,
319                                    int &ScaledConstant) {
320  assert(Scale > 0 && "Invalid scale!");
321
322  // Check that this is a constant.
323  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
324  if (!C)
325    return false;
326
327  ScaledConstant = (int) C->getZExtValue();
328  if ((ScaledConstant % Scale) != 0)
329    return false;
330
331  ScaledConstant /= Scale;
332  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
333}
334
335/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
336/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
337/// least on current ARM implementations) which should be avoidded.
338bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
339  if (OptLevel == CodeGenOpt::None)
340    return true;
341
342  if (!CheckVMLxHazard)
343    return true;
344
345  if (!Subtarget->isCortexA8() && !Subtarget->isCortexA9())
346    return true;
347
348  if (!N->hasOneUse())
349    return false;
350
351  SDNode *Use = *N->use_begin();
352  if (Use->getOpcode() == ISD::CopyToReg)
353    return true;
354  if (Use->isMachineOpcode()) {
355    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
356    if (MCID.mayStore())
357      return true;
358    unsigned Opcode = MCID.getOpcode();
359    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
360      return true;
361    // vmlx feeding into another vmlx. We actually want to unfold
362    // the use later in the MLxExpansion pass. e.g.
363    // vmla
364    // vmla (stall 8 cycles)
365    //
366    // vmul (5 cycles)
367    // vadd (5 cycles)
368    // vmla
369    // This adds up to about 18 - 19 cycles.
370    //
371    // vmla
372    // vmul (stall 4 cycles)
373    // vadd adds up to about 14 cycles.
374    return TII->isFpMLxInstruction(Opcode);
375  }
376
377  return false;
378}
379
380bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
381                                            ARM_AM::ShiftOpc ShOpcVal,
382                                            unsigned ShAmt) {
383  if (!Subtarget->isCortexA9())
384    return true;
385  if (Shift.hasOneUse())
386    return true;
387  // R << 2 is free.
388  return ShOpcVal == ARM_AM::lsl && ShAmt == 2;
389}
390
391bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
392                                              SDValue &BaseReg,
393                                              SDValue &Opc,
394                                              bool CheckProfitability) {
395  if (DisableShifterOp)
396    return false;
397
398  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
399
400  // Don't match base register only case. That is matched to a separate
401  // lower complexity pattern with explicit register operand.
402  if (ShOpcVal == ARM_AM::no_shift) return false;
403
404  BaseReg = N.getOperand(0);
405  unsigned ShImmVal = 0;
406  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
407  if (!RHS) return false;
408  ShImmVal = RHS->getZExtValue() & 31;
409  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
410                                  MVT::i32);
411  return true;
412}
413
414bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
415                                              SDValue &BaseReg,
416                                              SDValue &ShReg,
417                                              SDValue &Opc,
418                                              bool CheckProfitability) {
419  if (DisableShifterOp)
420    return false;
421
422  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
423
424  // Don't match base register only case. That is matched to a separate
425  // lower complexity pattern with explicit register operand.
426  if (ShOpcVal == ARM_AM::no_shift) return false;
427
428  BaseReg = N.getOperand(0);
429  unsigned ShImmVal = 0;
430  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
431  if (RHS) return false;
432
433  ShReg = N.getOperand(1);
434  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
435    return false;
436  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
437                                  MVT::i32);
438  return true;
439}
440
441
442bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
443                                          SDValue &Base,
444                                          SDValue &OffImm) {
445  // Match simple R + imm12 operands.
446
447  // Base only.
448  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
449      !CurDAG->isBaseWithConstantOffset(N)) {
450    if (N.getOpcode() == ISD::FrameIndex) {
451      // Match frame index.
452      int FI = cast<FrameIndexSDNode>(N)->getIndex();
453      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
454      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
455      return true;
456    }
457
458    if (N.getOpcode() == ARMISD::Wrapper &&
459        !(Subtarget->useMovt() &&
460                     N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
461      Base = N.getOperand(0);
462    } else
463      Base = N;
464    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
465    return true;
466  }
467
468  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
469    int RHSC = (int)RHS->getZExtValue();
470    if (N.getOpcode() == ISD::SUB)
471      RHSC = -RHSC;
472
473    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
474      Base   = N.getOperand(0);
475      if (Base.getOpcode() == ISD::FrameIndex) {
476        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
477        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
478      }
479      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
480      return true;
481    }
482  }
483
484  // Base only.
485  Base = N;
486  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
487  return true;
488}
489
490
491
492bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
493                                      SDValue &Opc) {
494  if (N.getOpcode() == ISD::MUL &&
495      (!Subtarget->isCortexA9() || N.hasOneUse())) {
496    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
497      // X * [3,5,9] -> X + X * [2,4,8] etc.
498      int RHSC = (int)RHS->getZExtValue();
499      if (RHSC & 1) {
500        RHSC = RHSC & ~1;
501        ARM_AM::AddrOpc AddSub = ARM_AM::add;
502        if (RHSC < 0) {
503          AddSub = ARM_AM::sub;
504          RHSC = - RHSC;
505        }
506        if (isPowerOf2_32(RHSC)) {
507          unsigned ShAmt = Log2_32(RHSC);
508          Base = Offset = N.getOperand(0);
509          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
510                                                            ARM_AM::lsl),
511                                          MVT::i32);
512          return true;
513        }
514      }
515    }
516  }
517
518  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
519      // ISD::OR that is equivalent to an ISD::ADD.
520      !CurDAG->isBaseWithConstantOffset(N))
521    return false;
522
523  // Leave simple R +/- imm12 operands for LDRi12
524  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
525    int RHSC;
526    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
527                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
528      return false;
529  }
530
531  // Otherwise this is R +/- [possibly shifted] R.
532  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
533  ARM_AM::ShiftOpc ShOpcVal =
534    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
535  unsigned ShAmt = 0;
536
537  Base   = N.getOperand(0);
538  Offset = N.getOperand(1);
539
540  if (ShOpcVal != ARM_AM::no_shift) {
541    // Check to see if the RHS of the shift is a constant, if not, we can't fold
542    // it.
543    if (ConstantSDNode *Sh =
544           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
545      ShAmt = Sh->getZExtValue();
546      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
547        Offset = N.getOperand(1).getOperand(0);
548      else {
549        ShAmt = 0;
550        ShOpcVal = ARM_AM::no_shift;
551      }
552    } else {
553      ShOpcVal = ARM_AM::no_shift;
554    }
555  }
556
557  // Try matching (R shl C) + (R).
558  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
559      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
560    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
561    if (ShOpcVal != ARM_AM::no_shift) {
562      // Check to see if the RHS of the shift is a constant, if not, we can't
563      // fold it.
564      if (ConstantSDNode *Sh =
565          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
566        ShAmt = Sh->getZExtValue();
567        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
568          Offset = N.getOperand(0).getOperand(0);
569          Base = N.getOperand(1);
570        } else {
571          ShAmt = 0;
572          ShOpcVal = ARM_AM::no_shift;
573        }
574      } else {
575        ShOpcVal = ARM_AM::no_shift;
576      }
577    }
578  }
579
580  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
581                                  MVT::i32);
582  return true;
583}
584
585
586//-----
587
588AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
589                                                     SDValue &Base,
590                                                     SDValue &Offset,
591                                                     SDValue &Opc) {
592  if (N.getOpcode() == ISD::MUL &&
593      (!Subtarget->isCortexA9() || N.hasOneUse())) {
594    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
595      // X * [3,5,9] -> X + X * [2,4,8] etc.
596      int RHSC = (int)RHS->getZExtValue();
597      if (RHSC & 1) {
598        RHSC = RHSC & ~1;
599        ARM_AM::AddrOpc AddSub = ARM_AM::add;
600        if (RHSC < 0) {
601          AddSub = ARM_AM::sub;
602          RHSC = - RHSC;
603        }
604        if (isPowerOf2_32(RHSC)) {
605          unsigned ShAmt = Log2_32(RHSC);
606          Base = Offset = N.getOperand(0);
607          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
608                                                            ARM_AM::lsl),
609                                          MVT::i32);
610          return AM2_SHOP;
611        }
612      }
613    }
614  }
615
616  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
617      // ISD::OR that is equivalent to an ADD.
618      !CurDAG->isBaseWithConstantOffset(N)) {
619    Base = N;
620    if (N.getOpcode() == ISD::FrameIndex) {
621      int FI = cast<FrameIndexSDNode>(N)->getIndex();
622      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
623    } else if (N.getOpcode() == ARMISD::Wrapper &&
624               !(Subtarget->useMovt() &&
625                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
626      Base = N.getOperand(0);
627    }
628    Offset = CurDAG->getRegister(0, MVT::i32);
629    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
630                                                      ARM_AM::no_shift),
631                                    MVT::i32);
632    return AM2_BASE;
633  }
634
635  // Match simple R +/- imm12 operands.
636  if (N.getOpcode() != ISD::SUB) {
637    int RHSC;
638    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
639                                -0x1000+1, 0x1000, RHSC)) { // 12 bits.
640      Base = N.getOperand(0);
641      if (Base.getOpcode() == ISD::FrameIndex) {
642        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
643        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
644      }
645      Offset = CurDAG->getRegister(0, MVT::i32);
646
647      ARM_AM::AddrOpc AddSub = ARM_AM::add;
648      if (RHSC < 0) {
649        AddSub = ARM_AM::sub;
650        RHSC = - RHSC;
651      }
652      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
653                                                        ARM_AM::no_shift),
654                                      MVT::i32);
655      return AM2_BASE;
656    }
657  }
658
659  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
660    // Compute R +/- (R << N) and reuse it.
661    Base = N;
662    Offset = CurDAG->getRegister(0, MVT::i32);
663    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
664                                                      ARM_AM::no_shift),
665                                    MVT::i32);
666    return AM2_BASE;
667  }
668
669  // Otherwise this is R +/- [possibly shifted] R.
670  ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
671  ARM_AM::ShiftOpc ShOpcVal =
672    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
673  unsigned ShAmt = 0;
674
675  Base   = N.getOperand(0);
676  Offset = N.getOperand(1);
677
678  if (ShOpcVal != ARM_AM::no_shift) {
679    // Check to see if the RHS of the shift is a constant, if not, we can't fold
680    // it.
681    if (ConstantSDNode *Sh =
682           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
683      ShAmt = Sh->getZExtValue();
684      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
685        Offset = N.getOperand(1).getOperand(0);
686      else {
687        ShAmt = 0;
688        ShOpcVal = ARM_AM::no_shift;
689      }
690    } else {
691      ShOpcVal = ARM_AM::no_shift;
692    }
693  }
694
695  // Try matching (R shl C) + (R).
696  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
697      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
698    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
699    if (ShOpcVal != ARM_AM::no_shift) {
700      // Check to see if the RHS of the shift is a constant, if not, we can't
701      // fold it.
702      if (ConstantSDNode *Sh =
703          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
704        ShAmt = Sh->getZExtValue();
705        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
706          Offset = N.getOperand(0).getOperand(0);
707          Base = N.getOperand(1);
708        } else {
709          ShAmt = 0;
710          ShOpcVal = ARM_AM::no_shift;
711        }
712      } else {
713        ShOpcVal = ARM_AM::no_shift;
714      }
715    }
716  }
717
718  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
719                                  MVT::i32);
720  return AM2_SHOP;
721}
722
723bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
724                                            SDValue &Offset, SDValue &Opc) {
725  unsigned Opcode = Op->getOpcode();
726  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
727    ? cast<LoadSDNode>(Op)->getAddressingMode()
728    : cast<StoreSDNode>(Op)->getAddressingMode();
729  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
730    ? ARM_AM::add : ARM_AM::sub;
731  int Val;
732  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
733    return false;
734
735  Offset = N;
736  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
737  unsigned ShAmt = 0;
738  if (ShOpcVal != ARM_AM::no_shift) {
739    // Check to see if the RHS of the shift is a constant, if not, we can't fold
740    // it.
741    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
742      ShAmt = Sh->getZExtValue();
743      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
744        Offset = N.getOperand(0);
745      else {
746        ShAmt = 0;
747        ShOpcVal = ARM_AM::no_shift;
748      }
749    } else {
750      ShOpcVal = ARM_AM::no_shift;
751    }
752  }
753
754  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
755                                  MVT::i32);
756  return true;
757}
758
759bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
760                                            SDValue &Offset, SDValue &Opc) {
761  unsigned Opcode = Op->getOpcode();
762  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
763    ? cast<LoadSDNode>(Op)->getAddressingMode()
764    : cast<StoreSDNode>(Op)->getAddressingMode();
765  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
766    ? ARM_AM::add : ARM_AM::sub;
767  int Val;
768  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
769    if (AddSub == ARM_AM::sub) Val *= -1;
770    Offset = CurDAG->getRegister(0, MVT::i32);
771    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
772    return true;
773  }
774
775  return false;
776}
777
778
779bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
780                                            SDValue &Offset, SDValue &Opc) {
781  unsigned Opcode = Op->getOpcode();
782  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
783    ? cast<LoadSDNode>(Op)->getAddressingMode()
784    : cast<StoreSDNode>(Op)->getAddressingMode();
785  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
786    ? ARM_AM::add : ARM_AM::sub;
787  int Val;
788  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
789    Offset = CurDAG->getRegister(0, MVT::i32);
790    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
791                                                      ARM_AM::no_shift),
792                                    MVT::i32);
793    return true;
794  }
795
796  return false;
797}
798
799bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
800  Base = N;
801  return true;
802}
803
804bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
805                                      SDValue &Base, SDValue &Offset,
806                                      SDValue &Opc) {
807  if (N.getOpcode() == ISD::SUB) {
808    // X - C  is canonicalize to X + -C, no need to handle it here.
809    Base = N.getOperand(0);
810    Offset = N.getOperand(1);
811    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
812    return true;
813  }
814
815  if (!CurDAG->isBaseWithConstantOffset(N)) {
816    Base = N;
817    if (N.getOpcode() == ISD::FrameIndex) {
818      int FI = cast<FrameIndexSDNode>(N)->getIndex();
819      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
820    }
821    Offset = CurDAG->getRegister(0, MVT::i32);
822    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
823    return true;
824  }
825
826  // If the RHS is +/- imm8, fold into addr mode.
827  int RHSC;
828  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
829                              -256 + 1, 256, RHSC)) { // 8 bits.
830    Base = N.getOperand(0);
831    if (Base.getOpcode() == ISD::FrameIndex) {
832      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
833      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
834    }
835    Offset = CurDAG->getRegister(0, MVT::i32);
836
837    ARM_AM::AddrOpc AddSub = ARM_AM::add;
838    if (RHSC < 0) {
839      AddSub = ARM_AM::sub;
840      RHSC = -RHSC;
841    }
842    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
843    return true;
844  }
845
846  Base = N.getOperand(0);
847  Offset = N.getOperand(1);
848  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
849  return true;
850}
851
852bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
853                                            SDValue &Offset, SDValue &Opc) {
854  unsigned Opcode = Op->getOpcode();
855  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
856    ? cast<LoadSDNode>(Op)->getAddressingMode()
857    : cast<StoreSDNode>(Op)->getAddressingMode();
858  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
859    ? ARM_AM::add : ARM_AM::sub;
860  int Val;
861  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
862    Offset = CurDAG->getRegister(0, MVT::i32);
863    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
864    return true;
865  }
866
867  Offset = N;
868  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
869  return true;
870}
871
872bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
873                                      SDValue &Base, SDValue &Offset) {
874  if (!CurDAG->isBaseWithConstantOffset(N)) {
875    Base = N;
876    if (N.getOpcode() == ISD::FrameIndex) {
877      int FI = cast<FrameIndexSDNode>(N)->getIndex();
878      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
879    } else if (N.getOpcode() == ARMISD::Wrapper &&
880               !(Subtarget->useMovt() &&
881                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
882      Base = N.getOperand(0);
883    }
884    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
885                                       MVT::i32);
886    return true;
887  }
888
889  // If the RHS is +/- imm8, fold into addr mode.
890  int RHSC;
891  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
892                              -256 + 1, 256, RHSC)) {
893    Base = N.getOperand(0);
894    if (Base.getOpcode() == ISD::FrameIndex) {
895      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
896      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
897    }
898
899    ARM_AM::AddrOpc AddSub = ARM_AM::add;
900    if (RHSC < 0) {
901      AddSub = ARM_AM::sub;
902      RHSC = -RHSC;
903    }
904    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
905                                       MVT::i32);
906    return true;
907  }
908
909  Base = N;
910  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
911                                     MVT::i32);
912  return true;
913}
914
915bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
916                                      SDValue &Align) {
917  Addr = N;
918
919  unsigned Alignment = 0;
920  if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
921    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
922    // The maximum alignment is equal to the memory size being referenced.
923    unsigned LSNAlign = LSN->getAlignment();
924    unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
925    if (LSNAlign >= MemSize && MemSize > 1)
926      Alignment = MemSize;
927  } else {
928    // All other uses of addrmode6 are for intrinsics.  For now just record
929    // the raw alignment value; it will be refined later based on the legal
930    // alignment operands for the intrinsic.
931    Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
932  }
933
934  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
935  return true;
936}
937
938bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
939                                            SDValue &Offset) {
940  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
941  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
942  if (AM != ISD::POST_INC)
943    return false;
944  Offset = N;
945  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
946    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
947      Offset = CurDAG->getRegister(0, MVT::i32);
948  }
949  return true;
950}
951
952bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
953                                       SDValue &Offset, SDValue &Label) {
954  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
955    Offset = N.getOperand(0);
956    SDValue N1 = N.getOperand(1);
957    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
958                                      MVT::i32);
959    return true;
960  }
961
962  return false;
963}
964
965
966//===----------------------------------------------------------------------===//
967//                         Thumb Addressing Modes
968//===----------------------------------------------------------------------===//
969
970bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
971                                            SDValue &Base, SDValue &Offset){
972  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
973    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
974    if (!NC || !NC->isNullValue())
975      return false;
976
977    Base = Offset = N;
978    return true;
979  }
980
981  Base = N.getOperand(0);
982  Offset = N.getOperand(1);
983  return true;
984}
985
986bool
987ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
988                                       SDValue &Offset, unsigned Scale) {
989  if (Scale == 4) {
990    SDValue TmpBase, TmpOffImm;
991    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
992      return false;  // We want to select tLDRspi / tSTRspi instead.
993
994    if (N.getOpcode() == ARMISD::Wrapper &&
995        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
996      return false;  // We want to select tLDRpci instead.
997  }
998
999  if (!CurDAG->isBaseWithConstantOffset(N))
1000    return false;
1001
1002  // Thumb does not have [sp, r] address mode.
1003  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1004  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1005  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1006      (RHSR && RHSR->getReg() == ARM::SP))
1007    return false;
1008
1009  // FIXME: Why do we explicitly check for a match here and then return false?
1010  // Presumably to allow something else to match, but shouldn't this be
1011  // documented?
1012  int RHSC;
1013  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
1014    return false;
1015
1016  Base = N.getOperand(0);
1017  Offset = N.getOperand(1);
1018  return true;
1019}
1020
1021bool
1022ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
1023                                          SDValue &Base,
1024                                          SDValue &Offset) {
1025  return SelectThumbAddrModeRI(N, Base, Offset, 1);
1026}
1027
1028bool
1029ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
1030                                          SDValue &Base,
1031                                          SDValue &Offset) {
1032  return SelectThumbAddrModeRI(N, Base, Offset, 2);
1033}
1034
1035bool
1036ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
1037                                          SDValue &Base,
1038                                          SDValue &Offset) {
1039  return SelectThumbAddrModeRI(N, Base, Offset, 4);
1040}
1041
1042bool
1043ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1044                                          SDValue &Base, SDValue &OffImm) {
1045  if (Scale == 4) {
1046    SDValue TmpBase, TmpOffImm;
1047    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1048      return false;  // We want to select tLDRspi / tSTRspi instead.
1049
1050    if (N.getOpcode() == ARMISD::Wrapper &&
1051        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1052      return false;  // We want to select tLDRpci instead.
1053  }
1054
1055  if (!CurDAG->isBaseWithConstantOffset(N)) {
1056    if (N.getOpcode() == ARMISD::Wrapper &&
1057        !(Subtarget->useMovt() &&
1058          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1059      Base = N.getOperand(0);
1060    } else {
1061      Base = N;
1062    }
1063
1064    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1065    return true;
1066  }
1067
1068  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1069  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1070  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1071      (RHSR && RHSR->getReg() == ARM::SP)) {
1072    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1073    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1074    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1075    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1076
1077    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1078    if (LHSC != 0 || RHSC != 0) return false;
1079
1080    Base = N;
1081    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1082    return true;
1083  }
1084
1085  // If the RHS is + imm5 * scale, fold into addr mode.
1086  int RHSC;
1087  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1088    Base = N.getOperand(0);
1089    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1090    return true;
1091  }
1092
1093  Base = N.getOperand(0);
1094  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1095  return true;
1096}
1097
1098bool
1099ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1100                                           SDValue &OffImm) {
1101  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1102}
1103
1104bool
1105ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1106                                           SDValue &OffImm) {
1107  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1108}
1109
1110bool
1111ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1112                                           SDValue &OffImm) {
1113  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1114}
1115
1116bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1117                                            SDValue &Base, SDValue &OffImm) {
1118  if (N.getOpcode() == ISD::FrameIndex) {
1119    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1120    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1121    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1122    return true;
1123  }
1124
1125  if (!CurDAG->isBaseWithConstantOffset(N))
1126    return false;
1127
1128  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1129  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1130      (LHSR && LHSR->getReg() == ARM::SP)) {
1131    // If the RHS is + imm8 * scale, fold into addr mode.
1132    int RHSC;
1133    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1134      Base = N.getOperand(0);
1135      if (Base.getOpcode() == ISD::FrameIndex) {
1136        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1137        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1138      }
1139      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1140      return true;
1141    }
1142  }
1143
1144  return false;
1145}
1146
1147
1148//===----------------------------------------------------------------------===//
1149//                        Thumb 2 Addressing Modes
1150//===----------------------------------------------------------------------===//
1151
1152
1153bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1154                                                SDValue &Opc) {
1155  if (DisableShifterOp)
1156    return false;
1157
1158  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1159
1160  // Don't match base register only case. That is matched to a separate
1161  // lower complexity pattern with explicit register operand.
1162  if (ShOpcVal == ARM_AM::no_shift) return false;
1163
1164  BaseReg = N.getOperand(0);
1165  unsigned ShImmVal = 0;
1166  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1167    ShImmVal = RHS->getZExtValue() & 31;
1168    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1169    return true;
1170  }
1171
1172  return false;
1173}
1174
1175bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1176                                            SDValue &Base, SDValue &OffImm) {
1177  // Match simple R + imm12 operands.
1178
1179  // Base only.
1180  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1181      !CurDAG->isBaseWithConstantOffset(N)) {
1182    if (N.getOpcode() == ISD::FrameIndex) {
1183      // Match frame index.
1184      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1185      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1186      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1187      return true;
1188    }
1189
1190    if (N.getOpcode() == ARMISD::Wrapper &&
1191               !(Subtarget->useMovt() &&
1192                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1193      Base = N.getOperand(0);
1194      if (Base.getOpcode() == ISD::TargetConstantPool)
1195        return false;  // We want to select t2LDRpci instead.
1196    } else
1197      Base = N;
1198    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1199    return true;
1200  }
1201
1202  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1203    if (SelectT2AddrModeImm8(N, Base, OffImm))
1204      // Let t2LDRi8 handle (R - imm8).
1205      return false;
1206
1207    int RHSC = (int)RHS->getZExtValue();
1208    if (N.getOpcode() == ISD::SUB)
1209      RHSC = -RHSC;
1210
1211    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1212      Base   = N.getOperand(0);
1213      if (Base.getOpcode() == ISD::FrameIndex) {
1214        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1215        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1216      }
1217      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1218      return true;
1219    }
1220  }
1221
1222  // Base only.
1223  Base = N;
1224  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1225  return true;
1226}
1227
1228bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1229                                           SDValue &Base, SDValue &OffImm) {
1230  // Match simple R - imm8 operands.
1231  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1232      !CurDAG->isBaseWithConstantOffset(N))
1233    return false;
1234
1235  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1236    int RHSC = (int)RHS->getSExtValue();
1237    if (N.getOpcode() == ISD::SUB)
1238      RHSC = -RHSC;
1239
1240    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1241      Base = N.getOperand(0);
1242      if (Base.getOpcode() == ISD::FrameIndex) {
1243        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1244        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1245      }
1246      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1247      return true;
1248    }
1249  }
1250
1251  return false;
1252}
1253
1254bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1255                                                 SDValue &OffImm){
1256  unsigned Opcode = Op->getOpcode();
1257  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1258    ? cast<LoadSDNode>(Op)->getAddressingMode()
1259    : cast<StoreSDNode>(Op)->getAddressingMode();
1260  int RHSC;
1261  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1262    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1263      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1264      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1265    return true;
1266  }
1267
1268  return false;
1269}
1270
1271bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1272                                            SDValue &Base,
1273                                            SDValue &OffReg, SDValue &ShImm) {
1274  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1275  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1276    return false;
1277
1278  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1279  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1280    int RHSC = (int)RHS->getZExtValue();
1281    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1282      return false;
1283    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1284      return false;
1285  }
1286
1287  // Look for (R + R) or (R + (R << [1,2,3])).
1288  unsigned ShAmt = 0;
1289  Base   = N.getOperand(0);
1290  OffReg = N.getOperand(1);
1291
1292  // Swap if it is ((R << c) + R).
1293  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1294  if (ShOpcVal != ARM_AM::lsl) {
1295    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1296    if (ShOpcVal == ARM_AM::lsl)
1297      std::swap(Base, OffReg);
1298  }
1299
1300  if (ShOpcVal == ARM_AM::lsl) {
1301    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1302    // it.
1303    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1304      ShAmt = Sh->getZExtValue();
1305      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1306        OffReg = OffReg.getOperand(0);
1307      else {
1308        ShAmt = 0;
1309        ShOpcVal = ARM_AM::no_shift;
1310      }
1311    } else {
1312      ShOpcVal = ARM_AM::no_shift;
1313    }
1314  }
1315
1316  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1317
1318  return true;
1319}
1320
1321//===--------------------------------------------------------------------===//
1322
1323/// getAL - Returns a ARMCC::AL immediate node.
1324static inline SDValue getAL(SelectionDAG *CurDAG) {
1325  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1326}
1327
1328SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1329  LoadSDNode *LD = cast<LoadSDNode>(N);
1330  ISD::MemIndexedMode AM = LD->getAddressingMode();
1331  if (AM == ISD::UNINDEXED)
1332    return NULL;
1333
1334  EVT LoadedVT = LD->getMemoryVT();
1335  SDValue Offset, AMOpc;
1336  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1337  unsigned Opcode = 0;
1338  bool Match = false;
1339  if (LoadedVT == MVT::i32 && isPre &&
1340      SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1341    Opcode = ARM::LDR_PRE_IMM;
1342    Match = true;
1343  } else if (LoadedVT == MVT::i32 && !isPre &&
1344      SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1345    Opcode = ARM::LDR_POST_IMM;
1346    Match = true;
1347  } else if (LoadedVT == MVT::i32 &&
1348      SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1349    Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1350    Match = true;
1351
1352  } else if (LoadedVT == MVT::i16 &&
1353             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1354    Match = true;
1355    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1356      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1357      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1358  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1359    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1360      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1361        Match = true;
1362        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1363      }
1364    } else {
1365      if (isPre &&
1366          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1367        Match = true;
1368        Opcode = ARM::LDRB_PRE_IMM;
1369      } else if (!isPre &&
1370                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1371        Match = true;
1372        Opcode = ARM::LDRB_POST_IMM;
1373      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1374        Match = true;
1375        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1376      }
1377    }
1378  }
1379
1380  if (Match) {
1381    if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1382      SDValue Chain = LD->getChain();
1383      SDValue Base = LD->getBasePtr();
1384      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
1385                       CurDAG->getRegister(0, MVT::i32), Chain };
1386      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1387                                    MVT::i32, MVT::Other, Ops, 5);
1388    } else {
1389      SDValue Chain = LD->getChain();
1390      SDValue Base = LD->getBasePtr();
1391      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1392                       CurDAG->getRegister(0, MVT::i32), Chain };
1393      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1394                                    MVT::i32, MVT::Other, Ops, 6);
1395    }
1396  }
1397
1398  return NULL;
1399}
1400
1401SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1402  LoadSDNode *LD = cast<LoadSDNode>(N);
1403  ISD::MemIndexedMode AM = LD->getAddressingMode();
1404  if (AM == ISD::UNINDEXED)
1405    return NULL;
1406
1407  EVT LoadedVT = LD->getMemoryVT();
1408  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1409  SDValue Offset;
1410  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1411  unsigned Opcode = 0;
1412  bool Match = false;
1413  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1414    switch (LoadedVT.getSimpleVT().SimpleTy) {
1415    case MVT::i32:
1416      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1417      break;
1418    case MVT::i16:
1419      if (isSExtLd)
1420        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1421      else
1422        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1423      break;
1424    case MVT::i8:
1425    case MVT::i1:
1426      if (isSExtLd)
1427        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1428      else
1429        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1430      break;
1431    default:
1432      return NULL;
1433    }
1434    Match = true;
1435  }
1436
1437  if (Match) {
1438    SDValue Chain = LD->getChain();
1439    SDValue Base = LD->getBasePtr();
1440    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1441                     CurDAG->getRegister(0, MVT::i32), Chain };
1442    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1443                                  MVT::Other, Ops, 5);
1444  }
1445
1446  return NULL;
1447}
1448
1449/// PairSRegs - Form a D register from a pair of S registers.
1450///
1451SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
1452  DebugLoc dl = V0.getNode()->getDebugLoc();
1453  SDValue RegClass =
1454    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1455  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1456  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1457  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1458  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1459}
1460
1461/// PairDRegs - Form a quad register from a pair of D registers.
1462///
1463SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
1464  DebugLoc dl = V0.getNode()->getDebugLoc();
1465  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1466  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1467  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1468  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1469  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1470}
1471
1472/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
1473///
1474SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
1475  DebugLoc dl = V0.getNode()->getDebugLoc();
1476  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1477  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1478  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1479  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1480  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1481}
1482
1483/// QuadSRegs - Form 4 consecutive S registers.
1484///
1485SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
1486                                   SDValue V2, SDValue V3) {
1487  DebugLoc dl = V0.getNode()->getDebugLoc();
1488  SDValue RegClass =
1489    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1490  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1491  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1492  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1493  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1494  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1495                                    V2, SubReg2, V3, SubReg3 };
1496  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1497}
1498
1499/// QuadDRegs - Form 4 consecutive D registers.
1500///
1501SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
1502                                   SDValue V2, SDValue V3) {
1503  DebugLoc dl = V0.getNode()->getDebugLoc();
1504  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1505  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1506  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1507  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1508  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1509  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1510                                    V2, SubReg2, V3, SubReg3 };
1511  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1512}
1513
1514/// QuadQRegs - Form 4 consecutive Q registers.
1515///
1516SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
1517                                   SDValue V2, SDValue V3) {
1518  DebugLoc dl = V0.getNode()->getDebugLoc();
1519  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1520  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1521  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1522  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1523  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1524  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1525                                    V2, SubReg2, V3, SubReg3 };
1526  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1527}
1528
1529/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1530/// of a NEON VLD or VST instruction.  The supported values depend on the
1531/// number of registers being loaded.
1532SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1533                                       bool is64BitVector) {
1534  unsigned NumRegs = NumVecs;
1535  if (!is64BitVector && NumVecs < 3)
1536    NumRegs *= 2;
1537
1538  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1539  if (Alignment >= 32 && NumRegs == 4)
1540    Alignment = 32;
1541  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1542    Alignment = 16;
1543  else if (Alignment >= 8)
1544    Alignment = 8;
1545  else
1546    Alignment = 0;
1547
1548  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1549}
1550
1551// Get the register stride update opcode of a VLD/VST instruction that
1552// is otherwise equivalent to the given fixed stride updating instruction.
1553static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1554  switch (Opc) {
1555  default: break;
1556  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1557  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1558  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1559  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1560  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1561  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1562  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1563  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1564
1565  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1566  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1567  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1568  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1569  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1570  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1571  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1572  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1573  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1574  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1575
1576  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1577  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1578  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1579  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1580  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1581  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1582
1583  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1584  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1585  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1586  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1587  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1588  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1589
1590  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1591  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1592  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1593  }
1594  return Opc; // If not one we handle, return it unchanged.
1595}
1596
1597SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1598                                   const uint16_t *DOpcodes,
1599                                   const uint16_t *QOpcodes0,
1600                                   const uint16_t *QOpcodes1) {
1601  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1602  DebugLoc dl = N->getDebugLoc();
1603
1604  SDValue MemAddr, Align;
1605  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1606  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1607    return NULL;
1608
1609  SDValue Chain = N->getOperand(0);
1610  EVT VT = N->getValueType(0);
1611  bool is64BitVector = VT.is64BitVector();
1612  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1613
1614  unsigned OpcodeIndex;
1615  switch (VT.getSimpleVT().SimpleTy) {
1616  default: llvm_unreachable("unhandled vld type");
1617    // Double-register operations:
1618  case MVT::v8i8:  OpcodeIndex = 0; break;
1619  case MVT::v4i16: OpcodeIndex = 1; break;
1620  case MVT::v2f32:
1621  case MVT::v2i32: OpcodeIndex = 2; break;
1622  case MVT::v1i64: OpcodeIndex = 3; break;
1623    // Quad-register operations:
1624  case MVT::v16i8: OpcodeIndex = 0; break;
1625  case MVT::v8i16: OpcodeIndex = 1; break;
1626  case MVT::v4f32:
1627  case MVT::v4i32: OpcodeIndex = 2; break;
1628  case MVT::v2i64: OpcodeIndex = 3;
1629    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1630    break;
1631  }
1632
1633  EVT ResTy;
1634  if (NumVecs == 1)
1635    ResTy = VT;
1636  else {
1637    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1638    if (!is64BitVector)
1639      ResTyElts *= 2;
1640    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1641  }
1642  std::vector<EVT> ResTys;
1643  ResTys.push_back(ResTy);
1644  if (isUpdating)
1645    ResTys.push_back(MVT::i32);
1646  ResTys.push_back(MVT::Other);
1647
1648  SDValue Pred = getAL(CurDAG);
1649  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1650  SDNode *VLd;
1651  SmallVector<SDValue, 7> Ops;
1652
1653  // Double registers and VLD1/VLD2 quad registers are directly supported.
1654  if (is64BitVector || NumVecs <= 2) {
1655    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1656                    QOpcodes0[OpcodeIndex]);
1657    Ops.push_back(MemAddr);
1658    Ops.push_back(Align);
1659    if (isUpdating) {
1660      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1661      // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
1662      // case entirely when the rest are updated to that form, too.
1663      if ((NumVecs == 1 || NumVecs == 2) && !isa<ConstantSDNode>(Inc.getNode()))
1664        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1665      // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1666      // check for that explicitly too. Horribly hacky, but temporary.
1667      if ((NumVecs != 1 && NumVecs != 2 && Opc != ARM::VLD1q64wb_fixed) ||
1668          !isa<ConstantSDNode>(Inc.getNode()))
1669        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1670    }
1671    Ops.push_back(Pred);
1672    Ops.push_back(Reg0);
1673    Ops.push_back(Chain);
1674    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1675
1676  } else {
1677    // Otherwise, quad registers are loaded with two separate instructions,
1678    // where one loads the even registers and the other loads the odd registers.
1679    EVT AddrTy = MemAddr.getValueType();
1680
1681    // Load the even subregs.  This is always an updating load, so that it
1682    // provides the address to the second load for the odd subregs.
1683    SDValue ImplDef =
1684      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1685    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1686    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1687                                          ResTy, AddrTy, MVT::Other, OpsA, 7);
1688    Chain = SDValue(VLdA, 2);
1689
1690    // Load the odd subregs.
1691    Ops.push_back(SDValue(VLdA, 1));
1692    Ops.push_back(Align);
1693    if (isUpdating) {
1694      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1695      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1696             "only constant post-increment update allowed for VLD3/4");
1697      (void)Inc;
1698      Ops.push_back(Reg0);
1699    }
1700    Ops.push_back(SDValue(VLdA, 0));
1701    Ops.push_back(Pred);
1702    Ops.push_back(Reg0);
1703    Ops.push_back(Chain);
1704    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1705                                 Ops.data(), Ops.size());
1706  }
1707
1708  // Transfer memoperands.
1709  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1710  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1711  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1712
1713  if (NumVecs == 1)
1714    return VLd;
1715
1716  // Extract out the subregisters.
1717  SDValue SuperReg = SDValue(VLd, 0);
1718  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1719         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1720  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1721  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1722    ReplaceUses(SDValue(N, Vec),
1723                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1724  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1725  if (isUpdating)
1726    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1727  return NULL;
1728}
1729
1730SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1731                                   const uint16_t *DOpcodes,
1732                                   const uint16_t *QOpcodes0,
1733                                   const uint16_t *QOpcodes1) {
1734  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1735  DebugLoc dl = N->getDebugLoc();
1736
1737  SDValue MemAddr, Align;
1738  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1739  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1740  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1741    return NULL;
1742
1743  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1744  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1745
1746  SDValue Chain = N->getOperand(0);
1747  EVT VT = N->getOperand(Vec0Idx).getValueType();
1748  bool is64BitVector = VT.is64BitVector();
1749  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1750
1751  unsigned OpcodeIndex;
1752  switch (VT.getSimpleVT().SimpleTy) {
1753  default: llvm_unreachable("unhandled vst type");
1754    // Double-register operations:
1755  case MVT::v8i8:  OpcodeIndex = 0; break;
1756  case MVT::v4i16: OpcodeIndex = 1; break;
1757  case MVT::v2f32:
1758  case MVT::v2i32: OpcodeIndex = 2; break;
1759  case MVT::v1i64: OpcodeIndex = 3; break;
1760    // Quad-register operations:
1761  case MVT::v16i8: OpcodeIndex = 0; break;
1762  case MVT::v8i16: OpcodeIndex = 1; break;
1763  case MVT::v4f32:
1764  case MVT::v4i32: OpcodeIndex = 2; break;
1765  case MVT::v2i64: OpcodeIndex = 3;
1766    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1767    break;
1768  }
1769
1770  std::vector<EVT> ResTys;
1771  if (isUpdating)
1772    ResTys.push_back(MVT::i32);
1773  ResTys.push_back(MVT::Other);
1774
1775  SDValue Pred = getAL(CurDAG);
1776  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1777  SmallVector<SDValue, 7> Ops;
1778
1779  // Double registers and VST1/VST2 quad registers are directly supported.
1780  if (is64BitVector || NumVecs <= 2) {
1781    SDValue SrcReg;
1782    if (NumVecs == 1) {
1783      SrcReg = N->getOperand(Vec0Idx);
1784    } else if (is64BitVector) {
1785      // Form a REG_SEQUENCE to force register allocation.
1786      SDValue V0 = N->getOperand(Vec0Idx + 0);
1787      SDValue V1 = N->getOperand(Vec0Idx + 1);
1788      if (NumVecs == 2)
1789        SrcReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1790      else {
1791        SDValue V2 = N->getOperand(Vec0Idx + 2);
1792        // If it's a vst3, form a quad D-register and leave the last part as
1793        // an undef.
1794        SDValue V3 = (NumVecs == 3)
1795          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1796          : N->getOperand(Vec0Idx + 3);
1797        SrcReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1798      }
1799    } else {
1800      // Form a QQ register.
1801      SDValue Q0 = N->getOperand(Vec0Idx);
1802      SDValue Q1 = N->getOperand(Vec0Idx + 1);
1803      SrcReg = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1804    }
1805
1806    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1807                    QOpcodes0[OpcodeIndex]);
1808    Ops.push_back(MemAddr);
1809    Ops.push_back(Align);
1810    if (isUpdating) {
1811      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1812      // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
1813      // case entirely when the rest are updated to that form, too.
1814      if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
1815        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1816      // We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
1817      // check for that explicitly too. Horribly hacky, but temporary.
1818      if ((NumVecs > 2 && Opc != ARM::VST1q64wb_fixed) ||
1819          !isa<ConstantSDNode>(Inc.getNode()))
1820        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1821    }
1822    Ops.push_back(SrcReg);
1823    Ops.push_back(Pred);
1824    Ops.push_back(Reg0);
1825    Ops.push_back(Chain);
1826    SDNode *VSt =
1827      CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1828
1829    // Transfer memoperands.
1830    cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
1831
1832    return VSt;
1833  }
1834
1835  // Otherwise, quad registers are stored with two separate instructions,
1836  // where one stores the even registers and the other stores the odd registers.
1837
1838  // Form the QQQQ REG_SEQUENCE.
1839  SDValue V0 = N->getOperand(Vec0Idx + 0);
1840  SDValue V1 = N->getOperand(Vec0Idx + 1);
1841  SDValue V2 = N->getOperand(Vec0Idx + 2);
1842  SDValue V3 = (NumVecs == 3)
1843    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1844    : N->getOperand(Vec0Idx + 3);
1845  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1846
1847  // Store the even D registers.  This is always an updating store, so that it
1848  // provides the address to the second store for the odd subregs.
1849  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
1850  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1851                                        MemAddr.getValueType(),
1852                                        MVT::Other, OpsA, 7);
1853  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
1854  Chain = SDValue(VStA, 1);
1855
1856  // Store the odd D registers.
1857  Ops.push_back(SDValue(VStA, 0));
1858  Ops.push_back(Align);
1859  if (isUpdating) {
1860    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1861    assert(isa<ConstantSDNode>(Inc.getNode()) &&
1862           "only constant post-increment update allowed for VST3/4");
1863    (void)Inc;
1864    Ops.push_back(Reg0);
1865  }
1866  Ops.push_back(RegSeq);
1867  Ops.push_back(Pred);
1868  Ops.push_back(Reg0);
1869  Ops.push_back(Chain);
1870  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1871                                        Ops.data(), Ops.size());
1872  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
1873  return VStB;
1874}
1875
1876SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1877                                         bool isUpdating, unsigned NumVecs,
1878                                         const uint16_t *DOpcodes,
1879                                         const uint16_t *QOpcodes) {
1880  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1881  DebugLoc dl = N->getDebugLoc();
1882
1883  SDValue MemAddr, Align;
1884  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1885  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1886  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1887    return NULL;
1888
1889  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1890  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1891
1892  SDValue Chain = N->getOperand(0);
1893  unsigned Lane =
1894    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
1895  EVT VT = N->getOperand(Vec0Idx).getValueType();
1896  bool is64BitVector = VT.is64BitVector();
1897
1898  unsigned Alignment = 0;
1899  if (NumVecs != 3) {
1900    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1901    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1902    if (Alignment > NumBytes)
1903      Alignment = NumBytes;
1904    if (Alignment < 8 && Alignment < NumBytes)
1905      Alignment = 0;
1906    // Alignment must be a power of two; make sure of that.
1907    Alignment = (Alignment & -Alignment);
1908    if (Alignment == 1)
1909      Alignment = 0;
1910  }
1911  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1912
1913  unsigned OpcodeIndex;
1914  switch (VT.getSimpleVT().SimpleTy) {
1915  default: llvm_unreachable("unhandled vld/vst lane type");
1916    // Double-register operations:
1917  case MVT::v8i8:  OpcodeIndex = 0; break;
1918  case MVT::v4i16: OpcodeIndex = 1; break;
1919  case MVT::v2f32:
1920  case MVT::v2i32: OpcodeIndex = 2; break;
1921    // Quad-register operations:
1922  case MVT::v8i16: OpcodeIndex = 0; break;
1923  case MVT::v4f32:
1924  case MVT::v4i32: OpcodeIndex = 1; break;
1925  }
1926
1927  std::vector<EVT> ResTys;
1928  if (IsLoad) {
1929    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1930    if (!is64BitVector)
1931      ResTyElts *= 2;
1932    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
1933                                      MVT::i64, ResTyElts));
1934  }
1935  if (isUpdating)
1936    ResTys.push_back(MVT::i32);
1937  ResTys.push_back(MVT::Other);
1938
1939  SDValue Pred = getAL(CurDAG);
1940  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1941
1942  SmallVector<SDValue, 8> Ops;
1943  Ops.push_back(MemAddr);
1944  Ops.push_back(Align);
1945  if (isUpdating) {
1946    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1947    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1948  }
1949
1950  SDValue SuperReg;
1951  SDValue V0 = N->getOperand(Vec0Idx + 0);
1952  SDValue V1 = N->getOperand(Vec0Idx + 1);
1953  if (NumVecs == 2) {
1954    if (is64BitVector)
1955      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1956    else
1957      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1958  } else {
1959    SDValue V2 = N->getOperand(Vec0Idx + 2);
1960    SDValue V3 = (NumVecs == 3)
1961      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1962      : N->getOperand(Vec0Idx + 3);
1963    if (is64BitVector)
1964      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1965    else
1966      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1967  }
1968  Ops.push_back(SuperReg);
1969  Ops.push_back(getI32Imm(Lane));
1970  Ops.push_back(Pred);
1971  Ops.push_back(Reg0);
1972  Ops.push_back(Chain);
1973
1974  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1975                                  QOpcodes[OpcodeIndex]);
1976  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys,
1977                                         Ops.data(), Ops.size());
1978  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
1979  if (!IsLoad)
1980    return VLdLn;
1981
1982  // Extract the subregisters.
1983  SuperReg = SDValue(VLdLn, 0);
1984  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1985         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1986  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1987  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1988    ReplaceUses(SDValue(N, Vec),
1989                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1990  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
1991  if (isUpdating)
1992    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
1993  return NULL;
1994}
1995
1996SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
1997                                      unsigned NumVecs,
1998                                      const uint16_t *Opcodes) {
1999  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2000  DebugLoc dl = N->getDebugLoc();
2001
2002  SDValue MemAddr, Align;
2003  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2004    return NULL;
2005
2006  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2007  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2008
2009  SDValue Chain = N->getOperand(0);
2010  EVT VT = N->getValueType(0);
2011
2012  unsigned Alignment = 0;
2013  if (NumVecs != 3) {
2014    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2015    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2016    if (Alignment > NumBytes)
2017      Alignment = NumBytes;
2018    if (Alignment < 8 && Alignment < NumBytes)
2019      Alignment = 0;
2020    // Alignment must be a power of two; make sure of that.
2021    Alignment = (Alignment & -Alignment);
2022    if (Alignment == 1)
2023      Alignment = 0;
2024  }
2025  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2026
2027  unsigned OpcodeIndex;
2028  switch (VT.getSimpleVT().SimpleTy) {
2029  default: llvm_unreachable("unhandled vld-dup type");
2030  case MVT::v8i8:  OpcodeIndex = 0; break;
2031  case MVT::v4i16: OpcodeIndex = 1; break;
2032  case MVT::v2f32:
2033  case MVT::v2i32: OpcodeIndex = 2; break;
2034  }
2035
2036  SDValue Pred = getAL(CurDAG);
2037  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2038  SDValue SuperReg;
2039  unsigned Opc = Opcodes[OpcodeIndex];
2040  SmallVector<SDValue, 6> Ops;
2041  Ops.push_back(MemAddr);
2042  Ops.push_back(Align);
2043  if (isUpdating) {
2044    // fixed-stride update instructions don't have an explicit writeback
2045    // operand. It's implicit in the opcode itself.
2046    SDValue Inc = N->getOperand(2);
2047    if (!isa<ConstantSDNode>(Inc.getNode()))
2048      Ops.push_back(Inc);
2049    // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2050    else if (NumVecs > 2)
2051      Ops.push_back(Reg0);
2052  }
2053  Ops.push_back(Pred);
2054  Ops.push_back(Reg0);
2055  Ops.push_back(Chain);
2056
2057  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2058  std::vector<EVT> ResTys;
2059  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2060  if (isUpdating)
2061    ResTys.push_back(MVT::i32);
2062  ResTys.push_back(MVT::Other);
2063  SDNode *VLdDup =
2064    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
2065  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2066  SuperReg = SDValue(VLdDup, 0);
2067
2068  // Extract the subregisters.
2069  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
2070  unsigned SubIdx = ARM::dsub_0;
2071  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2072    ReplaceUses(SDValue(N, Vec),
2073                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2074  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2075  if (isUpdating)
2076    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2077  return NULL;
2078}
2079
2080SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
2081                                    unsigned Opc) {
2082  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
2083  DebugLoc dl = N->getDebugLoc();
2084  EVT VT = N->getValueType(0);
2085  unsigned FirstTblReg = IsExt ? 2 : 1;
2086
2087  // Form a REG_SEQUENCE to force register allocation.
2088  SDValue RegSeq;
2089  SDValue V0 = N->getOperand(FirstTblReg + 0);
2090  SDValue V1 = N->getOperand(FirstTblReg + 1);
2091  if (NumVecs == 2)
2092    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
2093  else {
2094    SDValue V2 = N->getOperand(FirstTblReg + 2);
2095    // If it's a vtbl3, form a quad D-register and leave the last part as
2096    // an undef.
2097    SDValue V3 = (NumVecs == 3)
2098      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2099      : N->getOperand(FirstTblReg + 3);
2100    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
2101  }
2102
2103  SmallVector<SDValue, 6> Ops;
2104  if (IsExt)
2105    Ops.push_back(N->getOperand(1));
2106  Ops.push_back(RegSeq);
2107  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
2108  Ops.push_back(getAL(CurDAG)); // predicate
2109  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
2110  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
2111}
2112
2113SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
2114                                                     bool isSigned) {
2115  if (!Subtarget->hasV6T2Ops())
2116    return NULL;
2117
2118  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2119    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2120
2121
2122  // For unsigned extracts, check for a shift right and mask
2123  unsigned And_imm = 0;
2124  if (N->getOpcode() == ISD::AND) {
2125    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2126
2127      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2128      if (And_imm & (And_imm + 1))
2129        return NULL;
2130
2131      unsigned Srl_imm = 0;
2132      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2133                                Srl_imm)) {
2134        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2135
2136        // Note: The width operand is encoded as width-1.
2137        unsigned Width = CountTrailingOnes_32(And_imm) - 1;
2138        unsigned LSB = Srl_imm;
2139        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2140        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2141                          CurDAG->getTargetConstant(LSB, MVT::i32),
2142                          CurDAG->getTargetConstant(Width, MVT::i32),
2143          getAL(CurDAG), Reg0 };
2144        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2145      }
2146    }
2147    return NULL;
2148  }
2149
2150  // Otherwise, we're looking for a shift of a shift
2151  unsigned Shl_imm = 0;
2152  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2153    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2154    unsigned Srl_imm = 0;
2155    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2156      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2157      // Note: The width operand is encoded as width-1.
2158      unsigned Width = 32 - Srl_imm - 1;
2159      int LSB = Srl_imm - Shl_imm;
2160      if (LSB < 0)
2161        return NULL;
2162      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2163      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2164                        CurDAG->getTargetConstant(LSB, MVT::i32),
2165                        CurDAG->getTargetConstant(Width, MVT::i32),
2166                        getAL(CurDAG), Reg0 };
2167      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2168    }
2169  }
2170  return NULL;
2171}
2172
2173SDNode *ARMDAGToDAGISel::
2174SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2175                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2176  SDValue CPTmp0;
2177  SDValue CPTmp1;
2178  if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
2179    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
2180    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
2181    unsigned Opc = 0;
2182    switch (SOShOp) {
2183    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
2184    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
2185    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
2186    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
2187    default:
2188      llvm_unreachable("Unknown so_reg opcode!");
2189    }
2190    SDValue SOShImm =
2191      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
2192    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2193    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
2194    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
2195  }
2196  return 0;
2197}
2198
2199SDNode *ARMDAGToDAGISel::
2200SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2201                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2202  SDValue CPTmp0;
2203  SDValue CPTmp1;
2204  SDValue CPTmp2;
2205  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
2206    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2207    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag };
2208    return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6);
2209  }
2210
2211  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
2212    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2213    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
2214    return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7);
2215  }
2216  return 0;
2217}
2218
2219SDNode *ARMDAGToDAGISel::
2220SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2221                  ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2222  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2223  if (!T)
2224    return 0;
2225
2226  unsigned Opc = 0;
2227  unsigned TrueImm = T->getZExtValue();
2228  if (is_t2_so_imm(TrueImm)) {
2229    Opc = ARM::t2MOVCCi;
2230  } else if (TrueImm <= 0xffff) {
2231    Opc = ARM::t2MOVCCi16;
2232  } else if (is_t2_so_imm_not(TrueImm)) {
2233    TrueImm = ~TrueImm;
2234    Opc = ARM::t2MVNCCi;
2235  } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) {
2236    // Large immediate.
2237    Opc = ARM::t2MOVCCi32imm;
2238  }
2239
2240  if (Opc) {
2241    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2242    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2243    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2244    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2245  }
2246
2247  return 0;
2248}
2249
2250SDNode *ARMDAGToDAGISel::
2251SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2252                   ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2253  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2254  if (!T)
2255    return 0;
2256
2257  unsigned Opc = 0;
2258  unsigned TrueImm = T->getZExtValue();
2259  bool isSoImm = is_so_imm(TrueImm);
2260  if (isSoImm) {
2261    Opc = ARM::MOVCCi;
2262  } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) {
2263    Opc = ARM::MOVCCi16;
2264  } else if (is_so_imm_not(TrueImm)) {
2265    TrueImm = ~TrueImm;
2266    Opc = ARM::MVNCCi;
2267  } else if (TrueVal.getNode()->hasOneUse() &&
2268             (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) {
2269    // Large immediate.
2270    Opc = ARM::MOVCCi32imm;
2271  }
2272
2273  if (Opc) {
2274    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2275    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2276    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2277    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2278  }
2279
2280  return 0;
2281}
2282
2283SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
2284  EVT VT = N->getValueType(0);
2285  SDValue FalseVal = N->getOperand(0);
2286  SDValue TrueVal  = N->getOperand(1);
2287  SDValue CC = N->getOperand(2);
2288  SDValue CCR = N->getOperand(3);
2289  SDValue InFlag = N->getOperand(4);
2290  assert(CC.getOpcode() == ISD::Constant);
2291  assert(CCR.getOpcode() == ISD::Register);
2292  ARMCC::CondCodes CCVal =
2293    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
2294
2295  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
2296    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2297    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2298    // Pattern complexity = 18  cost = 1  size = 0
2299    if (Subtarget->isThumb()) {
2300      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
2301                                        CCVal, CCR, InFlag);
2302      if (!Res)
2303        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
2304                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2305      if (Res)
2306        return Res;
2307    } else {
2308      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
2309                                         CCVal, CCR, InFlag);
2310      if (!Res)
2311        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
2312                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2313      if (Res)
2314        return Res;
2315    }
2316
2317    // Pattern: (ARMcmov:i32 GPR:i32:$false,
2318    //             (imm:i32)<<P:Pred_so_imm>>:$true,
2319    //             (imm:i32):$cc)
2320    // Emits: (MOVCCi:i32 GPR:i32:$false,
2321    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
2322    // Pattern complexity = 10  cost = 1  size = 0
2323    if (Subtarget->isThumb()) {
2324      SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal,
2325                                        CCVal, CCR, InFlag);
2326      if (!Res)
2327        Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal,
2328                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2329      if (Res)
2330        return Res;
2331    } else {
2332      SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal,
2333                                         CCVal, CCR, InFlag);
2334      if (!Res)
2335        Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal,
2336                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2337      if (Res)
2338        return Res;
2339    }
2340  }
2341
2342  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2343  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2344  // Pattern complexity = 6  cost = 1  size = 0
2345  //
2346  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2347  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2348  // Pattern complexity = 6  cost = 11  size = 0
2349  //
2350  // Also VMOVScc and VMOVDcc.
2351  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
2352  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
2353  unsigned Opc = 0;
2354  switch (VT.getSimpleVT().SimpleTy) {
2355  default: llvm_unreachable("Illegal conditional move type!");
2356  case MVT::i32:
2357    Opc = Subtarget->isThumb()
2358      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
2359      : ARM::MOVCCr;
2360    break;
2361  case MVT::f32:
2362    Opc = ARM::VMOVScc;
2363    break;
2364  case MVT::f64:
2365    Opc = ARM::VMOVDcc;
2366    break;
2367  }
2368  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2369}
2370
2371SDNode *ARMDAGToDAGISel::SelectConditionalOp(SDNode *N) {
2372  SDValue FalseVal = N->getOperand(0);
2373  SDValue TrueVal  = N->getOperand(1);
2374  ARMCC::CondCodes CCVal =
2375    (ARMCC::CondCodes)cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
2376  SDValue CCR = N->getOperand(3);
2377  assert(CCR.getOpcode() == ISD::Register);
2378  SDValue InFlag = N->getOperand(4);
2379  SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2380  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2381
2382  if (Subtarget->isThumb()) {
2383    SDValue CPTmp0;
2384    SDValue CPTmp1;
2385    if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
2386      unsigned Opc;
2387      switch (N->getOpcode()) {
2388      default: llvm_unreachable("Unexpected node");
2389      case ARMISD::CAND: Opc = ARM::t2ANDCCrs; break;
2390      case ARMISD::COR:  Opc = ARM::t2ORRCCrs; break;
2391      case ARMISD::CXOR: Opc = ARM::t2EORCCrs; break;
2392      }
2393      SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CC, CCR, Reg0, InFlag };
2394      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
2395    }
2396
2397    ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2398    if (T) {
2399      unsigned TrueImm = T->getZExtValue();
2400      if (is_t2_so_imm(TrueImm)) {
2401        unsigned Opc;
2402        switch (N->getOpcode()) {
2403        default: llvm_unreachable("Unexpected node");
2404        case ARMISD::CAND: Opc = ARM::t2ANDCCri; break;
2405        case ARMISD::COR:  Opc = ARM::t2ORRCCri; break;
2406        case ARMISD::CXOR: Opc = ARM::t2EORCCri; break;
2407        }
2408        SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2409        SDValue Ops[] = { FalseVal, True, CC, CCR, Reg0, InFlag };
2410        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2411      }
2412    }
2413
2414    unsigned Opc;
2415    switch (N->getOpcode()) {
2416    default: llvm_unreachable("Unexpected node");
2417    case ARMISD::CAND: Opc = ARM::t2ANDCCrr; break;
2418    case ARMISD::COR:  Opc = ARM::t2ORRCCrr; break;
2419    case ARMISD::CXOR: Opc = ARM::t2EORCCrr; break;
2420    }
2421    SDValue Ops[] = { FalseVal, TrueVal, CC, CCR, Reg0, InFlag };
2422    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2423  }
2424
2425  SDValue CPTmp0;
2426  SDValue CPTmp1;
2427  SDValue CPTmp2;
2428  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
2429    unsigned Opc;
2430    switch (N->getOpcode()) {
2431    default: llvm_unreachable("Unexpected node");
2432    case ARMISD::CAND: Opc = ARM::ANDCCrsi; break;
2433    case ARMISD::COR:  Opc = ARM::ORRCCrsi; break;
2434    case ARMISD::CXOR: Opc = ARM::EORCCrsi; break;
2435    }
2436    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, Reg0, InFlag };
2437    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
2438  }
2439
2440  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
2441    unsigned Opc;
2442    switch (N->getOpcode()) {
2443    default: llvm_unreachable("Unexpected node");
2444    case ARMISD::CAND: Opc = ARM::ANDCCrsr; break;
2445    case ARMISD::COR:  Opc = ARM::ORRCCrsr; break;
2446    case ARMISD::CXOR: Opc = ARM::EORCCrsr; break;
2447    }
2448    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, Reg0, InFlag };
2449    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 8);
2450  }
2451
2452  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2453  if (T) {
2454    unsigned TrueImm = T->getZExtValue();
2455    if (is_so_imm(TrueImm)) {
2456      unsigned Opc;
2457      switch (N->getOpcode()) {
2458      default: llvm_unreachable("Unexpected node");
2459      case ARMISD::CAND: Opc = ARM::ANDCCri; break;
2460      case ARMISD::COR:  Opc = ARM::ORRCCri; break;
2461      case ARMISD::CXOR: Opc = ARM::EORCCri; break;
2462      }
2463      SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2464      SDValue Ops[] = { FalseVal, True, CC, CCR, Reg0, InFlag };
2465      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2466    }
2467  }
2468
2469  unsigned Opc;
2470  switch (N->getOpcode()) {
2471  default: llvm_unreachable("Unexpected node");
2472  case ARMISD::CAND: Opc = ARM::ANDCCrr; break;
2473  case ARMISD::COR:  Opc = ARM::ORRCCrr; break;
2474  case ARMISD::CXOR: Opc = ARM::EORCCrr; break;
2475  }
2476  SDValue Ops[] = { FalseVal, TrueVal, CC, CCR, Reg0, InFlag };
2477  return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2478}
2479
2480/// Target-specific DAG combining for ISD::XOR.
2481/// Target-independent combining lowers SELECT_CC nodes of the form
2482/// select_cc setg[ge] X,  0,  X, -X
2483/// select_cc setgt    X, -1,  X, -X
2484/// select_cc setl[te] X,  0, -X,  X
2485/// select_cc setlt    X,  1, -X,  X
2486/// which represent Integer ABS into:
2487/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2488/// ARM instruction selection detects the latter and matches it to
2489/// ARM::ABS or ARM::t2ABS machine node.
2490SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
2491  SDValue XORSrc0 = N->getOperand(0);
2492  SDValue XORSrc1 = N->getOperand(1);
2493  EVT VT = N->getValueType(0);
2494
2495  if (DisableARMIntABS)
2496    return NULL;
2497
2498  if (Subtarget->isThumb1Only())
2499    return NULL;
2500
2501  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2502    return NULL;
2503
2504  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2505  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2506  SDValue SRASrc0 = XORSrc1.getOperand(0);
2507  SDValue SRASrc1 = XORSrc1.getOperand(1);
2508  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
2509  EVT XType = SRASrc0.getValueType();
2510  unsigned Size = XType.getSizeInBits() - 1;
2511
2512  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2513      XType.isInteger() && SRAConstant != NULL &&
2514      Size == SRAConstant->getZExtValue()) {
2515    unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2516    return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2517  }
2518
2519  return NULL;
2520}
2521
2522SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2523  // The only time a CONCAT_VECTORS operation can have legal types is when
2524  // two 64-bit vectors are concatenated to a 128-bit vector.
2525  EVT VT = N->getValueType(0);
2526  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2527    llvm_unreachable("unexpected CONCAT_VECTORS");
2528  return PairDRegs(VT, N->getOperand(0), N->getOperand(1));
2529}
2530
2531SDNode *ARMDAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
2532  SmallVector<SDValue, 6> Ops;
2533  Ops.push_back(Node->getOperand(1)); // Ptr
2534  Ops.push_back(Node->getOperand(2)); // Low part of Val1
2535  Ops.push_back(Node->getOperand(3)); // High part of Val1
2536  if (Opc == ARM::ATOMCMPXCHG6432) {
2537    Ops.push_back(Node->getOperand(4)); // Low part of Val2
2538    Ops.push_back(Node->getOperand(5)); // High part of Val2
2539  }
2540  Ops.push_back(Node->getOperand(0)); // Chain
2541  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2542  MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
2543  SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
2544                                           MVT::i32, MVT::i32, MVT::Other,
2545                                           Ops.data() ,Ops.size());
2546  cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1);
2547  return ResNode;
2548}
2549
2550SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2551  DebugLoc dl = N->getDebugLoc();
2552
2553  if (N->isMachineOpcode())
2554    return NULL;   // Already selected.
2555
2556  switch (N->getOpcode()) {
2557  default: break;
2558  case ISD::XOR: {
2559    // Select special operations if XOR node forms integer ABS pattern
2560    SDNode *ResNode = SelectABSOp(N);
2561    if (ResNode)
2562      return ResNode;
2563    // Other cases are autogenerated.
2564    break;
2565  }
2566  case ISD::Constant: {
2567    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2568    bool UseCP = true;
2569    if (Subtarget->hasThumb2())
2570      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2571      // be done with MOV + MOVT, at worst.
2572      UseCP = 0;
2573    else {
2574      if (Subtarget->isThumb()) {
2575        UseCP = (Val > 255 &&                          // MOV
2576                 ~Val > 255 &&                         // MOV + MVN
2577                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
2578      } else
2579        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
2580                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
2581                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
2582    }
2583
2584    if (UseCP) {
2585      SDValue CPIdx =
2586        CurDAG->getTargetConstantPool(ConstantInt::get(
2587                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
2588                                      TLI.getPointerTy());
2589
2590      SDNode *ResNode;
2591      if (Subtarget->isThumb1Only()) {
2592        SDValue Pred = getAL(CurDAG);
2593        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2594        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2595        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2596                                         Ops, 4);
2597      } else {
2598        SDValue Ops[] = {
2599          CPIdx,
2600          CurDAG->getTargetConstant(0, MVT::i32),
2601          getAL(CurDAG),
2602          CurDAG->getRegister(0, MVT::i32),
2603          CurDAG->getEntryNode()
2604        };
2605        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2606                                       Ops, 5);
2607      }
2608      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2609      return NULL;
2610    }
2611
2612    // Other cases are autogenerated.
2613    break;
2614  }
2615  case ISD::FrameIndex: {
2616    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2617    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2618    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
2619    if (Subtarget->isThumb1Only()) {
2620      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2621                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2622      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops, 4);
2623    } else {
2624      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2625                      ARM::t2ADDri : ARM::ADDri);
2626      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2627                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2628                        CurDAG->getRegister(0, MVT::i32) };
2629      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2630    }
2631  }
2632  case ISD::SRL:
2633    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2634      return I;
2635    break;
2636  case ISD::SRA:
2637    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2638      return I;
2639    break;
2640  case ISD::MUL:
2641    if (Subtarget->isThumb1Only())
2642      break;
2643    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2644      unsigned RHSV = C->getZExtValue();
2645      if (!RHSV) break;
2646      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2647        unsigned ShImm = Log2_32(RHSV-1);
2648        if (ShImm >= 32)
2649          break;
2650        SDValue V = N->getOperand(0);
2651        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2652        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2653        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2654        if (Subtarget->isThumb()) {
2655          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2656          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
2657        } else {
2658          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2659          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7);
2660        }
2661      }
2662      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2663        unsigned ShImm = Log2_32(RHSV+1);
2664        if (ShImm >= 32)
2665          break;
2666        SDValue V = N->getOperand(0);
2667        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2668        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2669        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2670        if (Subtarget->isThumb()) {
2671          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2672          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
2673        } else {
2674          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2675          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7);
2676        }
2677      }
2678    }
2679    break;
2680  case ISD::AND: {
2681    // Check for unsigned bitfield extract
2682    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2683      return I;
2684
2685    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2686    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2687    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2688    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2689    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2690    EVT VT = N->getValueType(0);
2691    if (VT != MVT::i32)
2692      break;
2693    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2694      ? ARM::t2MOVTi16
2695      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2696    if (!Opc)
2697      break;
2698    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2699    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2700    if (!N1C)
2701      break;
2702    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2703      SDValue N2 = N0.getOperand(1);
2704      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2705      if (!N2C)
2706        break;
2707      unsigned N1CVal = N1C->getZExtValue();
2708      unsigned N2CVal = N2C->getZExtValue();
2709      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2710          (N1CVal & 0xffffU) == 0xffffU &&
2711          (N2CVal & 0xffffU) == 0x0U) {
2712        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2713                                                  MVT::i32);
2714        SDValue Ops[] = { N0.getOperand(0), Imm16,
2715                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2716        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
2717      }
2718    }
2719    break;
2720  }
2721  case ARMISD::VMOVRRD:
2722    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2723                                  N->getOperand(0), getAL(CurDAG),
2724                                  CurDAG->getRegister(0, MVT::i32));
2725  case ISD::UMUL_LOHI: {
2726    if (Subtarget->isThumb1Only())
2727      break;
2728    if (Subtarget->isThumb()) {
2729      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2730                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2731                        CurDAG->getRegister(0, MVT::i32) };
2732      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
2733    } else {
2734      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2735                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2736                        CurDAG->getRegister(0, MVT::i32) };
2737      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2738                                    ARM::UMULL : ARM::UMULLv5,
2739                                    dl, MVT::i32, MVT::i32, Ops, 5);
2740    }
2741  }
2742  case ISD::SMUL_LOHI: {
2743    if (Subtarget->isThumb1Only())
2744      break;
2745    if (Subtarget->isThumb()) {
2746      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2747                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2748      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
2749    } else {
2750      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2751                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2752                        CurDAG->getRegister(0, MVT::i32) };
2753      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2754                                    ARM::SMULL : ARM::SMULLv5,
2755                                    dl, MVT::i32, MVT::i32, Ops, 5);
2756    }
2757  }
2758  case ISD::LOAD: {
2759    SDNode *ResNode = 0;
2760    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2761      ResNode = SelectT2IndexedLoad(N);
2762    else
2763      ResNode = SelectARMIndexedLoad(N);
2764    if (ResNode)
2765      return ResNode;
2766    // Other cases are autogenerated.
2767    break;
2768  }
2769  case ARMISD::BRCOND: {
2770    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2771    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2772    // Pattern complexity = 6  cost = 1  size = 0
2773
2774    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2775    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2776    // Pattern complexity = 6  cost = 1  size = 0
2777
2778    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2779    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2780    // Pattern complexity = 6  cost = 1  size = 0
2781
2782    unsigned Opc = Subtarget->isThumb() ?
2783      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2784    SDValue Chain = N->getOperand(0);
2785    SDValue N1 = N->getOperand(1);
2786    SDValue N2 = N->getOperand(2);
2787    SDValue N3 = N->getOperand(3);
2788    SDValue InFlag = N->getOperand(4);
2789    assert(N1.getOpcode() == ISD::BasicBlock);
2790    assert(N2.getOpcode() == ISD::Constant);
2791    assert(N3.getOpcode() == ISD::Register);
2792
2793    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2794                               cast<ConstantSDNode>(N2)->getZExtValue()),
2795                               MVT::i32);
2796    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2797    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2798                                             MVT::Glue, Ops, 5);
2799    Chain = SDValue(ResNode, 0);
2800    if (N->getNumValues() == 2) {
2801      InFlag = SDValue(ResNode, 1);
2802      ReplaceUses(SDValue(N, 1), InFlag);
2803    }
2804    ReplaceUses(SDValue(N, 0),
2805                SDValue(Chain.getNode(), Chain.getResNo()));
2806    return NULL;
2807  }
2808  case ARMISD::CMOV:
2809    return SelectCMOVOp(N);
2810  case ARMISD::CAND:
2811  case ARMISD::COR:
2812  case ARMISD::CXOR:
2813    return SelectConditionalOp(N);
2814  case ARMISD::VZIP: {
2815    unsigned Opc = 0;
2816    EVT VT = N->getValueType(0);
2817    switch (VT.getSimpleVT().SimpleTy) {
2818    default: return NULL;
2819    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2820    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2821    case MVT::v2f32:
2822    // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2823    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2824    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2825    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2826    case MVT::v4f32:
2827    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2828    }
2829    SDValue Pred = getAL(CurDAG);
2830    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2831    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2832    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2833  }
2834  case ARMISD::VUZP: {
2835    unsigned Opc = 0;
2836    EVT VT = N->getValueType(0);
2837    switch (VT.getSimpleVT().SimpleTy) {
2838    default: return NULL;
2839    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2840    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2841    case MVT::v2f32:
2842    // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2843    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2844    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2845    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2846    case MVT::v4f32:
2847    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2848    }
2849    SDValue Pred = getAL(CurDAG);
2850    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2851    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2852    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2853  }
2854  case ARMISD::VTRN: {
2855    unsigned Opc = 0;
2856    EVT VT = N->getValueType(0);
2857    switch (VT.getSimpleVT().SimpleTy) {
2858    default: return NULL;
2859    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2860    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2861    case MVT::v2f32:
2862    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2863    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2864    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2865    case MVT::v4f32:
2866    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2867    }
2868    SDValue Pred = getAL(CurDAG);
2869    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2870    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2871    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2872  }
2873  case ARMISD::BUILD_VECTOR: {
2874    EVT VecVT = N->getValueType(0);
2875    EVT EltVT = VecVT.getVectorElementType();
2876    unsigned NumElts = VecVT.getVectorNumElements();
2877    if (EltVT == MVT::f64) {
2878      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2879      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
2880    }
2881    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2882    if (NumElts == 2)
2883      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
2884    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2885    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
2886                     N->getOperand(2), N->getOperand(3));
2887  }
2888
2889  case ARMISD::VLD2DUP: {
2890    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
2891                                        ARM::VLD2DUPd32 };
2892    return SelectVLDDup(N, false, 2, Opcodes);
2893  }
2894
2895  case ARMISD::VLD3DUP: {
2896    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
2897                                        ARM::VLD3DUPd16Pseudo,
2898                                        ARM::VLD3DUPd32Pseudo };
2899    return SelectVLDDup(N, false, 3, Opcodes);
2900  }
2901
2902  case ARMISD::VLD4DUP: {
2903    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
2904                                        ARM::VLD4DUPd16Pseudo,
2905                                        ARM::VLD4DUPd32Pseudo };
2906    return SelectVLDDup(N, false, 4, Opcodes);
2907  }
2908
2909  case ARMISD::VLD2DUP_UPD: {
2910    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
2911                                        ARM::VLD2DUPd16wb_fixed,
2912                                        ARM::VLD2DUPd32wb_fixed };
2913    return SelectVLDDup(N, true, 2, Opcodes);
2914  }
2915
2916  case ARMISD::VLD3DUP_UPD: {
2917    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
2918                                        ARM::VLD3DUPd16Pseudo_UPD,
2919                                        ARM::VLD3DUPd32Pseudo_UPD };
2920    return SelectVLDDup(N, true, 3, Opcodes);
2921  }
2922
2923  case ARMISD::VLD4DUP_UPD: {
2924    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
2925                                        ARM::VLD4DUPd16Pseudo_UPD,
2926                                        ARM::VLD4DUPd32Pseudo_UPD };
2927    return SelectVLDDup(N, true, 4, Opcodes);
2928  }
2929
2930  case ARMISD::VLD1_UPD: {
2931    static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
2932                                         ARM::VLD1d16wb_fixed,
2933                                         ARM::VLD1d32wb_fixed,
2934                                         ARM::VLD1d64wb_fixed };
2935    static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
2936                                         ARM::VLD1q16wb_fixed,
2937                                         ARM::VLD1q32wb_fixed,
2938                                         ARM::VLD1q64wb_fixed };
2939    return SelectVLD(N, true, 1, DOpcodes, QOpcodes, 0);
2940  }
2941
2942  case ARMISD::VLD2_UPD: {
2943    static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
2944                                         ARM::VLD2d16wb_fixed,
2945                                         ARM::VLD2d32wb_fixed,
2946                                         ARM::VLD1q64wb_fixed};
2947    static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
2948                                         ARM::VLD2q16PseudoWB_fixed,
2949                                         ARM::VLD2q32PseudoWB_fixed };
2950    return SelectVLD(N, true, 2, DOpcodes, QOpcodes, 0);
2951  }
2952
2953  case ARMISD::VLD3_UPD: {
2954    static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
2955                                         ARM::VLD3d16Pseudo_UPD,
2956                                         ARM::VLD3d32Pseudo_UPD,
2957                                         ARM::VLD1q64wb_fixed};
2958    static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2959                                          ARM::VLD3q16Pseudo_UPD,
2960                                          ARM::VLD3q32Pseudo_UPD };
2961    static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2962                                          ARM::VLD3q16oddPseudo_UPD,
2963                                          ARM::VLD3q32oddPseudo_UPD };
2964    return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2965  }
2966
2967  case ARMISD::VLD4_UPD: {
2968    static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
2969                                         ARM::VLD4d16Pseudo_UPD,
2970                                         ARM::VLD4d32Pseudo_UPD,
2971                                         ARM::VLD1q64wb_fixed};
2972    static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2973                                          ARM::VLD4q16Pseudo_UPD,
2974                                          ARM::VLD4q32Pseudo_UPD };
2975    static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2976                                          ARM::VLD4q16oddPseudo_UPD,
2977                                          ARM::VLD4q32oddPseudo_UPD };
2978    return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2979  }
2980
2981  case ARMISD::VLD2LN_UPD: {
2982    static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
2983                                         ARM::VLD2LNd16Pseudo_UPD,
2984                                         ARM::VLD2LNd32Pseudo_UPD };
2985    static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2986                                         ARM::VLD2LNq32Pseudo_UPD };
2987    return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2988  }
2989
2990  case ARMISD::VLD3LN_UPD: {
2991    static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
2992                                         ARM::VLD3LNd16Pseudo_UPD,
2993                                         ARM::VLD3LNd32Pseudo_UPD };
2994    static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2995                                         ARM::VLD3LNq32Pseudo_UPD };
2996    return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2997  }
2998
2999  case ARMISD::VLD4LN_UPD: {
3000    static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
3001                                         ARM::VLD4LNd16Pseudo_UPD,
3002                                         ARM::VLD4LNd32Pseudo_UPD };
3003    static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
3004                                         ARM::VLD4LNq32Pseudo_UPD };
3005    return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
3006  }
3007
3008  case ARMISD::VST1_UPD: {
3009    static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
3010                                         ARM::VST1d16wb_fixed,
3011                                         ARM::VST1d32wb_fixed,
3012                                         ARM::VST1d64wb_fixed };
3013    static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
3014                                         ARM::VST1q16wb_fixed,
3015                                         ARM::VST1q32wb_fixed,
3016                                         ARM::VST1q64wb_fixed };
3017    return SelectVST(N, true, 1, DOpcodes, QOpcodes, 0);
3018  }
3019
3020  case ARMISD::VST2_UPD: {
3021    static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
3022                                         ARM::VST2d16wb_fixed,
3023                                         ARM::VST2d32wb_fixed,
3024                                         ARM::VST1q64wb_fixed};
3025    static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
3026                                         ARM::VST2q16PseudoWB_fixed,
3027                                         ARM::VST2q32PseudoWB_fixed };
3028    return SelectVST(N, true, 2, DOpcodes, QOpcodes, 0);
3029  }
3030
3031  case ARMISD::VST3_UPD: {
3032    static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
3033                                         ARM::VST3d16Pseudo_UPD,
3034                                         ARM::VST3d32Pseudo_UPD,
3035                                         ARM::VST1d64TPseudoWB_fixed};
3036    static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3037                                          ARM::VST3q16Pseudo_UPD,
3038                                          ARM::VST3q32Pseudo_UPD };
3039    static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
3040                                          ARM::VST3q16oddPseudo_UPD,
3041                                          ARM::VST3q32oddPseudo_UPD };
3042    return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3043  }
3044
3045  case ARMISD::VST4_UPD: {
3046    static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
3047                                         ARM::VST4d16Pseudo_UPD,
3048                                         ARM::VST4d32Pseudo_UPD,
3049                                         ARM::VST1d64QPseudoWB_fixed};
3050    static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3051                                          ARM::VST4q16Pseudo_UPD,
3052                                          ARM::VST4q32Pseudo_UPD };
3053    static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
3054                                          ARM::VST4q16oddPseudo_UPD,
3055                                          ARM::VST4q32oddPseudo_UPD };
3056    return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3057  }
3058
3059  case ARMISD::VST2LN_UPD: {
3060    static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
3061                                         ARM::VST2LNd16Pseudo_UPD,
3062                                         ARM::VST2LNd32Pseudo_UPD };
3063    static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
3064                                         ARM::VST2LNq32Pseudo_UPD };
3065    return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
3066  }
3067
3068  case ARMISD::VST3LN_UPD: {
3069    static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
3070                                         ARM::VST3LNd16Pseudo_UPD,
3071                                         ARM::VST3LNd32Pseudo_UPD };
3072    static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
3073                                         ARM::VST3LNq32Pseudo_UPD };
3074    return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3075  }
3076
3077  case ARMISD::VST4LN_UPD: {
3078    static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
3079                                         ARM::VST4LNd16Pseudo_UPD,
3080                                         ARM::VST4LNd32Pseudo_UPD };
3081    static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3082                                         ARM::VST4LNq32Pseudo_UPD };
3083    return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3084  }
3085
3086  case ISD::INTRINSIC_VOID:
3087  case ISD::INTRINSIC_W_CHAIN: {
3088    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3089    switch (IntNo) {
3090    default:
3091      break;
3092
3093    case Intrinsic::arm_ldrexd: {
3094      SDValue MemAddr = N->getOperand(2);
3095      DebugLoc dl = N->getDebugLoc();
3096      SDValue Chain = N->getOperand(0);
3097
3098      unsigned NewOpc = ARM::LDREXD;
3099      if (Subtarget->isThumb() && Subtarget->hasThumb2())
3100        NewOpc = ARM::t2LDREXD;
3101
3102      // arm_ldrexd returns a i64 value in {i32, i32}
3103      std::vector<EVT> ResTys;
3104      ResTys.push_back(MVT::i32);
3105      ResTys.push_back(MVT::i32);
3106      ResTys.push_back(MVT::Other);
3107
3108      // place arguments in the right order
3109      SmallVector<SDValue, 7> Ops;
3110      Ops.push_back(MemAddr);
3111      Ops.push_back(getAL(CurDAG));
3112      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3113      Ops.push_back(Chain);
3114      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3115                                          Ops.size());
3116      // Transfer memoperands.
3117      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3118      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3119      cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3120
3121      // Until there's support for specifing explicit register constraints
3122      // like the use of even/odd register pair, hardcode ldrexd to always
3123      // use the pair [R0, R1] to hold the load result.
3124      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R0,
3125                                   SDValue(Ld, 0), SDValue(0,0));
3126      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R1,
3127                                   SDValue(Ld, 1), Chain.getValue(1));
3128
3129      // Remap uses.
3130      SDValue Glue = Chain.getValue(1);
3131      if (!SDValue(N, 0).use_empty()) {
3132        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3133                                                ARM::R0, MVT::i32, Glue);
3134        Glue = Result.getValue(2);
3135        ReplaceUses(SDValue(N, 0), Result);
3136      }
3137      if (!SDValue(N, 1).use_empty()) {
3138        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3139                                                ARM::R1, MVT::i32, Glue);
3140        Glue = Result.getValue(2);
3141        ReplaceUses(SDValue(N, 1), Result);
3142      }
3143
3144      ReplaceUses(SDValue(N, 2), SDValue(Ld, 2));
3145      return NULL;
3146    }
3147
3148    case Intrinsic::arm_strexd: {
3149      DebugLoc dl = N->getDebugLoc();
3150      SDValue Chain = N->getOperand(0);
3151      SDValue Val0 = N->getOperand(2);
3152      SDValue Val1 = N->getOperand(3);
3153      SDValue MemAddr = N->getOperand(4);
3154
3155      // Until there's support for specifing explicit register constraints
3156      // like the use of even/odd register pair, hardcode strexd to always
3157      // use the pair [R2, R3] to hold the i64 (i32, i32) value to be stored.
3158      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R2, Val0,
3159                                   SDValue(0, 0));
3160      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R3, Val1, Chain.getValue(1));
3161
3162      SDValue Glue = Chain.getValue(1);
3163      Val0 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3164                                    ARM::R2, MVT::i32, Glue);
3165      Glue = Val0.getValue(1);
3166      Val1 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3167                                    ARM::R3, MVT::i32, Glue);
3168
3169      // Store exclusive double return a i32 value which is the return status
3170      // of the issued store.
3171      std::vector<EVT> ResTys;
3172      ResTys.push_back(MVT::i32);
3173      ResTys.push_back(MVT::Other);
3174
3175      // place arguments in the right order
3176      SmallVector<SDValue, 7> Ops;
3177      Ops.push_back(Val0);
3178      Ops.push_back(Val1);
3179      Ops.push_back(MemAddr);
3180      Ops.push_back(getAL(CurDAG));
3181      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3182      Ops.push_back(Chain);
3183
3184      unsigned NewOpc = ARM::STREXD;
3185      if (Subtarget->isThumb() && Subtarget->hasThumb2())
3186        NewOpc = ARM::t2STREXD;
3187
3188      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3189                                          Ops.size());
3190      // Transfer memoperands.
3191      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3192      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3193      cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3194
3195      return St;
3196    }
3197
3198    case Intrinsic::arm_neon_vld1: {
3199      static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3200                                           ARM::VLD1d32, ARM::VLD1d64 };
3201      static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3202                                           ARM::VLD1q32, ARM::VLD1q64};
3203      return SelectVLD(N, false, 1, DOpcodes, QOpcodes, 0);
3204    }
3205
3206    case Intrinsic::arm_neon_vld2: {
3207      static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3208                                           ARM::VLD2d32, ARM::VLD1q64 };
3209      static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3210                                           ARM::VLD2q32Pseudo };
3211      return SelectVLD(N, false, 2, DOpcodes, QOpcodes, 0);
3212    }
3213
3214    case Intrinsic::arm_neon_vld3: {
3215      static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3216                                           ARM::VLD3d16Pseudo,
3217                                           ARM::VLD3d32Pseudo,
3218                                           ARM::VLD1d64TPseudo };
3219      static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3220                                            ARM::VLD3q16Pseudo_UPD,
3221                                            ARM::VLD3q32Pseudo_UPD };
3222      static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3223                                            ARM::VLD3q16oddPseudo,
3224                                            ARM::VLD3q32oddPseudo };
3225      return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3226    }
3227
3228    case Intrinsic::arm_neon_vld4: {
3229      static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3230                                           ARM::VLD4d16Pseudo,
3231                                           ARM::VLD4d32Pseudo,
3232                                           ARM::VLD1d64QPseudo };
3233      static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3234                                            ARM::VLD4q16Pseudo_UPD,
3235                                            ARM::VLD4q32Pseudo_UPD };
3236      static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3237                                            ARM::VLD4q16oddPseudo,
3238                                            ARM::VLD4q32oddPseudo };
3239      return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3240    }
3241
3242    case Intrinsic::arm_neon_vld2lane: {
3243      static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3244                                           ARM::VLD2LNd16Pseudo,
3245                                           ARM::VLD2LNd32Pseudo };
3246      static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3247                                           ARM::VLD2LNq32Pseudo };
3248      return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3249    }
3250
3251    case Intrinsic::arm_neon_vld3lane: {
3252      static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3253                                           ARM::VLD3LNd16Pseudo,
3254                                           ARM::VLD3LNd32Pseudo };
3255      static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3256                                           ARM::VLD3LNq32Pseudo };
3257      return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3258    }
3259
3260    case Intrinsic::arm_neon_vld4lane: {
3261      static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3262                                           ARM::VLD4LNd16Pseudo,
3263                                           ARM::VLD4LNd32Pseudo };
3264      static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3265                                           ARM::VLD4LNq32Pseudo };
3266      return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3267    }
3268
3269    case Intrinsic::arm_neon_vst1: {
3270      static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3271                                           ARM::VST1d32, ARM::VST1d64 };
3272      static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3273                                           ARM::VST1q32, ARM::VST1q64 };
3274      return SelectVST(N, false, 1, DOpcodes, QOpcodes, 0);
3275    }
3276
3277    case Intrinsic::arm_neon_vst2: {
3278      static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3279                                           ARM::VST2d32, ARM::VST1q64 };
3280      static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3281                                     ARM::VST2q32Pseudo };
3282      return SelectVST(N, false, 2, DOpcodes, QOpcodes, 0);
3283    }
3284
3285    case Intrinsic::arm_neon_vst3: {
3286      static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3287                                           ARM::VST3d16Pseudo,
3288                                           ARM::VST3d32Pseudo,
3289                                           ARM::VST1d64TPseudo };
3290      static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3291                                            ARM::VST3q16Pseudo_UPD,
3292                                            ARM::VST3q32Pseudo_UPD };
3293      static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3294                                            ARM::VST3q16oddPseudo,
3295                                            ARM::VST3q32oddPseudo };
3296      return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3297    }
3298
3299    case Intrinsic::arm_neon_vst4: {
3300      static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3301                                           ARM::VST4d16Pseudo,
3302                                           ARM::VST4d32Pseudo,
3303                                           ARM::VST1d64QPseudo };
3304      static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3305                                            ARM::VST4q16Pseudo_UPD,
3306                                            ARM::VST4q32Pseudo_UPD };
3307      static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3308                                            ARM::VST4q16oddPseudo,
3309                                            ARM::VST4q32oddPseudo };
3310      return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3311    }
3312
3313    case Intrinsic::arm_neon_vst2lane: {
3314      static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3315                                           ARM::VST2LNd16Pseudo,
3316                                           ARM::VST2LNd32Pseudo };
3317      static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3318                                           ARM::VST2LNq32Pseudo };
3319      return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3320    }
3321
3322    case Intrinsic::arm_neon_vst3lane: {
3323      static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3324                                           ARM::VST3LNd16Pseudo,
3325                                           ARM::VST3LNd32Pseudo };
3326      static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3327                                           ARM::VST3LNq32Pseudo };
3328      return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3329    }
3330
3331    case Intrinsic::arm_neon_vst4lane: {
3332      static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3333                                           ARM::VST4LNd16Pseudo,
3334                                           ARM::VST4LNd32Pseudo };
3335      static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3336                                           ARM::VST4LNq32Pseudo };
3337      return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3338    }
3339    }
3340    break;
3341  }
3342
3343  case ISD::INTRINSIC_WO_CHAIN: {
3344    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3345    switch (IntNo) {
3346    default:
3347      break;
3348
3349    case Intrinsic::arm_neon_vtbl2:
3350      return SelectVTBL(N, false, 2, ARM::VTBL2);
3351    case Intrinsic::arm_neon_vtbl3:
3352      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
3353    case Intrinsic::arm_neon_vtbl4:
3354      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
3355
3356    case Intrinsic::arm_neon_vtbx2:
3357      return SelectVTBL(N, true, 2, ARM::VTBX2);
3358    case Intrinsic::arm_neon_vtbx3:
3359      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
3360    case Intrinsic::arm_neon_vtbx4:
3361      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
3362    }
3363    break;
3364  }
3365
3366  case ARMISD::VTBL1: {
3367    DebugLoc dl = N->getDebugLoc();
3368    EVT VT = N->getValueType(0);
3369    SmallVector<SDValue, 6> Ops;
3370
3371    Ops.push_back(N->getOperand(0));
3372    Ops.push_back(N->getOperand(1));
3373    Ops.push_back(getAL(CurDAG));                    // Predicate
3374    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3375    return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops.data(), Ops.size());
3376  }
3377  case ARMISD::VTBL2: {
3378    DebugLoc dl = N->getDebugLoc();
3379    EVT VT = N->getValueType(0);
3380
3381    // Form a REG_SEQUENCE to force register allocation.
3382    SDValue V0 = N->getOperand(0);
3383    SDValue V1 = N->getOperand(1);
3384    SDValue RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
3385
3386    SmallVector<SDValue, 6> Ops;
3387    Ops.push_back(RegSeq);
3388    Ops.push_back(N->getOperand(2));
3389    Ops.push_back(getAL(CurDAG));                    // Predicate
3390    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3391    return CurDAG->getMachineNode(ARM::VTBL2, dl, VT,
3392                                  Ops.data(), Ops.size());
3393  }
3394
3395  case ISD::CONCAT_VECTORS:
3396    return SelectConcatVector(N);
3397
3398  case ARMISD::ATOMOR64_DAG:
3399    return SelectAtomic64(N, ARM::ATOMOR6432);
3400  case ARMISD::ATOMXOR64_DAG:
3401    return SelectAtomic64(N, ARM::ATOMXOR6432);
3402  case ARMISD::ATOMADD64_DAG:
3403    return SelectAtomic64(N, ARM::ATOMADD6432);
3404  case ARMISD::ATOMSUB64_DAG:
3405    return SelectAtomic64(N, ARM::ATOMSUB6432);
3406  case ARMISD::ATOMNAND64_DAG:
3407    return SelectAtomic64(N, ARM::ATOMNAND6432);
3408  case ARMISD::ATOMAND64_DAG:
3409    return SelectAtomic64(N, ARM::ATOMAND6432);
3410  case ARMISD::ATOMSWAP64_DAG:
3411    return SelectAtomic64(N, ARM::ATOMSWAP6432);
3412  case ARMISD::ATOMCMPXCHG64_DAG:
3413    return SelectAtomic64(N, ARM::ATOMCMPXCHG6432);
3414  }
3415
3416  return SelectCode(N);
3417}
3418
3419bool ARMDAGToDAGISel::
3420SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
3421                             std::vector<SDValue> &OutOps) {
3422  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
3423  // Require the address to be in a register.  That is safe for all ARM
3424  // variants and it is hard to do anything much smarter without knowing
3425  // how the operand is used.
3426  OutOps.push_back(Op);
3427  return false;
3428}
3429
3430/// createARMISelDag - This pass converts a legalized DAG into a
3431/// ARM-specific DAG, ready for instruction scheduling.
3432///
3433FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3434                                     CodeGenOpt::Level OptLevel) {
3435  return new ARMDAGToDAGISel(TM, OptLevel);
3436}
3437