ARMISelDAGToDAG.cpp revision e38993f89288f8dd96451fe3ba514950520757ad 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMBaseInstrInfo.h"
17#include "ARMTargetMachine.h"
18#include "MCTargetDesc/ARMAddressingModes.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
37
38using namespace llvm;
39
40static cl::opt<bool>
41DisableShifterOp("disable-shifter-op", cl::Hidden,
42  cl::desc("Disable isel of shifter-op"),
43  cl::init(false));
44
45static cl::opt<bool>
46CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
47  cl::desc("Check fp vmla / vmls hazard at isel time"),
48  cl::init(true));
49
50static cl::opt<bool>
51DisableARMIntABS("disable-arm-int-abs", cl::Hidden,
52  cl::desc("Enable / disable ARM integer abs transform"),
53  cl::init(false));
54
55//===--------------------------------------------------------------------===//
56/// ARMDAGToDAGISel - ARM specific code to select ARM machine
57/// instructions for SelectionDAG operations.
58///
59namespace {
60
61enum AddrMode2Type {
62  AM2_BASE, // Simple AM2 (+-imm12)
63  AM2_SHOP  // Shifter-op AM2
64};
65
66class ARMDAGToDAGISel : public SelectionDAGISel {
67  ARMBaseTargetMachine &TM;
68  const ARMBaseInstrInfo *TII;
69
70  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
71  /// make the right decision when generating code for different targets.
72  const ARMSubtarget *Subtarget;
73
74public:
75  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
76                           CodeGenOpt::Level OptLevel)
77    : SelectionDAGISel(tm, OptLevel), TM(tm),
78      TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())),
79      Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
80  }
81
82  virtual const char *getPassName() const {
83    return "ARM Instruction Selection";
84  }
85
86  /// getI32Imm - Return a target constant of type i32 with the specified
87  /// value.
88  inline SDValue getI32Imm(unsigned Imm) {
89    return CurDAG->getTargetConstant(Imm, MVT::i32);
90  }
91
92  SDNode *Select(SDNode *N);
93
94
95  bool hasNoVMLxHazardUse(SDNode *N) const;
96  bool isShifterOpProfitable(const SDValue &Shift,
97                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
98  bool SelectRegShifterOperand(SDValue N, SDValue &A,
99                               SDValue &B, SDValue &C,
100                               bool CheckProfitability = true);
101  bool SelectImmShifterOperand(SDValue N, SDValue &A,
102                               SDValue &B, bool CheckProfitability = true);
103  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
104                                    SDValue &B, SDValue &C) {
105    // Don't apply the profitability check
106    return SelectRegShifterOperand(N, A, B, C, false);
107  }
108  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
109                                    SDValue &B) {
110    // Don't apply the profitability check
111    return SelectImmShifterOperand(N, A, B, false);
112  }
113
114  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
115  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
116
117  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
118                                      SDValue &Offset, SDValue &Opc);
119  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
120                           SDValue &Opc) {
121    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
122  }
123
124  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
125                           SDValue &Opc) {
126    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
127  }
128
129  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
130                       SDValue &Opc) {
131    SelectAddrMode2Worker(N, Base, Offset, Opc);
132//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
133    // This always matches one way or another.
134    return true;
135  }
136
137  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
138                             SDValue &Offset, SDValue &Opc);
139  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
140                             SDValue &Offset, SDValue &Opc);
141  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
142                             SDValue &Offset, SDValue &Opc);
143  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
144  bool SelectAddrMode3(SDValue N, SDValue &Base,
145                       SDValue &Offset, SDValue &Opc);
146  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
147                             SDValue &Offset, SDValue &Opc);
148  bool SelectAddrMode5(SDValue N, SDValue &Base,
149                       SDValue &Offset);
150  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
151  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
152
153  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
154
155  // Thumb Addressing Modes:
156  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
157  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
158                             unsigned Scale);
159  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
160  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
161  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
162  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
163                                SDValue &OffImm);
164  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
165                                 SDValue &OffImm);
166  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
167                                 SDValue &OffImm);
168  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
169                                 SDValue &OffImm);
170  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
171
172  // Thumb 2 Addressing Modes:
173  bool SelectT2ShifterOperandReg(SDValue N,
174                                 SDValue &BaseReg, SDValue &Opc);
175  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
176  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
177                            SDValue &OffImm);
178  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
179                                 SDValue &OffImm);
180  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
181                             SDValue &OffReg, SDValue &ShImm);
182
183  inline bool is_so_imm(unsigned Imm) const {
184    return ARM_AM::getSOImmVal(Imm) != -1;
185  }
186
187  inline bool is_so_imm_not(unsigned Imm) const {
188    return ARM_AM::getSOImmVal(~Imm) != -1;
189  }
190
191  inline bool is_t2_so_imm(unsigned Imm) const {
192    return ARM_AM::getT2SOImmVal(Imm) != -1;
193  }
194
195  inline bool is_t2_so_imm_not(unsigned Imm) const {
196    return ARM_AM::getT2SOImmVal(~Imm) != -1;
197  }
198
199  // Include the pieces autogenerated from the target description.
200#include "ARMGenDAGISel.inc"
201
202private:
203  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
204  /// ARM.
205  SDNode *SelectARMIndexedLoad(SDNode *N);
206  SDNode *SelectT2IndexedLoad(SDNode *N);
207
208  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
209  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
210  /// loads of D registers and even subregs and odd subregs of Q registers.
211  /// For NumVecs <= 2, QOpcodes1 is not used.
212  SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
213                    unsigned *DOpcodes,
214                    unsigned *QOpcodes0, unsigned *QOpcodes1);
215
216  /// SelectVST - Select NEON store intrinsics.  NumVecs should
217  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
218  /// stores of D registers and even subregs and odd subregs of Q registers.
219  /// For NumVecs <= 2, QOpcodes1 is not used.
220  SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
221                    unsigned *DOpcodes,
222                    unsigned *QOpcodes0, unsigned *QOpcodes1);
223
224  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
225  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
226  /// load/store of D registers and Q registers.
227  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
228                          bool isUpdating, unsigned NumVecs,
229                          unsigned *DOpcodes, unsigned *QOpcodes);
230
231  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
232  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
233  /// for loading D registers.  (Q registers are not supported.)
234  SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
235                       unsigned *Opcodes);
236
237  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
238  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
239  /// generated to force the table registers to be consecutive.
240  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
241
242  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
243  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
244
245  /// SelectCMOVOp - Select CMOV instructions for ARM.
246  SDNode *SelectCMOVOp(SDNode *N);
247  SDNode *SelectConditionalOp(SDNode *N);
248  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
249                              ARMCC::CondCodes CCVal, SDValue CCR,
250                              SDValue InFlag);
251  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
252                               ARMCC::CondCodes CCVal, SDValue CCR,
253                               SDValue InFlag);
254  SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
255                              ARMCC::CondCodes CCVal, SDValue CCR,
256                              SDValue InFlag);
257  SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
258                               ARMCC::CondCodes CCVal, SDValue CCR,
259                               SDValue InFlag);
260
261  // Select special operations if node forms integer ABS pattern
262  SDNode *SelectABSOp(SDNode *N);
263
264  SDNode *SelectConcatVector(SDNode *N);
265
266  SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
267
268  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
269  /// inline asm expressions.
270  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
271                                            char ConstraintCode,
272                                            std::vector<SDValue> &OutOps);
273
274  // Form pairs of consecutive S, D, or Q registers.
275  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
276  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
277  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
278
279  // Form sequences of 4 consecutive S, D, or Q registers.
280  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
281  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
282  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
283
284  // Get the alignment operand for a NEON VLD or VST instruction.
285  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
286};
287}
288
289/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
290/// operand. If so Imm will receive the 32-bit value.
291static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
292  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
293    Imm = cast<ConstantSDNode>(N)->getZExtValue();
294    return true;
295  }
296  return false;
297}
298
299// isInt32Immediate - This method tests to see if a constant operand.
300// If so Imm will receive the 32 bit value.
301static bool isInt32Immediate(SDValue N, unsigned &Imm) {
302  return isInt32Immediate(N.getNode(), Imm);
303}
304
305// isOpcWithIntImmediate - This method tests to see if the node is a specific
306// opcode and that it has a immediate integer right operand.
307// If so Imm will receive the 32 bit value.
308static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
309  return N->getOpcode() == Opc &&
310         isInt32Immediate(N->getOperand(1).getNode(), Imm);
311}
312
313/// \brief Check whether a particular node is a constant value representable as
314/// (N * Scale) where (N in [\arg RangeMin, \arg RangeMax).
315///
316/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
317static bool isScaledConstantInRange(SDValue Node, int Scale,
318                                    int RangeMin, int RangeMax,
319                                    int &ScaledConstant) {
320  assert(Scale > 0 && "Invalid scale!");
321
322  // Check that this is a constant.
323  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
324  if (!C)
325    return false;
326
327  ScaledConstant = (int) C->getZExtValue();
328  if ((ScaledConstant % Scale) != 0)
329    return false;
330
331  ScaledConstant /= Scale;
332  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
333}
334
335/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
336/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
337/// least on current ARM implementations) which should be avoidded.
338bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
339  if (OptLevel == CodeGenOpt::None)
340    return true;
341
342  if (!CheckVMLxHazard)
343    return true;
344
345  if (!Subtarget->isCortexA8() && !Subtarget->isCortexA9())
346    return true;
347
348  if (!N->hasOneUse())
349    return false;
350
351  SDNode *Use = *N->use_begin();
352  if (Use->getOpcode() == ISD::CopyToReg)
353    return true;
354  if (Use->isMachineOpcode()) {
355    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
356    if (MCID.mayStore())
357      return true;
358    unsigned Opcode = MCID.getOpcode();
359    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
360      return true;
361    // vmlx feeding into another vmlx. We actually want to unfold
362    // the use later in the MLxExpansion pass. e.g.
363    // vmla
364    // vmla (stall 8 cycles)
365    //
366    // vmul (5 cycles)
367    // vadd (5 cycles)
368    // vmla
369    // This adds up to about 18 - 19 cycles.
370    //
371    // vmla
372    // vmul (stall 4 cycles)
373    // vadd adds up to about 14 cycles.
374    return TII->isFpMLxInstruction(Opcode);
375  }
376
377  return false;
378}
379
380bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
381                                            ARM_AM::ShiftOpc ShOpcVal,
382                                            unsigned ShAmt) {
383  if (!Subtarget->isCortexA9())
384    return true;
385  if (Shift.hasOneUse())
386    return true;
387  // R << 2 is free.
388  return ShOpcVal == ARM_AM::lsl && ShAmt == 2;
389}
390
391bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
392                                              SDValue &BaseReg,
393                                              SDValue &Opc,
394                                              bool CheckProfitability) {
395  if (DisableShifterOp)
396    return false;
397
398  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
399
400  // Don't match base register only case. That is matched to a separate
401  // lower complexity pattern with explicit register operand.
402  if (ShOpcVal == ARM_AM::no_shift) return false;
403
404  BaseReg = N.getOperand(0);
405  unsigned ShImmVal = 0;
406  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
407  if (!RHS) return false;
408  ShImmVal = RHS->getZExtValue() & 31;
409  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
410                                  MVT::i32);
411  return true;
412}
413
414bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
415                                              SDValue &BaseReg,
416                                              SDValue &ShReg,
417                                              SDValue &Opc,
418                                              bool CheckProfitability) {
419  if (DisableShifterOp)
420    return false;
421
422  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
423
424  // Don't match base register only case. That is matched to a separate
425  // lower complexity pattern with explicit register operand.
426  if (ShOpcVal == ARM_AM::no_shift) return false;
427
428  BaseReg = N.getOperand(0);
429  unsigned ShImmVal = 0;
430  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
431  if (RHS) return false;
432
433  ShReg = N.getOperand(1);
434  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
435    return false;
436  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
437                                  MVT::i32);
438  return true;
439}
440
441
442bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
443                                          SDValue &Base,
444                                          SDValue &OffImm) {
445  // Match simple R + imm12 operands.
446
447  // Base only.
448  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
449      !CurDAG->isBaseWithConstantOffset(N)) {
450    if (N.getOpcode() == ISD::FrameIndex) {
451      // Match frame index.
452      int FI = cast<FrameIndexSDNode>(N)->getIndex();
453      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
454      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
455      return true;
456    }
457
458    if (N.getOpcode() == ARMISD::Wrapper &&
459        !(Subtarget->useMovt() &&
460                     N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
461      Base = N.getOperand(0);
462    } else
463      Base = N;
464    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
465    return true;
466  }
467
468  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
469    int RHSC = (int)RHS->getZExtValue();
470    if (N.getOpcode() == ISD::SUB)
471      RHSC = -RHSC;
472
473    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
474      Base   = N.getOperand(0);
475      if (Base.getOpcode() == ISD::FrameIndex) {
476        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
477        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
478      }
479      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
480      return true;
481    }
482  }
483
484  // Base only.
485  Base = N;
486  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
487  return true;
488}
489
490
491
492bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
493                                      SDValue &Opc) {
494  if (N.getOpcode() == ISD::MUL &&
495      (!Subtarget->isCortexA9() || N.hasOneUse())) {
496    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
497      // X * [3,5,9] -> X + X * [2,4,8] etc.
498      int RHSC = (int)RHS->getZExtValue();
499      if (RHSC & 1) {
500        RHSC = RHSC & ~1;
501        ARM_AM::AddrOpc AddSub = ARM_AM::add;
502        if (RHSC < 0) {
503          AddSub = ARM_AM::sub;
504          RHSC = - RHSC;
505        }
506        if (isPowerOf2_32(RHSC)) {
507          unsigned ShAmt = Log2_32(RHSC);
508          Base = Offset = N.getOperand(0);
509          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
510                                                            ARM_AM::lsl),
511                                          MVT::i32);
512          return true;
513        }
514      }
515    }
516  }
517
518  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
519      // ISD::OR that is equivalent to an ISD::ADD.
520      !CurDAG->isBaseWithConstantOffset(N))
521    return false;
522
523  // Leave simple R +/- imm12 operands for LDRi12
524  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
525    int RHSC;
526    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
527                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
528      return false;
529  }
530
531  // Otherwise this is R +/- [possibly shifted] R.
532  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
533  ARM_AM::ShiftOpc ShOpcVal =
534    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
535  unsigned ShAmt = 0;
536
537  Base   = N.getOperand(0);
538  Offset = N.getOperand(1);
539
540  if (ShOpcVal != ARM_AM::no_shift) {
541    // Check to see if the RHS of the shift is a constant, if not, we can't fold
542    // it.
543    if (ConstantSDNode *Sh =
544           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
545      ShAmt = Sh->getZExtValue();
546      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
547        Offset = N.getOperand(1).getOperand(0);
548      else {
549        ShAmt = 0;
550        ShOpcVal = ARM_AM::no_shift;
551      }
552    } else {
553      ShOpcVal = ARM_AM::no_shift;
554    }
555  }
556
557  // Try matching (R shl C) + (R).
558  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
559      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
560    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
561    if (ShOpcVal != ARM_AM::no_shift) {
562      // Check to see if the RHS of the shift is a constant, if not, we can't
563      // fold it.
564      if (ConstantSDNode *Sh =
565          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
566        ShAmt = Sh->getZExtValue();
567        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
568          Offset = N.getOperand(0).getOperand(0);
569          Base = N.getOperand(1);
570        } else {
571          ShAmt = 0;
572          ShOpcVal = ARM_AM::no_shift;
573        }
574      } else {
575        ShOpcVal = ARM_AM::no_shift;
576      }
577    }
578  }
579
580  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
581                                  MVT::i32);
582  return true;
583}
584
585
586//-----
587
588AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
589                                                     SDValue &Base,
590                                                     SDValue &Offset,
591                                                     SDValue &Opc) {
592  if (N.getOpcode() == ISD::MUL &&
593      (!Subtarget->isCortexA9() || N.hasOneUse())) {
594    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
595      // X * [3,5,9] -> X + X * [2,4,8] etc.
596      int RHSC = (int)RHS->getZExtValue();
597      if (RHSC & 1) {
598        RHSC = RHSC & ~1;
599        ARM_AM::AddrOpc AddSub = ARM_AM::add;
600        if (RHSC < 0) {
601          AddSub = ARM_AM::sub;
602          RHSC = - RHSC;
603        }
604        if (isPowerOf2_32(RHSC)) {
605          unsigned ShAmt = Log2_32(RHSC);
606          Base = Offset = N.getOperand(0);
607          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
608                                                            ARM_AM::lsl),
609                                          MVT::i32);
610          return AM2_SHOP;
611        }
612      }
613    }
614  }
615
616  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
617      // ISD::OR that is equivalent to an ADD.
618      !CurDAG->isBaseWithConstantOffset(N)) {
619    Base = N;
620    if (N.getOpcode() == ISD::FrameIndex) {
621      int FI = cast<FrameIndexSDNode>(N)->getIndex();
622      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
623    } else if (N.getOpcode() == ARMISD::Wrapper &&
624               !(Subtarget->useMovt() &&
625                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
626      Base = N.getOperand(0);
627    }
628    Offset = CurDAG->getRegister(0, MVT::i32);
629    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
630                                                      ARM_AM::no_shift),
631                                    MVT::i32);
632    return AM2_BASE;
633  }
634
635  // Match simple R +/- imm12 operands.
636  if (N.getOpcode() != ISD::SUB) {
637    int RHSC;
638    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
639                                -0x1000+1, 0x1000, RHSC)) { // 12 bits.
640      Base = N.getOperand(0);
641      if (Base.getOpcode() == ISD::FrameIndex) {
642        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
643        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
644      }
645      Offset = CurDAG->getRegister(0, MVT::i32);
646
647      ARM_AM::AddrOpc AddSub = ARM_AM::add;
648      if (RHSC < 0) {
649        AddSub = ARM_AM::sub;
650        RHSC = - RHSC;
651      }
652      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
653                                                        ARM_AM::no_shift),
654                                      MVT::i32);
655      return AM2_BASE;
656    }
657  }
658
659  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
660    // Compute R +/- (R << N) and reuse it.
661    Base = N;
662    Offset = CurDAG->getRegister(0, MVT::i32);
663    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
664                                                      ARM_AM::no_shift),
665                                    MVT::i32);
666    return AM2_BASE;
667  }
668
669  // Otherwise this is R +/- [possibly shifted] R.
670  ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
671  ARM_AM::ShiftOpc ShOpcVal =
672    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
673  unsigned ShAmt = 0;
674
675  Base   = N.getOperand(0);
676  Offset = N.getOperand(1);
677
678  if (ShOpcVal != ARM_AM::no_shift) {
679    // Check to see if the RHS of the shift is a constant, if not, we can't fold
680    // it.
681    if (ConstantSDNode *Sh =
682           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
683      ShAmt = Sh->getZExtValue();
684      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
685        Offset = N.getOperand(1).getOperand(0);
686      else {
687        ShAmt = 0;
688        ShOpcVal = ARM_AM::no_shift;
689      }
690    } else {
691      ShOpcVal = ARM_AM::no_shift;
692    }
693  }
694
695  // Try matching (R shl C) + (R).
696  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
697      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
698    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
699    if (ShOpcVal != ARM_AM::no_shift) {
700      // Check to see if the RHS of the shift is a constant, if not, we can't
701      // fold it.
702      if (ConstantSDNode *Sh =
703          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
704        ShAmt = Sh->getZExtValue();
705        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
706          Offset = N.getOperand(0).getOperand(0);
707          Base = N.getOperand(1);
708        } else {
709          ShAmt = 0;
710          ShOpcVal = ARM_AM::no_shift;
711        }
712      } else {
713        ShOpcVal = ARM_AM::no_shift;
714      }
715    }
716  }
717
718  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
719                                  MVT::i32);
720  return AM2_SHOP;
721}
722
723bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
724                                            SDValue &Offset, SDValue &Opc) {
725  unsigned Opcode = Op->getOpcode();
726  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
727    ? cast<LoadSDNode>(Op)->getAddressingMode()
728    : cast<StoreSDNode>(Op)->getAddressingMode();
729  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
730    ? ARM_AM::add : ARM_AM::sub;
731  int Val;
732  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
733    return false;
734
735  Offset = N;
736  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
737  unsigned ShAmt = 0;
738  if (ShOpcVal != ARM_AM::no_shift) {
739    // Check to see if the RHS of the shift is a constant, if not, we can't fold
740    // it.
741    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
742      ShAmt = Sh->getZExtValue();
743      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
744        Offset = N.getOperand(0);
745      else {
746        ShAmt = 0;
747        ShOpcVal = ARM_AM::no_shift;
748      }
749    } else {
750      ShOpcVal = ARM_AM::no_shift;
751    }
752  }
753
754  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
755                                  MVT::i32);
756  return true;
757}
758
759bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
760                                            SDValue &Offset, SDValue &Opc) {
761  unsigned Opcode = Op->getOpcode();
762  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
763    ? cast<LoadSDNode>(Op)->getAddressingMode()
764    : cast<StoreSDNode>(Op)->getAddressingMode();
765  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
766    ? ARM_AM::add : ARM_AM::sub;
767  int Val;
768  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
769    if (AddSub == ARM_AM::sub) Val *= -1;
770    Offset = CurDAG->getRegister(0, MVT::i32);
771    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
772    return true;
773  }
774
775  return false;
776}
777
778
779bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
780                                            SDValue &Offset, SDValue &Opc) {
781  unsigned Opcode = Op->getOpcode();
782  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
783    ? cast<LoadSDNode>(Op)->getAddressingMode()
784    : cast<StoreSDNode>(Op)->getAddressingMode();
785  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
786    ? ARM_AM::add : ARM_AM::sub;
787  int Val;
788  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
789    Offset = CurDAG->getRegister(0, MVT::i32);
790    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
791                                                      ARM_AM::no_shift),
792                                    MVT::i32);
793    return true;
794  }
795
796  return false;
797}
798
799bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
800  Base = N;
801  return true;
802}
803
804bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
805                                      SDValue &Base, SDValue &Offset,
806                                      SDValue &Opc) {
807  if (N.getOpcode() == ISD::SUB) {
808    // X - C  is canonicalize to X + -C, no need to handle it here.
809    Base = N.getOperand(0);
810    Offset = N.getOperand(1);
811    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
812    return true;
813  }
814
815  if (!CurDAG->isBaseWithConstantOffset(N)) {
816    Base = N;
817    if (N.getOpcode() == ISD::FrameIndex) {
818      int FI = cast<FrameIndexSDNode>(N)->getIndex();
819      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
820    }
821    Offset = CurDAG->getRegister(0, MVT::i32);
822    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
823    return true;
824  }
825
826  // If the RHS is +/- imm8, fold into addr mode.
827  int RHSC;
828  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
829                              -256 + 1, 256, RHSC)) { // 8 bits.
830    Base = N.getOperand(0);
831    if (Base.getOpcode() == ISD::FrameIndex) {
832      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
833      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
834    }
835    Offset = CurDAG->getRegister(0, MVT::i32);
836
837    ARM_AM::AddrOpc AddSub = ARM_AM::add;
838    if (RHSC < 0) {
839      AddSub = ARM_AM::sub;
840      RHSC = -RHSC;
841    }
842    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
843    return true;
844  }
845
846  Base = N.getOperand(0);
847  Offset = N.getOperand(1);
848  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
849  return true;
850}
851
852bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
853                                            SDValue &Offset, SDValue &Opc) {
854  unsigned Opcode = Op->getOpcode();
855  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
856    ? cast<LoadSDNode>(Op)->getAddressingMode()
857    : cast<StoreSDNode>(Op)->getAddressingMode();
858  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
859    ? ARM_AM::add : ARM_AM::sub;
860  int Val;
861  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
862    Offset = CurDAG->getRegister(0, MVT::i32);
863    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
864    return true;
865  }
866
867  Offset = N;
868  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
869  return true;
870}
871
872bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
873                                      SDValue &Base, SDValue &Offset) {
874  if (!CurDAG->isBaseWithConstantOffset(N)) {
875    Base = N;
876    if (N.getOpcode() == ISD::FrameIndex) {
877      int FI = cast<FrameIndexSDNode>(N)->getIndex();
878      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
879    } else if (N.getOpcode() == ARMISD::Wrapper &&
880               !(Subtarget->useMovt() &&
881                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
882      Base = N.getOperand(0);
883    }
884    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
885                                       MVT::i32);
886    return true;
887  }
888
889  // If the RHS is +/- imm8, fold into addr mode.
890  int RHSC;
891  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
892                              -256 + 1, 256, RHSC)) {
893    Base = N.getOperand(0);
894    if (Base.getOpcode() == ISD::FrameIndex) {
895      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
896      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
897    }
898
899    ARM_AM::AddrOpc AddSub = ARM_AM::add;
900    if (RHSC < 0) {
901      AddSub = ARM_AM::sub;
902      RHSC = -RHSC;
903    }
904    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
905                                       MVT::i32);
906    return true;
907  }
908
909  Base = N;
910  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
911                                     MVT::i32);
912  return true;
913}
914
915bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
916                                      SDValue &Align) {
917  Addr = N;
918
919  unsigned Alignment = 0;
920  if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
921    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
922    // The maximum alignment is equal to the memory size being referenced.
923    unsigned LSNAlign = LSN->getAlignment();
924    unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
925    if (LSNAlign >= MemSize && MemSize > 1)
926      Alignment = MemSize;
927  } else {
928    // All other uses of addrmode6 are for intrinsics.  For now just record
929    // the raw alignment value; it will be refined later based on the legal
930    // alignment operands for the intrinsic.
931    Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
932  }
933
934  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
935  return true;
936}
937
938bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
939                                            SDValue &Offset) {
940  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
941  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
942  if (AM != ISD::POST_INC)
943    return false;
944  Offset = N;
945  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
946    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
947      Offset = CurDAG->getRegister(0, MVT::i32);
948  }
949  return true;
950}
951
952bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
953                                       SDValue &Offset, SDValue &Label) {
954  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
955    Offset = N.getOperand(0);
956    SDValue N1 = N.getOperand(1);
957    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
958                                      MVT::i32);
959    return true;
960  }
961
962  return false;
963}
964
965
966//===----------------------------------------------------------------------===//
967//                         Thumb Addressing Modes
968//===----------------------------------------------------------------------===//
969
970bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
971                                            SDValue &Base, SDValue &Offset){
972  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
973    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
974    if (!NC || !NC->isNullValue())
975      return false;
976
977    Base = Offset = N;
978    return true;
979  }
980
981  Base = N.getOperand(0);
982  Offset = N.getOperand(1);
983  return true;
984}
985
986bool
987ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
988                                       SDValue &Offset, unsigned Scale) {
989  if (Scale == 4) {
990    SDValue TmpBase, TmpOffImm;
991    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
992      return false;  // We want to select tLDRspi / tSTRspi instead.
993
994    if (N.getOpcode() == ARMISD::Wrapper &&
995        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
996      return false;  // We want to select tLDRpci instead.
997  }
998
999  if (!CurDAG->isBaseWithConstantOffset(N))
1000    return false;
1001
1002  // Thumb does not have [sp, r] address mode.
1003  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1004  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1005  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1006      (RHSR && RHSR->getReg() == ARM::SP))
1007    return false;
1008
1009  // FIXME: Why do we explicitly check for a match here and then return false?
1010  // Presumably to allow something else to match, but shouldn't this be
1011  // documented?
1012  int RHSC;
1013  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
1014    return false;
1015
1016  Base = N.getOperand(0);
1017  Offset = N.getOperand(1);
1018  return true;
1019}
1020
1021bool
1022ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
1023                                          SDValue &Base,
1024                                          SDValue &Offset) {
1025  return SelectThumbAddrModeRI(N, Base, Offset, 1);
1026}
1027
1028bool
1029ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
1030                                          SDValue &Base,
1031                                          SDValue &Offset) {
1032  return SelectThumbAddrModeRI(N, Base, Offset, 2);
1033}
1034
1035bool
1036ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
1037                                          SDValue &Base,
1038                                          SDValue &Offset) {
1039  return SelectThumbAddrModeRI(N, Base, Offset, 4);
1040}
1041
1042bool
1043ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1044                                          SDValue &Base, SDValue &OffImm) {
1045  if (Scale == 4) {
1046    SDValue TmpBase, TmpOffImm;
1047    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1048      return false;  // We want to select tLDRspi / tSTRspi instead.
1049
1050    if (N.getOpcode() == ARMISD::Wrapper &&
1051        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1052      return false;  // We want to select tLDRpci instead.
1053  }
1054
1055  if (!CurDAG->isBaseWithConstantOffset(N)) {
1056    if (N.getOpcode() == ARMISD::Wrapper &&
1057        !(Subtarget->useMovt() &&
1058          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1059      Base = N.getOperand(0);
1060    } else {
1061      Base = N;
1062    }
1063
1064    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1065    return true;
1066  }
1067
1068  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1069  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1070  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1071      (RHSR && RHSR->getReg() == ARM::SP)) {
1072    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1073    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1074    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1075    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1076
1077    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1078    if (LHSC != 0 || RHSC != 0) return false;
1079
1080    Base = N;
1081    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1082    return true;
1083  }
1084
1085  // If the RHS is + imm5 * scale, fold into addr mode.
1086  int RHSC;
1087  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1088    Base = N.getOperand(0);
1089    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1090    return true;
1091  }
1092
1093  Base = N.getOperand(0);
1094  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1095  return true;
1096}
1097
1098bool
1099ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1100                                           SDValue &OffImm) {
1101  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1102}
1103
1104bool
1105ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1106                                           SDValue &OffImm) {
1107  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1108}
1109
1110bool
1111ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1112                                           SDValue &OffImm) {
1113  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1114}
1115
1116bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1117                                            SDValue &Base, SDValue &OffImm) {
1118  if (N.getOpcode() == ISD::FrameIndex) {
1119    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1120    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1121    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1122    return true;
1123  }
1124
1125  if (!CurDAG->isBaseWithConstantOffset(N))
1126    return false;
1127
1128  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1129  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1130      (LHSR && LHSR->getReg() == ARM::SP)) {
1131    // If the RHS is + imm8 * scale, fold into addr mode.
1132    int RHSC;
1133    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1134      Base = N.getOperand(0);
1135      if (Base.getOpcode() == ISD::FrameIndex) {
1136        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1137        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1138      }
1139      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1140      return true;
1141    }
1142  }
1143
1144  return false;
1145}
1146
1147
1148//===----------------------------------------------------------------------===//
1149//                        Thumb 2 Addressing Modes
1150//===----------------------------------------------------------------------===//
1151
1152
1153bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1154                                                SDValue &Opc) {
1155  if (DisableShifterOp)
1156    return false;
1157
1158  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1159
1160  // Don't match base register only case. That is matched to a separate
1161  // lower complexity pattern with explicit register operand.
1162  if (ShOpcVal == ARM_AM::no_shift) return false;
1163
1164  BaseReg = N.getOperand(0);
1165  unsigned ShImmVal = 0;
1166  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1167    ShImmVal = RHS->getZExtValue() & 31;
1168    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1169    return true;
1170  }
1171
1172  return false;
1173}
1174
1175bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1176                                            SDValue &Base, SDValue &OffImm) {
1177  // Match simple R + imm12 operands.
1178
1179  // Base only.
1180  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1181      !CurDAG->isBaseWithConstantOffset(N)) {
1182    if (N.getOpcode() == ISD::FrameIndex) {
1183      // Match frame index.
1184      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1185      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1186      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1187      return true;
1188    }
1189
1190    if (N.getOpcode() == ARMISD::Wrapper &&
1191               !(Subtarget->useMovt() &&
1192                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1193      Base = N.getOperand(0);
1194      if (Base.getOpcode() == ISD::TargetConstantPool)
1195        return false;  // We want to select t2LDRpci instead.
1196    } else
1197      Base = N;
1198    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1199    return true;
1200  }
1201
1202  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1203    if (SelectT2AddrModeImm8(N, Base, OffImm))
1204      // Let t2LDRi8 handle (R - imm8).
1205      return false;
1206
1207    int RHSC = (int)RHS->getZExtValue();
1208    if (N.getOpcode() == ISD::SUB)
1209      RHSC = -RHSC;
1210
1211    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1212      Base   = N.getOperand(0);
1213      if (Base.getOpcode() == ISD::FrameIndex) {
1214        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1215        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1216      }
1217      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1218      return true;
1219    }
1220  }
1221
1222  // Base only.
1223  Base = N;
1224  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1225  return true;
1226}
1227
1228bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1229                                           SDValue &Base, SDValue &OffImm) {
1230  // Match simple R - imm8 operands.
1231  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1232      !CurDAG->isBaseWithConstantOffset(N))
1233    return false;
1234
1235  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1236    int RHSC = (int)RHS->getSExtValue();
1237    if (N.getOpcode() == ISD::SUB)
1238      RHSC = -RHSC;
1239
1240    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1241      Base = N.getOperand(0);
1242      if (Base.getOpcode() == ISD::FrameIndex) {
1243        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1244        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1245      }
1246      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1247      return true;
1248    }
1249  }
1250
1251  return false;
1252}
1253
1254bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1255                                                 SDValue &OffImm){
1256  unsigned Opcode = Op->getOpcode();
1257  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1258    ? cast<LoadSDNode>(Op)->getAddressingMode()
1259    : cast<StoreSDNode>(Op)->getAddressingMode();
1260  int RHSC;
1261  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1262    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1263      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1264      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1265    return true;
1266  }
1267
1268  return false;
1269}
1270
1271bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1272                                            SDValue &Base,
1273                                            SDValue &OffReg, SDValue &ShImm) {
1274  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1275  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1276    return false;
1277
1278  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1279  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1280    int RHSC = (int)RHS->getZExtValue();
1281    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1282      return false;
1283    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1284      return false;
1285  }
1286
1287  // Look for (R + R) or (R + (R << [1,2,3])).
1288  unsigned ShAmt = 0;
1289  Base   = N.getOperand(0);
1290  OffReg = N.getOperand(1);
1291
1292  // Swap if it is ((R << c) + R).
1293  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1294  if (ShOpcVal != ARM_AM::lsl) {
1295    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1296    if (ShOpcVal == ARM_AM::lsl)
1297      std::swap(Base, OffReg);
1298  }
1299
1300  if (ShOpcVal == ARM_AM::lsl) {
1301    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1302    // it.
1303    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1304      ShAmt = Sh->getZExtValue();
1305      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1306        OffReg = OffReg.getOperand(0);
1307      else {
1308        ShAmt = 0;
1309        ShOpcVal = ARM_AM::no_shift;
1310      }
1311    } else {
1312      ShOpcVal = ARM_AM::no_shift;
1313    }
1314  }
1315
1316  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1317
1318  return true;
1319}
1320
1321//===--------------------------------------------------------------------===//
1322
1323/// getAL - Returns a ARMCC::AL immediate node.
1324static inline SDValue getAL(SelectionDAG *CurDAG) {
1325  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1326}
1327
1328SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1329  LoadSDNode *LD = cast<LoadSDNode>(N);
1330  ISD::MemIndexedMode AM = LD->getAddressingMode();
1331  if (AM == ISD::UNINDEXED)
1332    return NULL;
1333
1334  EVT LoadedVT = LD->getMemoryVT();
1335  SDValue Offset, AMOpc;
1336  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1337  unsigned Opcode = 0;
1338  bool Match = false;
1339  if (LoadedVT == MVT::i32 && isPre &&
1340      SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1341    Opcode = ARM::LDR_PRE_IMM;
1342    Match = true;
1343  } else if (LoadedVT == MVT::i32 && !isPre &&
1344      SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1345    Opcode = ARM::LDR_POST_IMM;
1346    Match = true;
1347  } else if (LoadedVT == MVT::i32 &&
1348      SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1349    Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1350    Match = true;
1351
1352  } else if (LoadedVT == MVT::i16 &&
1353             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1354    Match = true;
1355    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1356      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1357      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1358  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1359    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1360      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1361        Match = true;
1362        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1363      }
1364    } else {
1365      if (isPre &&
1366          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1367        Match = true;
1368        Opcode = ARM::LDRB_PRE_IMM;
1369      } else if (!isPre &&
1370                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1371        Match = true;
1372        Opcode = ARM::LDRB_POST_IMM;
1373      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1374        Match = true;
1375        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1376      }
1377    }
1378  }
1379
1380  if (Match) {
1381    if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1382      SDValue Chain = LD->getChain();
1383      SDValue Base = LD->getBasePtr();
1384      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
1385                       CurDAG->getRegister(0, MVT::i32), Chain };
1386      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1387                                    MVT::i32, MVT::Other, Ops, 5);
1388    } else {
1389      SDValue Chain = LD->getChain();
1390      SDValue Base = LD->getBasePtr();
1391      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1392                       CurDAG->getRegister(0, MVT::i32), Chain };
1393      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1394                                    MVT::i32, MVT::Other, Ops, 6);
1395    }
1396  }
1397
1398  return NULL;
1399}
1400
1401SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1402  LoadSDNode *LD = cast<LoadSDNode>(N);
1403  ISD::MemIndexedMode AM = LD->getAddressingMode();
1404  if (AM == ISD::UNINDEXED)
1405    return NULL;
1406
1407  EVT LoadedVT = LD->getMemoryVT();
1408  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1409  SDValue Offset;
1410  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1411  unsigned Opcode = 0;
1412  bool Match = false;
1413  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1414    switch (LoadedVT.getSimpleVT().SimpleTy) {
1415    case MVT::i32:
1416      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1417      break;
1418    case MVT::i16:
1419      if (isSExtLd)
1420        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1421      else
1422        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1423      break;
1424    case MVT::i8:
1425    case MVT::i1:
1426      if (isSExtLd)
1427        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1428      else
1429        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1430      break;
1431    default:
1432      return NULL;
1433    }
1434    Match = true;
1435  }
1436
1437  if (Match) {
1438    SDValue Chain = LD->getChain();
1439    SDValue Base = LD->getBasePtr();
1440    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1441                     CurDAG->getRegister(0, MVT::i32), Chain };
1442    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1443                                  MVT::Other, Ops, 5);
1444  }
1445
1446  return NULL;
1447}
1448
1449/// PairSRegs - Form a D register from a pair of S registers.
1450///
1451SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
1452  DebugLoc dl = V0.getNode()->getDebugLoc();
1453  SDValue RegClass =
1454    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1455  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1456  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1457  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1458  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1459}
1460
1461/// PairDRegs - Form a quad register from a pair of D registers.
1462///
1463SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
1464  DebugLoc dl = V0.getNode()->getDebugLoc();
1465  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1466  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1467  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1468  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1469  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1470}
1471
1472/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
1473///
1474SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
1475  DebugLoc dl = V0.getNode()->getDebugLoc();
1476  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1477  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1478  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1479  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1480  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1481}
1482
1483/// QuadSRegs - Form 4 consecutive S registers.
1484///
1485SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
1486                                   SDValue V2, SDValue V3) {
1487  DebugLoc dl = V0.getNode()->getDebugLoc();
1488  SDValue RegClass =
1489    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1490  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1491  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1492  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1493  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1494  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1495                                    V2, SubReg2, V3, SubReg3 };
1496  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1497}
1498
1499/// QuadDRegs - Form 4 consecutive D registers.
1500///
1501SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
1502                                   SDValue V2, SDValue V3) {
1503  DebugLoc dl = V0.getNode()->getDebugLoc();
1504  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1505  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1506  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1507  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1508  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1509  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1510                                    V2, SubReg2, V3, SubReg3 };
1511  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1512}
1513
1514/// QuadQRegs - Form 4 consecutive Q registers.
1515///
1516SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
1517                                   SDValue V2, SDValue V3) {
1518  DebugLoc dl = V0.getNode()->getDebugLoc();
1519  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1520  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1521  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1522  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1523  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1524  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1525                                    V2, SubReg2, V3, SubReg3 };
1526  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1527}
1528
1529/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1530/// of a NEON VLD or VST instruction.  The supported values depend on the
1531/// number of registers being loaded.
1532SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1533                                       bool is64BitVector) {
1534  unsigned NumRegs = NumVecs;
1535  if (!is64BitVector && NumVecs < 3)
1536    NumRegs *= 2;
1537
1538  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1539  if (Alignment >= 32 && NumRegs == 4)
1540    Alignment = 32;
1541  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1542    Alignment = 16;
1543  else if (Alignment >= 8)
1544    Alignment = 8;
1545  else
1546    Alignment = 0;
1547
1548  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1549}
1550
1551// Get the register stride update opcode of a VLD/VST instruction that
1552// is otherwise equivalent to the given fixed stride updating instruction.
1553static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1554  switch (Opc) {
1555  default: break;
1556  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1557  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1558  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1559  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1560  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1561  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1562  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1563  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1564
1565  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1566  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1567  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1568  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1569  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1570  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1571  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1572  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1573  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1574  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1575
1576  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1577  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1578  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1579  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1580  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1581  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1582
1583  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1584  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1585  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1586  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1587  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1588  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1589
1590  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1591  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1592  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1593  }
1594  return Opc; // If not one we handle, return it unchanged.
1595}
1596
1597SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1598                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1599                                   unsigned *QOpcodes1) {
1600  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1601  DebugLoc dl = N->getDebugLoc();
1602
1603  SDValue MemAddr, Align;
1604  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1605  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1606    return NULL;
1607
1608  SDValue Chain = N->getOperand(0);
1609  EVT VT = N->getValueType(0);
1610  bool is64BitVector = VT.is64BitVector();
1611  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1612
1613  unsigned OpcodeIndex;
1614  switch (VT.getSimpleVT().SimpleTy) {
1615  default: llvm_unreachable("unhandled vld type");
1616    // Double-register operations:
1617  case MVT::v8i8:  OpcodeIndex = 0; break;
1618  case MVT::v4i16: OpcodeIndex = 1; break;
1619  case MVT::v2f32:
1620  case MVT::v2i32: OpcodeIndex = 2; break;
1621  case MVT::v1i64: OpcodeIndex = 3; break;
1622    // Quad-register operations:
1623  case MVT::v16i8: OpcodeIndex = 0; break;
1624  case MVT::v8i16: OpcodeIndex = 1; break;
1625  case MVT::v4f32:
1626  case MVT::v4i32: OpcodeIndex = 2; break;
1627  case MVT::v2i64: OpcodeIndex = 3;
1628    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1629    break;
1630  }
1631
1632  EVT ResTy;
1633  if (NumVecs == 1)
1634    ResTy = VT;
1635  else {
1636    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1637    if (!is64BitVector)
1638      ResTyElts *= 2;
1639    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1640  }
1641  std::vector<EVT> ResTys;
1642  ResTys.push_back(ResTy);
1643  if (isUpdating)
1644    ResTys.push_back(MVT::i32);
1645  ResTys.push_back(MVT::Other);
1646
1647  SDValue Pred = getAL(CurDAG);
1648  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1649  SDNode *VLd;
1650  SmallVector<SDValue, 7> Ops;
1651
1652  // Double registers and VLD1/VLD2 quad registers are directly supported.
1653  if (is64BitVector || NumVecs <= 2) {
1654    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1655                    QOpcodes0[OpcodeIndex]);
1656    Ops.push_back(MemAddr);
1657    Ops.push_back(Align);
1658    if (isUpdating) {
1659      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1660      // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
1661      // case entirely when the rest are updated to that form, too.
1662      if ((NumVecs == 1 || NumVecs == 2) && !isa<ConstantSDNode>(Inc.getNode()))
1663        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1664      // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1665      // check for that explicitly too. Horribly hacky, but temporary.
1666      if ((NumVecs != 1 && NumVecs != 2 && Opc != ARM::VLD1q64wb_fixed) ||
1667          !isa<ConstantSDNode>(Inc.getNode()))
1668        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1669    }
1670    Ops.push_back(Pred);
1671    Ops.push_back(Reg0);
1672    Ops.push_back(Chain);
1673    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1674
1675  } else {
1676    // Otherwise, quad registers are loaded with two separate instructions,
1677    // where one loads the even registers and the other loads the odd registers.
1678    EVT AddrTy = MemAddr.getValueType();
1679
1680    // Load the even subregs.  This is always an updating load, so that it
1681    // provides the address to the second load for the odd subregs.
1682    SDValue ImplDef =
1683      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1684    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1685    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1686                                          ResTy, AddrTy, MVT::Other, OpsA, 7);
1687    Chain = SDValue(VLdA, 2);
1688
1689    // Load the odd subregs.
1690    Ops.push_back(SDValue(VLdA, 1));
1691    Ops.push_back(Align);
1692    if (isUpdating) {
1693      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1694      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1695             "only constant post-increment update allowed for VLD3/4");
1696      (void)Inc;
1697      Ops.push_back(Reg0);
1698    }
1699    Ops.push_back(SDValue(VLdA, 0));
1700    Ops.push_back(Pred);
1701    Ops.push_back(Reg0);
1702    Ops.push_back(Chain);
1703    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1704                                 Ops.data(), Ops.size());
1705  }
1706
1707  // Transfer memoperands.
1708  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1709  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1710  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1711
1712  if (NumVecs == 1)
1713    return VLd;
1714
1715  // Extract out the subregisters.
1716  SDValue SuperReg = SDValue(VLd, 0);
1717  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1718         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1719  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1720  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1721    ReplaceUses(SDValue(N, Vec),
1722                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1723  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1724  if (isUpdating)
1725    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1726  return NULL;
1727}
1728
1729SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1730                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1731                                   unsigned *QOpcodes1) {
1732  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1733  DebugLoc dl = N->getDebugLoc();
1734
1735  SDValue MemAddr, Align;
1736  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1737  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1738  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1739    return NULL;
1740
1741  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1742  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1743
1744  SDValue Chain = N->getOperand(0);
1745  EVT VT = N->getOperand(Vec0Idx).getValueType();
1746  bool is64BitVector = VT.is64BitVector();
1747  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1748
1749  unsigned OpcodeIndex;
1750  switch (VT.getSimpleVT().SimpleTy) {
1751  default: llvm_unreachable("unhandled vst type");
1752    // Double-register operations:
1753  case MVT::v8i8:  OpcodeIndex = 0; break;
1754  case MVT::v4i16: OpcodeIndex = 1; break;
1755  case MVT::v2f32:
1756  case MVT::v2i32: OpcodeIndex = 2; break;
1757  case MVT::v1i64: OpcodeIndex = 3; break;
1758    // Quad-register operations:
1759  case MVT::v16i8: OpcodeIndex = 0; break;
1760  case MVT::v8i16: OpcodeIndex = 1; break;
1761  case MVT::v4f32:
1762  case MVT::v4i32: OpcodeIndex = 2; break;
1763  case MVT::v2i64: OpcodeIndex = 3;
1764    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1765    break;
1766  }
1767
1768  std::vector<EVT> ResTys;
1769  if (isUpdating)
1770    ResTys.push_back(MVT::i32);
1771  ResTys.push_back(MVT::Other);
1772
1773  SDValue Pred = getAL(CurDAG);
1774  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1775  SmallVector<SDValue, 7> Ops;
1776
1777  // Double registers and VST1/VST2 quad registers are directly supported.
1778  if (is64BitVector || NumVecs <= 2) {
1779    SDValue SrcReg;
1780    if (NumVecs == 1) {
1781      SrcReg = N->getOperand(Vec0Idx);
1782    } else if (is64BitVector) {
1783      // Form a REG_SEQUENCE to force register allocation.
1784      SDValue V0 = N->getOperand(Vec0Idx + 0);
1785      SDValue V1 = N->getOperand(Vec0Idx + 1);
1786      if (NumVecs == 2)
1787        SrcReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1788      else {
1789        SDValue V2 = N->getOperand(Vec0Idx + 2);
1790        // If it's a vst3, form a quad D-register and leave the last part as
1791        // an undef.
1792        SDValue V3 = (NumVecs == 3)
1793          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1794          : N->getOperand(Vec0Idx + 3);
1795        SrcReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1796      }
1797    } else {
1798      // Form a QQ register.
1799      SDValue Q0 = N->getOperand(Vec0Idx);
1800      SDValue Q1 = N->getOperand(Vec0Idx + 1);
1801      SrcReg = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1802    }
1803
1804    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1805                    QOpcodes0[OpcodeIndex]);
1806    Ops.push_back(MemAddr);
1807    Ops.push_back(Align);
1808    if (isUpdating) {
1809      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1810      // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
1811      // case entirely when the rest are updated to that form, too.
1812      if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
1813        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1814      // We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
1815      // check for that explicitly too. Horribly hacky, but temporary.
1816      if ((NumVecs > 2 && Opc != ARM::VST1q64wb_fixed) ||
1817          !isa<ConstantSDNode>(Inc.getNode()))
1818        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1819    }
1820    Ops.push_back(SrcReg);
1821    Ops.push_back(Pred);
1822    Ops.push_back(Reg0);
1823    Ops.push_back(Chain);
1824    SDNode *VSt =
1825      CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1826
1827    // Transfer memoperands.
1828    cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
1829
1830    return VSt;
1831  }
1832
1833  // Otherwise, quad registers are stored with two separate instructions,
1834  // where one stores the even registers and the other stores the odd registers.
1835
1836  // Form the QQQQ REG_SEQUENCE.
1837  SDValue V0 = N->getOperand(Vec0Idx + 0);
1838  SDValue V1 = N->getOperand(Vec0Idx + 1);
1839  SDValue V2 = N->getOperand(Vec0Idx + 2);
1840  SDValue V3 = (NumVecs == 3)
1841    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1842    : N->getOperand(Vec0Idx + 3);
1843  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1844
1845  // Store the even D registers.  This is always an updating store, so that it
1846  // provides the address to the second store for the odd subregs.
1847  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
1848  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1849                                        MemAddr.getValueType(),
1850                                        MVT::Other, OpsA, 7);
1851  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
1852  Chain = SDValue(VStA, 1);
1853
1854  // Store the odd D registers.
1855  Ops.push_back(SDValue(VStA, 0));
1856  Ops.push_back(Align);
1857  if (isUpdating) {
1858    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1859    assert(isa<ConstantSDNode>(Inc.getNode()) &&
1860           "only constant post-increment update allowed for VST3/4");
1861    (void)Inc;
1862    Ops.push_back(Reg0);
1863  }
1864  Ops.push_back(RegSeq);
1865  Ops.push_back(Pred);
1866  Ops.push_back(Reg0);
1867  Ops.push_back(Chain);
1868  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1869                                        Ops.data(), Ops.size());
1870  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
1871  return VStB;
1872}
1873
1874SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1875                                         bool isUpdating, unsigned NumVecs,
1876                                         unsigned *DOpcodes,
1877                                         unsigned *QOpcodes) {
1878  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1879  DebugLoc dl = N->getDebugLoc();
1880
1881  SDValue MemAddr, Align;
1882  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1883  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1884  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1885    return NULL;
1886
1887  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1888  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1889
1890  SDValue Chain = N->getOperand(0);
1891  unsigned Lane =
1892    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
1893  EVT VT = N->getOperand(Vec0Idx).getValueType();
1894  bool is64BitVector = VT.is64BitVector();
1895
1896  unsigned Alignment = 0;
1897  if (NumVecs != 3) {
1898    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1899    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1900    if (Alignment > NumBytes)
1901      Alignment = NumBytes;
1902    if (Alignment < 8 && Alignment < NumBytes)
1903      Alignment = 0;
1904    // Alignment must be a power of two; make sure of that.
1905    Alignment = (Alignment & -Alignment);
1906    if (Alignment == 1)
1907      Alignment = 0;
1908  }
1909  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1910
1911  unsigned OpcodeIndex;
1912  switch (VT.getSimpleVT().SimpleTy) {
1913  default: llvm_unreachable("unhandled vld/vst lane type");
1914    // Double-register operations:
1915  case MVT::v8i8:  OpcodeIndex = 0; break;
1916  case MVT::v4i16: OpcodeIndex = 1; break;
1917  case MVT::v2f32:
1918  case MVT::v2i32: OpcodeIndex = 2; break;
1919    // Quad-register operations:
1920  case MVT::v8i16: OpcodeIndex = 0; break;
1921  case MVT::v4f32:
1922  case MVT::v4i32: OpcodeIndex = 1; break;
1923  }
1924
1925  std::vector<EVT> ResTys;
1926  if (IsLoad) {
1927    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1928    if (!is64BitVector)
1929      ResTyElts *= 2;
1930    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
1931                                      MVT::i64, ResTyElts));
1932  }
1933  if (isUpdating)
1934    ResTys.push_back(MVT::i32);
1935  ResTys.push_back(MVT::Other);
1936
1937  SDValue Pred = getAL(CurDAG);
1938  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1939
1940  SmallVector<SDValue, 8> Ops;
1941  Ops.push_back(MemAddr);
1942  Ops.push_back(Align);
1943  if (isUpdating) {
1944    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1945    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1946  }
1947
1948  SDValue SuperReg;
1949  SDValue V0 = N->getOperand(Vec0Idx + 0);
1950  SDValue V1 = N->getOperand(Vec0Idx + 1);
1951  if (NumVecs == 2) {
1952    if (is64BitVector)
1953      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1954    else
1955      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1956  } else {
1957    SDValue V2 = N->getOperand(Vec0Idx + 2);
1958    SDValue V3 = (NumVecs == 3)
1959      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1960      : N->getOperand(Vec0Idx + 3);
1961    if (is64BitVector)
1962      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1963    else
1964      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1965  }
1966  Ops.push_back(SuperReg);
1967  Ops.push_back(getI32Imm(Lane));
1968  Ops.push_back(Pred);
1969  Ops.push_back(Reg0);
1970  Ops.push_back(Chain);
1971
1972  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1973                                  QOpcodes[OpcodeIndex]);
1974  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys,
1975                                         Ops.data(), Ops.size());
1976  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
1977  if (!IsLoad)
1978    return VLdLn;
1979
1980  // Extract the subregisters.
1981  SuperReg = SDValue(VLdLn, 0);
1982  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1983         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1984  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1985  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1986    ReplaceUses(SDValue(N, Vec),
1987                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1988  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
1989  if (isUpdating)
1990    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
1991  return NULL;
1992}
1993
1994SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
1995                                      unsigned NumVecs, unsigned *Opcodes) {
1996  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
1997  DebugLoc dl = N->getDebugLoc();
1998
1999  SDValue MemAddr, Align;
2000  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2001    return NULL;
2002
2003  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2004  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2005
2006  SDValue Chain = N->getOperand(0);
2007  EVT VT = N->getValueType(0);
2008
2009  unsigned Alignment = 0;
2010  if (NumVecs != 3) {
2011    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2012    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2013    if (Alignment > NumBytes)
2014      Alignment = NumBytes;
2015    if (Alignment < 8 && Alignment < NumBytes)
2016      Alignment = 0;
2017    // Alignment must be a power of two; make sure of that.
2018    Alignment = (Alignment & -Alignment);
2019    if (Alignment == 1)
2020      Alignment = 0;
2021  }
2022  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2023
2024  unsigned OpcodeIndex;
2025  switch (VT.getSimpleVT().SimpleTy) {
2026  default: llvm_unreachable("unhandled vld-dup type");
2027  case MVT::v8i8:  OpcodeIndex = 0; break;
2028  case MVT::v4i16: OpcodeIndex = 1; break;
2029  case MVT::v2f32:
2030  case MVT::v2i32: OpcodeIndex = 2; break;
2031  }
2032
2033  SDValue Pred = getAL(CurDAG);
2034  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2035  SDValue SuperReg;
2036  unsigned Opc = Opcodes[OpcodeIndex];
2037  SmallVector<SDValue, 6> Ops;
2038  Ops.push_back(MemAddr);
2039  Ops.push_back(Align);
2040  if (isUpdating) {
2041    // fixed-stride update instructions don't have an explicit writeback
2042    // operand. It's implicit in the opcode itself.
2043    SDValue Inc = N->getOperand(2);
2044    if (!isa<ConstantSDNode>(Inc.getNode()))
2045      Ops.push_back(Inc);
2046    // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2047    else if (NumVecs > 2)
2048      Ops.push_back(Reg0);
2049  }
2050  Ops.push_back(Pred);
2051  Ops.push_back(Reg0);
2052  Ops.push_back(Chain);
2053
2054  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2055  std::vector<EVT> ResTys;
2056  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2057  if (isUpdating)
2058    ResTys.push_back(MVT::i32);
2059  ResTys.push_back(MVT::Other);
2060  SDNode *VLdDup =
2061    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
2062  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2063  SuperReg = SDValue(VLdDup, 0);
2064
2065  // Extract the subregisters.
2066  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
2067  unsigned SubIdx = ARM::dsub_0;
2068  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2069    ReplaceUses(SDValue(N, Vec),
2070                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2071  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2072  if (isUpdating)
2073    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2074  return NULL;
2075}
2076
2077SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
2078                                    unsigned Opc) {
2079  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
2080  DebugLoc dl = N->getDebugLoc();
2081  EVT VT = N->getValueType(0);
2082  unsigned FirstTblReg = IsExt ? 2 : 1;
2083
2084  // Form a REG_SEQUENCE to force register allocation.
2085  SDValue RegSeq;
2086  SDValue V0 = N->getOperand(FirstTblReg + 0);
2087  SDValue V1 = N->getOperand(FirstTblReg + 1);
2088  if (NumVecs == 2)
2089    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
2090  else {
2091    SDValue V2 = N->getOperand(FirstTblReg + 2);
2092    // If it's a vtbl3, form a quad D-register and leave the last part as
2093    // an undef.
2094    SDValue V3 = (NumVecs == 3)
2095      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2096      : N->getOperand(FirstTblReg + 3);
2097    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
2098  }
2099
2100  SmallVector<SDValue, 6> Ops;
2101  if (IsExt)
2102    Ops.push_back(N->getOperand(1));
2103  Ops.push_back(RegSeq);
2104  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
2105  Ops.push_back(getAL(CurDAG)); // predicate
2106  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
2107  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
2108}
2109
2110SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
2111                                                     bool isSigned) {
2112  if (!Subtarget->hasV6T2Ops())
2113    return NULL;
2114
2115  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2116    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2117
2118
2119  // For unsigned extracts, check for a shift right and mask
2120  unsigned And_imm = 0;
2121  if (N->getOpcode() == ISD::AND) {
2122    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2123
2124      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2125      if (And_imm & (And_imm + 1))
2126        return NULL;
2127
2128      unsigned Srl_imm = 0;
2129      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2130                                Srl_imm)) {
2131        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2132
2133        // Note: The width operand is encoded as width-1.
2134        unsigned Width = CountTrailingOnes_32(And_imm) - 1;
2135        unsigned LSB = Srl_imm;
2136        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2137        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2138                          CurDAG->getTargetConstant(LSB, MVT::i32),
2139                          CurDAG->getTargetConstant(Width, MVT::i32),
2140          getAL(CurDAG), Reg0 };
2141        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2142      }
2143    }
2144    return NULL;
2145  }
2146
2147  // Otherwise, we're looking for a shift of a shift
2148  unsigned Shl_imm = 0;
2149  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2150    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2151    unsigned Srl_imm = 0;
2152    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2153      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2154      // Note: The width operand is encoded as width-1.
2155      unsigned Width = 32 - Srl_imm - 1;
2156      int LSB = Srl_imm - Shl_imm;
2157      if (LSB < 0)
2158        return NULL;
2159      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2160      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2161                        CurDAG->getTargetConstant(LSB, MVT::i32),
2162                        CurDAG->getTargetConstant(Width, MVT::i32),
2163                        getAL(CurDAG), Reg0 };
2164      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2165    }
2166  }
2167  return NULL;
2168}
2169
2170SDNode *ARMDAGToDAGISel::
2171SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2172                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2173  SDValue CPTmp0;
2174  SDValue CPTmp1;
2175  if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
2176    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
2177    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
2178    unsigned Opc = 0;
2179    switch (SOShOp) {
2180    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
2181    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
2182    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
2183    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
2184    default:
2185      llvm_unreachable("Unknown so_reg opcode!");
2186    }
2187    SDValue SOShImm =
2188      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
2189    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2190    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
2191    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
2192  }
2193  return 0;
2194}
2195
2196SDNode *ARMDAGToDAGISel::
2197SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2198                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2199  SDValue CPTmp0;
2200  SDValue CPTmp1;
2201  SDValue CPTmp2;
2202  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
2203    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2204    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag };
2205    return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6);
2206  }
2207
2208  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
2209    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2210    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
2211    return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7);
2212  }
2213  return 0;
2214}
2215
2216SDNode *ARMDAGToDAGISel::
2217SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2218                  ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2219  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2220  if (!T)
2221    return 0;
2222
2223  unsigned Opc = 0;
2224  unsigned TrueImm = T->getZExtValue();
2225  if (is_t2_so_imm(TrueImm)) {
2226    Opc = ARM::t2MOVCCi;
2227  } else if (TrueImm <= 0xffff) {
2228    Opc = ARM::t2MOVCCi16;
2229  } else if (is_t2_so_imm_not(TrueImm)) {
2230    TrueImm = ~TrueImm;
2231    Opc = ARM::t2MVNCCi;
2232  } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) {
2233    // Large immediate.
2234    Opc = ARM::t2MOVCCi32imm;
2235  }
2236
2237  if (Opc) {
2238    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2239    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2240    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2241    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2242  }
2243
2244  return 0;
2245}
2246
2247SDNode *ARMDAGToDAGISel::
2248SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2249                   ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2250  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2251  if (!T)
2252    return 0;
2253
2254  unsigned Opc = 0;
2255  unsigned TrueImm = T->getZExtValue();
2256  bool isSoImm = is_so_imm(TrueImm);
2257  if (isSoImm) {
2258    Opc = ARM::MOVCCi;
2259  } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) {
2260    Opc = ARM::MOVCCi16;
2261  } else if (is_so_imm_not(TrueImm)) {
2262    TrueImm = ~TrueImm;
2263    Opc = ARM::MVNCCi;
2264  } else if (TrueVal.getNode()->hasOneUse() &&
2265             (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) {
2266    // Large immediate.
2267    Opc = ARM::MOVCCi32imm;
2268  }
2269
2270  if (Opc) {
2271    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2272    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2273    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2274    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2275  }
2276
2277  return 0;
2278}
2279
2280SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
2281  EVT VT = N->getValueType(0);
2282  SDValue FalseVal = N->getOperand(0);
2283  SDValue TrueVal  = N->getOperand(1);
2284  SDValue CC = N->getOperand(2);
2285  SDValue CCR = N->getOperand(3);
2286  SDValue InFlag = N->getOperand(4);
2287  assert(CC.getOpcode() == ISD::Constant);
2288  assert(CCR.getOpcode() == ISD::Register);
2289  ARMCC::CondCodes CCVal =
2290    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
2291
2292  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
2293    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2294    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2295    // Pattern complexity = 18  cost = 1  size = 0
2296    if (Subtarget->isThumb()) {
2297      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
2298                                        CCVal, CCR, InFlag);
2299      if (!Res)
2300        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
2301                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2302      if (Res)
2303        return Res;
2304    } else {
2305      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
2306                                         CCVal, CCR, InFlag);
2307      if (!Res)
2308        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
2309                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2310      if (Res)
2311        return Res;
2312    }
2313
2314    // Pattern: (ARMcmov:i32 GPR:i32:$false,
2315    //             (imm:i32)<<P:Pred_so_imm>>:$true,
2316    //             (imm:i32):$cc)
2317    // Emits: (MOVCCi:i32 GPR:i32:$false,
2318    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
2319    // Pattern complexity = 10  cost = 1  size = 0
2320    if (Subtarget->isThumb()) {
2321      SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal,
2322                                        CCVal, CCR, InFlag);
2323      if (!Res)
2324        Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal,
2325                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2326      if (Res)
2327        return Res;
2328    } else {
2329      SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal,
2330                                         CCVal, CCR, InFlag);
2331      if (!Res)
2332        Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal,
2333                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2334      if (Res)
2335        return Res;
2336    }
2337  }
2338
2339  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2340  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2341  // Pattern complexity = 6  cost = 1  size = 0
2342  //
2343  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2344  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2345  // Pattern complexity = 6  cost = 11  size = 0
2346  //
2347  // Also VMOVScc and VMOVDcc.
2348  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
2349  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
2350  unsigned Opc = 0;
2351  switch (VT.getSimpleVT().SimpleTy) {
2352  default: llvm_unreachable("Illegal conditional move type!");
2353  case MVT::i32:
2354    Opc = Subtarget->isThumb()
2355      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
2356      : ARM::MOVCCr;
2357    break;
2358  case MVT::f32:
2359    Opc = ARM::VMOVScc;
2360    break;
2361  case MVT::f64:
2362    Opc = ARM::VMOVDcc;
2363    break;
2364  }
2365  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2366}
2367
2368SDNode *ARMDAGToDAGISel::SelectConditionalOp(SDNode *N) {
2369  SDValue FalseVal = N->getOperand(0);
2370  SDValue TrueVal  = N->getOperand(1);
2371  ARMCC::CondCodes CCVal =
2372    (ARMCC::CondCodes)cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
2373  SDValue CCR = N->getOperand(3);
2374  assert(CCR.getOpcode() == ISD::Register);
2375  SDValue InFlag = N->getOperand(4);
2376  SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2377  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2378
2379  if (Subtarget->isThumb()) {
2380    SDValue CPTmp0;
2381    SDValue CPTmp1;
2382    if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
2383      unsigned Opc;
2384      switch (N->getOpcode()) {
2385      default: llvm_unreachable("Unexpected node");
2386      case ARMISD::CAND: Opc = ARM::t2ANDCCrs; break;
2387      case ARMISD::COR:  Opc = ARM::t2ORRCCrs; break;
2388      case ARMISD::CXOR: Opc = ARM::t2EORCCrs; break;
2389      }
2390      SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CC, CCR, Reg0, InFlag };
2391      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
2392    }
2393
2394    ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2395    if (T) {
2396      unsigned TrueImm = T->getZExtValue();
2397      if (is_t2_so_imm(TrueImm)) {
2398        unsigned Opc;
2399        switch (N->getOpcode()) {
2400        default: llvm_unreachable("Unexpected node");
2401        case ARMISD::CAND: Opc = ARM::t2ANDCCri; break;
2402        case ARMISD::COR:  Opc = ARM::t2ORRCCri; break;
2403        case ARMISD::CXOR: Opc = ARM::t2EORCCri; break;
2404        }
2405        SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2406        SDValue Ops[] = { FalseVal, True, CC, CCR, Reg0, InFlag };
2407        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2408      }
2409    }
2410
2411    unsigned Opc;
2412    switch (N->getOpcode()) {
2413    default: llvm_unreachable("Unexpected node");
2414    case ARMISD::CAND: Opc = ARM::t2ANDCCrr; break;
2415    case ARMISD::COR:  Opc = ARM::t2ORRCCrr; break;
2416    case ARMISD::CXOR: Opc = ARM::t2EORCCrr; break;
2417    }
2418    SDValue Ops[] = { FalseVal, TrueVal, CC, CCR, Reg0, InFlag };
2419    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2420  }
2421
2422  SDValue CPTmp0;
2423  SDValue CPTmp1;
2424  SDValue CPTmp2;
2425  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
2426    unsigned Opc;
2427    switch (N->getOpcode()) {
2428    default: llvm_unreachable("Unexpected node");
2429    case ARMISD::CAND: Opc = ARM::ANDCCrsi; break;
2430    case ARMISD::COR:  Opc = ARM::ORRCCrsi; break;
2431    case ARMISD::CXOR: Opc = ARM::EORCCrsi; break;
2432    }
2433    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, Reg0, InFlag };
2434    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
2435  }
2436
2437  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
2438    unsigned Opc;
2439    switch (N->getOpcode()) {
2440    default: llvm_unreachable("Unexpected node");
2441    case ARMISD::CAND: Opc = ARM::ANDCCrsr; break;
2442    case ARMISD::COR:  Opc = ARM::ORRCCrsr; break;
2443    case ARMISD::CXOR: Opc = ARM::EORCCrsr; break;
2444    }
2445    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, Reg0, InFlag };
2446    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 8);
2447  }
2448
2449  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2450  if (T) {
2451    unsigned TrueImm = T->getZExtValue();
2452    if (is_so_imm(TrueImm)) {
2453      unsigned Opc;
2454      switch (N->getOpcode()) {
2455      default: llvm_unreachable("Unexpected node");
2456      case ARMISD::CAND: Opc = ARM::ANDCCri; break;
2457      case ARMISD::COR:  Opc = ARM::ORRCCri; break;
2458      case ARMISD::CXOR: Opc = ARM::EORCCri; break;
2459      }
2460      SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2461      SDValue Ops[] = { FalseVal, True, CC, CCR, Reg0, InFlag };
2462      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2463    }
2464  }
2465
2466  unsigned Opc;
2467  switch (N->getOpcode()) {
2468  default: llvm_unreachable("Unexpected node");
2469  case ARMISD::CAND: Opc = ARM::ANDCCrr; break;
2470  case ARMISD::COR:  Opc = ARM::ORRCCrr; break;
2471  case ARMISD::CXOR: Opc = ARM::EORCCrr; break;
2472  }
2473  SDValue Ops[] = { FalseVal, TrueVal, CC, CCR, Reg0, InFlag };
2474  return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 6);
2475}
2476
2477/// Target-specific DAG combining for ISD::XOR.
2478/// Target-independent combining lowers SELECT_CC nodes of the form
2479/// select_cc setg[ge] X,  0,  X, -X
2480/// select_cc setgt    X, -1,  X, -X
2481/// select_cc setl[te] X,  0, -X,  X
2482/// select_cc setlt    X,  1, -X,  X
2483/// which represent Integer ABS into:
2484/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2485/// ARM instruction selection detects the latter and matches it to
2486/// ARM::ABS or ARM::t2ABS machine node.
2487SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
2488  SDValue XORSrc0 = N->getOperand(0);
2489  SDValue XORSrc1 = N->getOperand(1);
2490  EVT VT = N->getValueType(0);
2491
2492  if (DisableARMIntABS)
2493    return NULL;
2494
2495  if (Subtarget->isThumb1Only())
2496    return NULL;
2497
2498  if (XORSrc0.getOpcode() != ISD::ADD ||
2499    XORSrc1.getOpcode() != ISD::SRA)
2500    return NULL;
2501
2502  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2503  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2504  SDValue SRASrc0 = XORSrc1.getOperand(0);
2505  SDValue SRASrc1 = XORSrc1.getOperand(1);
2506  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
2507  EVT XType = SRASrc0.getValueType();
2508  unsigned Size = XType.getSizeInBits() - 1;
2509
2510  if (ADDSrc1 == XORSrc1  &&
2511      ADDSrc0 == SRASrc0 &&
2512      XType.isInteger() &&
2513      SRAConstant != NULL &&
2514      Size == SRAConstant->getZExtValue()) {
2515
2516    unsigned Opcode = ARM::ABS;
2517    if (Subtarget->isThumb2())
2518      Opcode = ARM::t2ABS;
2519
2520    return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2521  }
2522
2523  return NULL;
2524}
2525
2526SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2527  // The only time a CONCAT_VECTORS operation can have legal types is when
2528  // two 64-bit vectors are concatenated to a 128-bit vector.
2529  EVT VT = N->getValueType(0);
2530  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2531    llvm_unreachable("unexpected CONCAT_VECTORS");
2532  return PairDRegs(VT, N->getOperand(0), N->getOperand(1));
2533}
2534
2535SDNode *ARMDAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
2536  SmallVector<SDValue, 6> Ops;
2537  Ops.push_back(Node->getOperand(1)); // Ptr
2538  Ops.push_back(Node->getOperand(2)); // Low part of Val1
2539  Ops.push_back(Node->getOperand(3)); // High part of Val1
2540  if (Opc == ARM::ATOMCMPXCHG6432) {
2541    Ops.push_back(Node->getOperand(4)); // Low part of Val2
2542    Ops.push_back(Node->getOperand(5)); // High part of Val2
2543  }
2544  Ops.push_back(Node->getOperand(0)); // Chain
2545  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2546  MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
2547  SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
2548                                           MVT::i32, MVT::i32, MVT::Other,
2549                                           Ops.data() ,Ops.size());
2550  cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1);
2551  return ResNode;
2552}
2553
2554SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2555  DebugLoc dl = N->getDebugLoc();
2556
2557  if (N->isMachineOpcode())
2558    return NULL;   // Already selected.
2559
2560  switch (N->getOpcode()) {
2561  default: break;
2562  case ISD::XOR: {
2563    // Select special operations if XOR node forms integer ABS pattern
2564    SDNode *ResNode = SelectABSOp(N);
2565    if (ResNode)
2566      return ResNode;
2567    // Other cases are autogenerated.
2568    break;
2569  }
2570  case ISD::Constant: {
2571    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2572    bool UseCP = true;
2573    if (Subtarget->hasThumb2())
2574      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2575      // be done with MOV + MOVT, at worst.
2576      UseCP = 0;
2577    else {
2578      if (Subtarget->isThumb()) {
2579        UseCP = (Val > 255 &&                          // MOV
2580                 ~Val > 255 &&                         // MOV + MVN
2581                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
2582      } else
2583        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
2584                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
2585                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
2586    }
2587
2588    if (UseCP) {
2589      SDValue CPIdx =
2590        CurDAG->getTargetConstantPool(ConstantInt::get(
2591                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
2592                                      TLI.getPointerTy());
2593
2594      SDNode *ResNode;
2595      if (Subtarget->isThumb1Only()) {
2596        SDValue Pred = getAL(CurDAG);
2597        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2598        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2599        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2600                                         Ops, 4);
2601      } else {
2602        SDValue Ops[] = {
2603          CPIdx,
2604          CurDAG->getTargetConstant(0, MVT::i32),
2605          getAL(CurDAG),
2606          CurDAG->getRegister(0, MVT::i32),
2607          CurDAG->getEntryNode()
2608        };
2609        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2610                                       Ops, 5);
2611      }
2612      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2613      return NULL;
2614    }
2615
2616    // Other cases are autogenerated.
2617    break;
2618  }
2619  case ISD::FrameIndex: {
2620    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2621    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2622    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
2623    if (Subtarget->isThumb1Only()) {
2624      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2625                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2626      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops, 4);
2627    } else {
2628      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2629                      ARM::t2ADDri : ARM::ADDri);
2630      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2631                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2632                        CurDAG->getRegister(0, MVT::i32) };
2633      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2634    }
2635  }
2636  case ISD::SRL:
2637    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2638      return I;
2639    break;
2640  case ISD::SRA:
2641    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2642      return I;
2643    break;
2644  case ISD::MUL:
2645    if (Subtarget->isThumb1Only())
2646      break;
2647    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2648      unsigned RHSV = C->getZExtValue();
2649      if (!RHSV) break;
2650      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2651        unsigned ShImm = Log2_32(RHSV-1);
2652        if (ShImm >= 32)
2653          break;
2654        SDValue V = N->getOperand(0);
2655        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2656        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2657        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2658        if (Subtarget->isThumb()) {
2659          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2660          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
2661        } else {
2662          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2663          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7);
2664        }
2665      }
2666      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2667        unsigned ShImm = Log2_32(RHSV+1);
2668        if (ShImm >= 32)
2669          break;
2670        SDValue V = N->getOperand(0);
2671        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2672        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2673        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2674        if (Subtarget->isThumb()) {
2675          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2676          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
2677        } else {
2678          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2679          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7);
2680        }
2681      }
2682    }
2683    break;
2684  case ISD::AND: {
2685    // Check for unsigned bitfield extract
2686    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2687      return I;
2688
2689    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2690    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2691    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2692    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2693    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2694    EVT VT = N->getValueType(0);
2695    if (VT != MVT::i32)
2696      break;
2697    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2698      ? ARM::t2MOVTi16
2699      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2700    if (!Opc)
2701      break;
2702    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2703    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2704    if (!N1C)
2705      break;
2706    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2707      SDValue N2 = N0.getOperand(1);
2708      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2709      if (!N2C)
2710        break;
2711      unsigned N1CVal = N1C->getZExtValue();
2712      unsigned N2CVal = N2C->getZExtValue();
2713      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2714          (N1CVal & 0xffffU) == 0xffffU &&
2715          (N2CVal & 0xffffU) == 0x0U) {
2716        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2717                                                  MVT::i32);
2718        SDValue Ops[] = { N0.getOperand(0), Imm16,
2719                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2720        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
2721      }
2722    }
2723    break;
2724  }
2725  case ARMISD::VMOVRRD:
2726    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2727                                  N->getOperand(0), getAL(CurDAG),
2728                                  CurDAG->getRegister(0, MVT::i32));
2729  case ISD::UMUL_LOHI: {
2730    if (Subtarget->isThumb1Only())
2731      break;
2732    if (Subtarget->isThumb()) {
2733      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2734                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2735                        CurDAG->getRegister(0, MVT::i32) };
2736      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
2737    } else {
2738      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2739                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2740                        CurDAG->getRegister(0, MVT::i32) };
2741      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2742                                    ARM::UMULL : ARM::UMULLv5,
2743                                    dl, MVT::i32, MVT::i32, Ops, 5);
2744    }
2745  }
2746  case ISD::SMUL_LOHI: {
2747    if (Subtarget->isThumb1Only())
2748      break;
2749    if (Subtarget->isThumb()) {
2750      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2751                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2752      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
2753    } else {
2754      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2755                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2756                        CurDAG->getRegister(0, MVT::i32) };
2757      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2758                                    ARM::SMULL : ARM::SMULLv5,
2759                                    dl, MVT::i32, MVT::i32, Ops, 5);
2760    }
2761  }
2762  case ISD::LOAD: {
2763    SDNode *ResNode = 0;
2764    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2765      ResNode = SelectT2IndexedLoad(N);
2766    else
2767      ResNode = SelectARMIndexedLoad(N);
2768    if (ResNode)
2769      return ResNode;
2770    // Other cases are autogenerated.
2771    break;
2772  }
2773  case ARMISD::BRCOND: {
2774    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2775    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2776    // Pattern complexity = 6  cost = 1  size = 0
2777
2778    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2779    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2780    // Pattern complexity = 6  cost = 1  size = 0
2781
2782    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2783    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2784    // Pattern complexity = 6  cost = 1  size = 0
2785
2786    unsigned Opc = Subtarget->isThumb() ?
2787      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2788    SDValue Chain = N->getOperand(0);
2789    SDValue N1 = N->getOperand(1);
2790    SDValue N2 = N->getOperand(2);
2791    SDValue N3 = N->getOperand(3);
2792    SDValue InFlag = N->getOperand(4);
2793    assert(N1.getOpcode() == ISD::BasicBlock);
2794    assert(N2.getOpcode() == ISD::Constant);
2795    assert(N3.getOpcode() == ISD::Register);
2796
2797    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2798                               cast<ConstantSDNode>(N2)->getZExtValue()),
2799                               MVT::i32);
2800    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2801    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2802                                             MVT::Glue, Ops, 5);
2803    Chain = SDValue(ResNode, 0);
2804    if (N->getNumValues() == 2) {
2805      InFlag = SDValue(ResNode, 1);
2806      ReplaceUses(SDValue(N, 1), InFlag);
2807    }
2808    ReplaceUses(SDValue(N, 0),
2809                SDValue(Chain.getNode(), Chain.getResNo()));
2810    return NULL;
2811  }
2812  case ARMISD::CMOV:
2813    return SelectCMOVOp(N);
2814  case ARMISD::CAND:
2815  case ARMISD::COR:
2816  case ARMISD::CXOR:
2817    return SelectConditionalOp(N);
2818  case ARMISD::VZIP: {
2819    unsigned Opc = 0;
2820    EVT VT = N->getValueType(0);
2821    switch (VT.getSimpleVT().SimpleTy) {
2822    default: return NULL;
2823    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2824    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2825    case MVT::v2f32:
2826    // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2827    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2828    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2829    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2830    case MVT::v4f32:
2831    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2832    }
2833    SDValue Pred = getAL(CurDAG);
2834    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2835    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2836    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2837  }
2838  case ARMISD::VUZP: {
2839    unsigned Opc = 0;
2840    EVT VT = N->getValueType(0);
2841    switch (VT.getSimpleVT().SimpleTy) {
2842    default: return NULL;
2843    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2844    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2845    case MVT::v2f32:
2846    // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2847    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2848    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2849    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2850    case MVT::v4f32:
2851    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2852    }
2853    SDValue Pred = getAL(CurDAG);
2854    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2855    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2856    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2857  }
2858  case ARMISD::VTRN: {
2859    unsigned Opc = 0;
2860    EVT VT = N->getValueType(0);
2861    switch (VT.getSimpleVT().SimpleTy) {
2862    default: return NULL;
2863    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2864    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2865    case MVT::v2f32:
2866    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2867    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2868    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2869    case MVT::v4f32:
2870    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2871    }
2872    SDValue Pred = getAL(CurDAG);
2873    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2874    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2875    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2876  }
2877  case ARMISD::BUILD_VECTOR: {
2878    EVT VecVT = N->getValueType(0);
2879    EVT EltVT = VecVT.getVectorElementType();
2880    unsigned NumElts = VecVT.getVectorNumElements();
2881    if (EltVT == MVT::f64) {
2882      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2883      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
2884    }
2885    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2886    if (NumElts == 2)
2887      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
2888    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2889    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
2890                     N->getOperand(2), N->getOperand(3));
2891  }
2892
2893  case ARMISD::VLD2DUP: {
2894    unsigned Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
2895                           ARM::VLD2DUPd32 };
2896    return SelectVLDDup(N, false, 2, Opcodes);
2897  }
2898
2899  case ARMISD::VLD3DUP: {
2900    unsigned Opcodes[] = { ARM::VLD3DUPd8Pseudo, ARM::VLD3DUPd16Pseudo,
2901                           ARM::VLD3DUPd32Pseudo };
2902    return SelectVLDDup(N, false, 3, Opcodes);
2903  }
2904
2905  case ARMISD::VLD4DUP: {
2906    unsigned Opcodes[] = { ARM::VLD4DUPd8Pseudo, ARM::VLD4DUPd16Pseudo,
2907                           ARM::VLD4DUPd32Pseudo };
2908    return SelectVLDDup(N, false, 4, Opcodes);
2909  }
2910
2911  case ARMISD::VLD2DUP_UPD: {
2912    unsigned Opcodes[] = { ARM::VLD2DUPd8wb_fixed, ARM::VLD2DUPd16wb_fixed,
2913                           ARM::VLD2DUPd32wb_fixed };
2914    return SelectVLDDup(N, true, 2, Opcodes);
2915  }
2916
2917  case ARMISD::VLD3DUP_UPD: {
2918    unsigned Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, ARM::VLD3DUPd16Pseudo_UPD,
2919                           ARM::VLD3DUPd32Pseudo_UPD };
2920    return SelectVLDDup(N, true, 3, Opcodes);
2921  }
2922
2923  case ARMISD::VLD4DUP_UPD: {
2924    unsigned Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, ARM::VLD4DUPd16Pseudo_UPD,
2925                           ARM::VLD4DUPd32Pseudo_UPD };
2926    return SelectVLDDup(N, true, 4, Opcodes);
2927  }
2928
2929  case ARMISD::VLD1_UPD: {
2930    unsigned DOpcodes[] = { ARM::VLD1d8wb_fixed, ARM::VLD1d16wb_fixed,
2931                            ARM::VLD1d32wb_fixed, ARM::VLD1d64wb_fixed };
2932    unsigned QOpcodes[] = { ARM::VLD1q8wb_fixed,
2933                            ARM::VLD1q16wb_fixed,
2934                            ARM::VLD1q32wb_fixed,
2935                            ARM::VLD1q64wb_fixed };
2936    return SelectVLD(N, true, 1, DOpcodes, QOpcodes, 0);
2937  }
2938
2939  case ARMISD::VLD2_UPD: {
2940    unsigned DOpcodes[] = { ARM::VLD2d8wb_fixed,
2941                            ARM::VLD2d16wb_fixed,
2942                            ARM::VLD2d32wb_fixed,
2943                            ARM::VLD1q64wb_fixed};
2944    unsigned QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
2945                            ARM::VLD2q16PseudoWB_fixed,
2946                            ARM::VLD2q32PseudoWB_fixed };
2947    return SelectVLD(N, true, 2, DOpcodes, QOpcodes, 0);
2948  }
2949
2950  case ARMISD::VLD3_UPD: {
2951    unsigned DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, ARM::VLD3d16Pseudo_UPD,
2952                            ARM::VLD3d32Pseudo_UPD, ARM::VLD1q64wb_fixed};
2953    unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2954                             ARM::VLD3q16Pseudo_UPD,
2955                             ARM::VLD3q32Pseudo_UPD };
2956    unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2957                             ARM::VLD3q16oddPseudo_UPD,
2958                             ARM::VLD3q32oddPseudo_UPD };
2959    return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2960  }
2961
2962  case ARMISD::VLD4_UPD: {
2963    unsigned DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD,
2964                            ARM::VLD4d32Pseudo_UPD, ARM::VLD1q64wb_fixed};
2965    unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2966                             ARM::VLD4q16Pseudo_UPD,
2967                             ARM::VLD4q32Pseudo_UPD };
2968    unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2969                             ARM::VLD4q16oddPseudo_UPD,
2970                             ARM::VLD4q32oddPseudo_UPD };
2971    return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2972  }
2973
2974  case ARMISD::VLD2LN_UPD: {
2975    unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, ARM::VLD2LNd16Pseudo_UPD,
2976                            ARM::VLD2LNd32Pseudo_UPD };
2977    unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2978                            ARM::VLD2LNq32Pseudo_UPD };
2979    return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2980  }
2981
2982  case ARMISD::VLD3LN_UPD: {
2983    unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, ARM::VLD3LNd16Pseudo_UPD,
2984                            ARM::VLD3LNd32Pseudo_UPD };
2985    unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2986                            ARM::VLD3LNq32Pseudo_UPD };
2987    return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2988  }
2989
2990  case ARMISD::VLD4LN_UPD: {
2991    unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, ARM::VLD4LNd16Pseudo_UPD,
2992                            ARM::VLD4LNd32Pseudo_UPD };
2993    unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
2994                            ARM::VLD4LNq32Pseudo_UPD };
2995    return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
2996  }
2997
2998  case ARMISD::VST1_UPD: {
2999    unsigned DOpcodes[] = { ARM::VST1d8wb_fixed, ARM::VST1d16wb_fixed,
3000                            ARM::VST1d32wb_fixed, ARM::VST1d64wb_fixed };
3001    unsigned QOpcodes[] = { ARM::VST1q8wb_fixed,
3002                            ARM::VST1q16wb_fixed,
3003                            ARM::VST1q32wb_fixed,
3004                            ARM::VST1q64wb_fixed };
3005    return SelectVST(N, true, 1, DOpcodes, QOpcodes, 0);
3006  }
3007
3008  case ARMISD::VST2_UPD: {
3009    unsigned DOpcodes[] = { ARM::VST2d8wb_fixed,
3010                            ARM::VST2d16wb_fixed,
3011                            ARM::VST2d32wb_fixed,
3012                            ARM::VST1q64wb_fixed};
3013    unsigned QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
3014                            ARM::VST2q16PseudoWB_fixed,
3015                            ARM::VST2q32PseudoWB_fixed };
3016    return SelectVST(N, true, 2, DOpcodes, QOpcodes, 0);
3017  }
3018
3019  case ARMISD::VST3_UPD: {
3020    unsigned DOpcodes[] = { ARM::VST3d8Pseudo_UPD, ARM::VST3d16Pseudo_UPD,
3021                            ARM::VST3d32Pseudo_UPD,ARM::VST1d64TPseudoWB_fixed};
3022    unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3023                             ARM::VST3q16Pseudo_UPD,
3024                             ARM::VST3q32Pseudo_UPD };
3025    unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
3026                             ARM::VST3q16oddPseudo_UPD,
3027                             ARM::VST3q32oddPseudo_UPD };
3028    return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3029  }
3030
3031  case ARMISD::VST4_UPD: {
3032    unsigned DOpcodes[] = { ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD,
3033                            ARM::VST4d32Pseudo_UPD,ARM::VST1d64QPseudoWB_fixed};
3034    unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3035                             ARM::VST4q16Pseudo_UPD,
3036                             ARM::VST4q32Pseudo_UPD };
3037    unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
3038                             ARM::VST4q16oddPseudo_UPD,
3039                             ARM::VST4q32oddPseudo_UPD };
3040    return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3041  }
3042
3043  case ARMISD::VST2LN_UPD: {
3044    unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, ARM::VST2LNd16Pseudo_UPD,
3045                            ARM::VST2LNd32Pseudo_UPD };
3046    unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
3047                            ARM::VST2LNq32Pseudo_UPD };
3048    return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
3049  }
3050
3051  case ARMISD::VST3LN_UPD: {
3052    unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, ARM::VST3LNd16Pseudo_UPD,
3053                            ARM::VST3LNd32Pseudo_UPD };
3054    unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
3055                            ARM::VST3LNq32Pseudo_UPD };
3056    return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3057  }
3058
3059  case ARMISD::VST4LN_UPD: {
3060    unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, ARM::VST4LNd16Pseudo_UPD,
3061                            ARM::VST4LNd32Pseudo_UPD };
3062    unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3063                            ARM::VST4LNq32Pseudo_UPD };
3064    return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3065  }
3066
3067  case ISD::INTRINSIC_VOID:
3068  case ISD::INTRINSIC_W_CHAIN: {
3069    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3070    switch (IntNo) {
3071    default:
3072      break;
3073
3074    case Intrinsic::arm_ldrexd: {
3075      SDValue MemAddr = N->getOperand(2);
3076      DebugLoc dl = N->getDebugLoc();
3077      SDValue Chain = N->getOperand(0);
3078
3079      unsigned NewOpc = ARM::LDREXD;
3080      if (Subtarget->isThumb() && Subtarget->hasThumb2())
3081        NewOpc = ARM::t2LDREXD;
3082
3083      // arm_ldrexd returns a i64 value in {i32, i32}
3084      std::vector<EVT> ResTys;
3085      ResTys.push_back(MVT::i32);
3086      ResTys.push_back(MVT::i32);
3087      ResTys.push_back(MVT::Other);
3088
3089      // place arguments in the right order
3090      SmallVector<SDValue, 7> Ops;
3091      Ops.push_back(MemAddr);
3092      Ops.push_back(getAL(CurDAG));
3093      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3094      Ops.push_back(Chain);
3095      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3096                                          Ops.size());
3097      // Transfer memoperands.
3098      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3099      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3100      cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3101
3102      // Until there's support for specifing explicit register constraints
3103      // like the use of even/odd register pair, hardcode ldrexd to always
3104      // use the pair [R0, R1] to hold the load result.
3105      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R0,
3106                                   SDValue(Ld, 0), SDValue(0,0));
3107      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R1,
3108                                   SDValue(Ld, 1), Chain.getValue(1));
3109
3110      // Remap uses.
3111      SDValue Glue = Chain.getValue(1);
3112      if (!SDValue(N, 0).use_empty()) {
3113        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3114                                                ARM::R0, MVT::i32, Glue);
3115        Glue = Result.getValue(2);
3116        ReplaceUses(SDValue(N, 0), Result);
3117      }
3118      if (!SDValue(N, 1).use_empty()) {
3119        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3120                                                ARM::R1, MVT::i32, Glue);
3121        Glue = Result.getValue(2);
3122        ReplaceUses(SDValue(N, 1), Result);
3123      }
3124
3125      ReplaceUses(SDValue(N, 2), SDValue(Ld, 2));
3126      return NULL;
3127    }
3128
3129    case Intrinsic::arm_strexd: {
3130      DebugLoc dl = N->getDebugLoc();
3131      SDValue Chain = N->getOperand(0);
3132      SDValue Val0 = N->getOperand(2);
3133      SDValue Val1 = N->getOperand(3);
3134      SDValue MemAddr = N->getOperand(4);
3135
3136      // Until there's support for specifing explicit register constraints
3137      // like the use of even/odd register pair, hardcode strexd to always
3138      // use the pair [R2, R3] to hold the i64 (i32, i32) value to be stored.
3139      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R2, Val0,
3140                                   SDValue(0, 0));
3141      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R3, Val1, Chain.getValue(1));
3142
3143      SDValue Glue = Chain.getValue(1);
3144      Val0 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3145                                    ARM::R2, MVT::i32, Glue);
3146      Glue = Val0.getValue(1);
3147      Val1 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3148                                    ARM::R3, MVT::i32, Glue);
3149
3150      // Store exclusive double return a i32 value which is the return status
3151      // of the issued store.
3152      std::vector<EVT> ResTys;
3153      ResTys.push_back(MVT::i32);
3154      ResTys.push_back(MVT::Other);
3155
3156      // place arguments in the right order
3157      SmallVector<SDValue, 7> Ops;
3158      Ops.push_back(Val0);
3159      Ops.push_back(Val1);
3160      Ops.push_back(MemAddr);
3161      Ops.push_back(getAL(CurDAG));
3162      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3163      Ops.push_back(Chain);
3164
3165      unsigned NewOpc = ARM::STREXD;
3166      if (Subtarget->isThumb() && Subtarget->hasThumb2())
3167        NewOpc = ARM::t2STREXD;
3168
3169      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3170                                          Ops.size());
3171      // Transfer memoperands.
3172      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3173      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3174      cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3175
3176      return St;
3177    }
3178
3179    case Intrinsic::arm_neon_vld1: {
3180      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3181                              ARM::VLD1d32, ARM::VLD1d64 };
3182      unsigned QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3183                              ARM::VLD1q32, ARM::VLD1q64};
3184      return SelectVLD(N, false, 1, DOpcodes, QOpcodes, 0);
3185    }
3186
3187    case Intrinsic::arm_neon_vld2: {
3188      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3189                              ARM::VLD2d32, ARM::VLD1q64 };
3190      unsigned QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3191                              ARM::VLD2q32Pseudo };
3192      return SelectVLD(N, false, 2, DOpcodes, QOpcodes, 0);
3193    }
3194
3195    case Intrinsic::arm_neon_vld3: {
3196      unsigned DOpcodes[] = { ARM::VLD3d8Pseudo, ARM::VLD3d16Pseudo,
3197                              ARM::VLD3d32Pseudo, ARM::VLD1d64TPseudo };
3198      unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3199                               ARM::VLD3q16Pseudo_UPD,
3200                               ARM::VLD3q32Pseudo_UPD };
3201      unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3202                               ARM::VLD3q16oddPseudo,
3203                               ARM::VLD3q32oddPseudo };
3204      return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3205    }
3206
3207    case Intrinsic::arm_neon_vld4: {
3208      unsigned DOpcodes[] = { ARM::VLD4d8Pseudo, ARM::VLD4d16Pseudo,
3209                              ARM::VLD4d32Pseudo, ARM::VLD1d64QPseudo };
3210      unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3211                               ARM::VLD4q16Pseudo_UPD,
3212                               ARM::VLD4q32Pseudo_UPD };
3213      unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3214                               ARM::VLD4q16oddPseudo,
3215                               ARM::VLD4q32oddPseudo };
3216      return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3217    }
3218
3219    case Intrinsic::arm_neon_vld2lane: {
3220      unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo, ARM::VLD2LNd16Pseudo,
3221                              ARM::VLD2LNd32Pseudo };
3222      unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo, ARM::VLD2LNq32Pseudo };
3223      return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3224    }
3225
3226    case Intrinsic::arm_neon_vld3lane: {
3227      unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo, ARM::VLD3LNd16Pseudo,
3228                              ARM::VLD3LNd32Pseudo };
3229      unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo, ARM::VLD3LNq32Pseudo };
3230      return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3231    }
3232
3233    case Intrinsic::arm_neon_vld4lane: {
3234      unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo, ARM::VLD4LNd16Pseudo,
3235                              ARM::VLD4LNd32Pseudo };
3236      unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo, ARM::VLD4LNq32Pseudo };
3237      return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3238    }
3239
3240    case Intrinsic::arm_neon_vst1: {
3241      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3242                              ARM::VST1d32, ARM::VST1d64 };
3243      unsigned QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3244                              ARM::VST1q32, ARM::VST1q64 };
3245      return SelectVST(N, false, 1, DOpcodes, QOpcodes, 0);
3246    }
3247
3248    case Intrinsic::arm_neon_vst2: {
3249      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3250                              ARM::VST2d32, ARM::VST1q64 };
3251      unsigned QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3252                              ARM::VST2q32Pseudo };
3253      return SelectVST(N, false, 2, DOpcodes, QOpcodes, 0);
3254    }
3255
3256    case Intrinsic::arm_neon_vst3: {
3257      unsigned DOpcodes[] = { ARM::VST3d8Pseudo, ARM::VST3d16Pseudo,
3258                              ARM::VST3d32Pseudo, ARM::VST1d64TPseudo };
3259      unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3260                               ARM::VST3q16Pseudo_UPD,
3261                               ARM::VST3q32Pseudo_UPD };
3262      unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo,
3263                               ARM::VST3q16oddPseudo,
3264                               ARM::VST3q32oddPseudo };
3265      return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3266    }
3267
3268    case Intrinsic::arm_neon_vst4: {
3269      unsigned DOpcodes[] = { ARM::VST4d8Pseudo, ARM::VST4d16Pseudo,
3270                              ARM::VST4d32Pseudo, ARM::VST1d64QPseudo };
3271      unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3272                               ARM::VST4q16Pseudo_UPD,
3273                               ARM::VST4q32Pseudo_UPD };
3274      unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo,
3275                               ARM::VST4q16oddPseudo,
3276                               ARM::VST4q32oddPseudo };
3277      return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3278    }
3279
3280    case Intrinsic::arm_neon_vst2lane: {
3281      unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo, ARM::VST2LNd16Pseudo,
3282                              ARM::VST2LNd32Pseudo };
3283      unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo, ARM::VST2LNq32Pseudo };
3284      return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3285    }
3286
3287    case Intrinsic::arm_neon_vst3lane: {
3288      unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo, ARM::VST3LNd16Pseudo,
3289                              ARM::VST3LNd32Pseudo };
3290      unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo, ARM::VST3LNq32Pseudo };
3291      return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3292    }
3293
3294    case Intrinsic::arm_neon_vst4lane: {
3295      unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo, ARM::VST4LNd16Pseudo,
3296                              ARM::VST4LNd32Pseudo };
3297      unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo, ARM::VST4LNq32Pseudo };
3298      return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3299    }
3300    }
3301    break;
3302  }
3303
3304  case ISD::INTRINSIC_WO_CHAIN: {
3305    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3306    switch (IntNo) {
3307    default:
3308      break;
3309
3310    case Intrinsic::arm_neon_vtbl2:
3311      return SelectVTBL(N, false, 2, ARM::VTBL2);
3312    case Intrinsic::arm_neon_vtbl3:
3313      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
3314    case Intrinsic::arm_neon_vtbl4:
3315      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
3316
3317    case Intrinsic::arm_neon_vtbx2:
3318      return SelectVTBL(N, true, 2, ARM::VTBX2);
3319    case Intrinsic::arm_neon_vtbx3:
3320      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
3321    case Intrinsic::arm_neon_vtbx4:
3322      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
3323    }
3324    break;
3325  }
3326
3327  case ARMISD::VTBL1: {
3328    DebugLoc dl = N->getDebugLoc();
3329    EVT VT = N->getValueType(0);
3330    SmallVector<SDValue, 6> Ops;
3331
3332    Ops.push_back(N->getOperand(0));
3333    Ops.push_back(N->getOperand(1));
3334    Ops.push_back(getAL(CurDAG));                    // Predicate
3335    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3336    return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops.data(), Ops.size());
3337  }
3338  case ARMISD::VTBL2: {
3339    DebugLoc dl = N->getDebugLoc();
3340    EVT VT = N->getValueType(0);
3341
3342    // Form a REG_SEQUENCE to force register allocation.
3343    SDValue V0 = N->getOperand(0);
3344    SDValue V1 = N->getOperand(1);
3345    SDValue RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
3346
3347    SmallVector<SDValue, 6> Ops;
3348    Ops.push_back(RegSeq);
3349    Ops.push_back(N->getOperand(2));
3350    Ops.push_back(getAL(CurDAG));                    // Predicate
3351    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3352    return CurDAG->getMachineNode(ARM::VTBL2, dl, VT,
3353                                  Ops.data(), Ops.size());
3354  }
3355
3356  case ISD::CONCAT_VECTORS:
3357    return SelectConcatVector(N);
3358
3359  case ARMISD::ATOMOR64_DAG:
3360    return SelectAtomic64(N, ARM::ATOMOR6432);
3361  case ARMISD::ATOMXOR64_DAG:
3362    return SelectAtomic64(N, ARM::ATOMXOR6432);
3363  case ARMISD::ATOMADD64_DAG:
3364    return SelectAtomic64(N, ARM::ATOMADD6432);
3365  case ARMISD::ATOMSUB64_DAG:
3366    return SelectAtomic64(N, ARM::ATOMSUB6432);
3367  case ARMISD::ATOMNAND64_DAG:
3368    return SelectAtomic64(N, ARM::ATOMNAND6432);
3369  case ARMISD::ATOMAND64_DAG:
3370    return SelectAtomic64(N, ARM::ATOMAND6432);
3371  case ARMISD::ATOMSWAP64_DAG:
3372    return SelectAtomic64(N, ARM::ATOMSWAP6432);
3373  case ARMISD::ATOMCMPXCHG64_DAG:
3374    return SelectAtomic64(N, ARM::ATOMCMPXCHG6432);
3375  }
3376
3377  return SelectCode(N);
3378}
3379
3380bool ARMDAGToDAGISel::
3381SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
3382                             std::vector<SDValue> &OutOps) {
3383  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
3384  // Require the address to be in a register.  That is safe for all ARM
3385  // variants and it is hard to do anything much smarter without knowing
3386  // how the operand is used.
3387  OutOps.push_back(Op);
3388  return false;
3389}
3390
3391/// createARMISelDag - This pass converts a legalized DAG into a
3392/// ARM-specific DAG, ready for instruction scheduling.
3393///
3394FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3395                                     CodeGenOpt::Level OptLevel) {
3396  return new ARMDAGToDAGISel(TM, OptLevel);
3397}
3398