ARMISelDAGToDAG.cpp revision e0a03143df398c17a435c136b14316fd43f27fb7 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMBaseInstrInfo.h"
17#include "ARMTargetMachine.h"
18#include "MCTargetDesc/ARMAddressingModes.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
37
38using namespace llvm;
39
40static cl::opt<bool>
41DisableShifterOp("disable-shifter-op", cl::Hidden,
42  cl::desc("Disable isel of shifter-op"),
43  cl::init(false));
44
45static cl::opt<bool>
46CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
47  cl::desc("Check fp vmla / vmls hazard at isel time"),
48  cl::init(true));
49
50//===--------------------------------------------------------------------===//
51/// ARMDAGToDAGISel - ARM specific code to select ARM machine
52/// instructions for SelectionDAG operations.
53///
54namespace {
55
56enum AddrMode2Type {
57  AM2_BASE, // Simple AM2 (+-imm12)
58  AM2_SHOP  // Shifter-op AM2
59};
60
61class ARMDAGToDAGISel : public SelectionDAGISel {
62  ARMBaseTargetMachine &TM;
63  const ARMBaseInstrInfo *TII;
64
65  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
66  /// make the right decision when generating code for different targets.
67  const ARMSubtarget *Subtarget;
68
69public:
70  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
71                           CodeGenOpt::Level OptLevel)
72    : SelectionDAGISel(tm, OptLevel), TM(tm),
73      TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())),
74      Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
75  }
76
77  virtual const char *getPassName() const {
78    return "ARM Instruction Selection";
79  }
80
81  /// getI32Imm - Return a target constant of type i32 with the specified
82  /// value.
83  inline SDValue getI32Imm(unsigned Imm) {
84    return CurDAG->getTargetConstant(Imm, MVT::i32);
85  }
86
87  SDNode *Select(SDNode *N);
88
89
90  bool hasNoVMLxHazardUse(SDNode *N) const;
91  bool isShifterOpProfitable(const SDValue &Shift,
92                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
93  bool SelectRegShifterOperand(SDValue N, SDValue &A,
94                               SDValue &B, SDValue &C,
95                               bool CheckProfitability = true);
96  bool SelectImmShifterOperand(SDValue N, SDValue &A,
97                               SDValue &B, bool CheckProfitability = true);
98  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
99                                    SDValue &B, SDValue &C) {
100    // Don't apply the profitability check
101    return SelectRegShifterOperand(N, A, B, C, false);
102  }
103  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
104                                    SDValue &B) {
105    // Don't apply the profitability check
106    return SelectImmShifterOperand(N, A, B, false);
107  }
108
109  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
110  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
111
112  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
113                                      SDValue &Offset, SDValue &Opc);
114  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
115                           SDValue &Opc) {
116    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
117  }
118
119  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
120                           SDValue &Opc) {
121    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
122  }
123
124  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
125                       SDValue &Opc) {
126    SelectAddrMode2Worker(N, Base, Offset, Opc);
127//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
128    // This always matches one way or another.
129    return true;
130  }
131
132  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
133                             SDValue &Offset, SDValue &Opc);
134  bool SelectAddrMode3(SDValue N, SDValue &Base,
135                       SDValue &Offset, SDValue &Opc);
136  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
137                             SDValue &Offset, SDValue &Opc);
138  bool SelectAddrMode5(SDValue N, SDValue &Base,
139                       SDValue &Offset);
140  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
141  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
142
143  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
144
145  // Thumb Addressing Modes:
146  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
147  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
148                             unsigned Scale);
149  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
150  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
151  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
152  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
153                                SDValue &OffImm);
154  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
155                                 SDValue &OffImm);
156  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
157                                 SDValue &OffImm);
158  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
159                                 SDValue &OffImm);
160  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
161
162  // Thumb 2 Addressing Modes:
163  bool SelectT2ShifterOperandReg(SDValue N,
164                                 SDValue &BaseReg, SDValue &Opc);
165  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
166  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
167                            SDValue &OffImm);
168  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
169                                 SDValue &OffImm);
170  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
171                             SDValue &OffReg, SDValue &ShImm);
172
173  inline bool is_so_imm(unsigned Imm) const {
174    return ARM_AM::getSOImmVal(Imm) != -1;
175  }
176
177  inline bool is_so_imm_not(unsigned Imm) const {
178    return ARM_AM::getSOImmVal(~Imm) != -1;
179  }
180
181  inline bool is_t2_so_imm(unsigned Imm) const {
182    return ARM_AM::getT2SOImmVal(Imm) != -1;
183  }
184
185  inline bool is_t2_so_imm_not(unsigned Imm) const {
186    return ARM_AM::getT2SOImmVal(~Imm) != -1;
187  }
188
189  // Include the pieces autogenerated from the target description.
190#include "ARMGenDAGISel.inc"
191
192private:
193  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
194  /// ARM.
195  SDNode *SelectARMIndexedLoad(SDNode *N);
196  SDNode *SelectT2IndexedLoad(SDNode *N);
197
198  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
199  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
200  /// loads of D registers and even subregs and odd subregs of Q registers.
201  /// For NumVecs <= 2, QOpcodes1 is not used.
202  SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
203                    unsigned *DOpcodes,
204                    unsigned *QOpcodes0, unsigned *QOpcodes1);
205
206  /// SelectVST - Select NEON store intrinsics.  NumVecs should
207  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
208  /// stores of D registers and even subregs and odd subregs of Q registers.
209  /// For NumVecs <= 2, QOpcodes1 is not used.
210  SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
211                    unsigned *DOpcodes,
212                    unsigned *QOpcodes0, unsigned *QOpcodes1);
213
214  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
215  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
216  /// load/store of D registers and Q registers.
217  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
218                          bool isUpdating, unsigned NumVecs,
219                          unsigned *DOpcodes, unsigned *QOpcodes);
220
221  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
222  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
223  /// for loading D registers.  (Q registers are not supported.)
224  SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
225                       unsigned *Opcodes);
226
227  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
228  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
229  /// generated to force the table registers to be consecutive.
230  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
231
232  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
233  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
234
235  /// SelectCMOVOp - Select CMOV instructions for ARM.
236  SDNode *SelectCMOVOp(SDNode *N);
237  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
238                              ARMCC::CondCodes CCVal, SDValue CCR,
239                              SDValue InFlag);
240  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
241                               ARMCC::CondCodes CCVal, SDValue CCR,
242                               SDValue InFlag);
243  SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
244                              ARMCC::CondCodes CCVal, SDValue CCR,
245                              SDValue InFlag);
246  SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
247                               ARMCC::CondCodes CCVal, SDValue CCR,
248                               SDValue InFlag);
249
250  SDNode *SelectConcatVector(SDNode *N);
251
252  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
253  /// inline asm expressions.
254  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
255                                            char ConstraintCode,
256                                            std::vector<SDValue> &OutOps);
257
258  // Form pairs of consecutive S, D, or Q registers.
259  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
260  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
261  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
262
263  // Form sequences of 4 consecutive S, D, or Q registers.
264  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
265  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
266  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
267
268  // Get the alignment operand for a NEON VLD or VST instruction.
269  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
270};
271}
272
273/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
274/// operand. If so Imm will receive the 32-bit value.
275static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
276  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
277    Imm = cast<ConstantSDNode>(N)->getZExtValue();
278    return true;
279  }
280  return false;
281}
282
283// isInt32Immediate - This method tests to see if a constant operand.
284// If so Imm will receive the 32 bit value.
285static bool isInt32Immediate(SDValue N, unsigned &Imm) {
286  return isInt32Immediate(N.getNode(), Imm);
287}
288
289// isOpcWithIntImmediate - This method tests to see if the node is a specific
290// opcode and that it has a immediate integer right operand.
291// If so Imm will receive the 32 bit value.
292static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
293  return N->getOpcode() == Opc &&
294         isInt32Immediate(N->getOperand(1).getNode(), Imm);
295}
296
297/// \brief Check whether a particular node is a constant value representable as
298/// (N * Scale) where (N in [\arg RangeMin, \arg RangeMax).
299///
300/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
301static bool isScaledConstantInRange(SDValue Node, unsigned Scale,
302                                    int RangeMin, int RangeMax,
303                                    int &ScaledConstant) {
304  assert(Scale && "Invalid scale!");
305
306  // Check that this is a constant.
307  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
308  if (!C)
309    return false;
310
311  ScaledConstant = (int) C->getZExtValue();
312  if ((ScaledConstant % Scale) != 0)
313    return false;
314
315  ScaledConstant /= Scale;
316  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
317}
318
319/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
320/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
321/// least on current ARM implementations) which should be avoidded.
322bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
323  if (OptLevel == CodeGenOpt::None)
324    return true;
325
326  if (!CheckVMLxHazard)
327    return true;
328
329  if (!Subtarget->isCortexA8() && !Subtarget->isCortexA9())
330    return true;
331
332  if (!N->hasOneUse())
333    return false;
334
335  SDNode *Use = *N->use_begin();
336  if (Use->getOpcode() == ISD::CopyToReg)
337    return true;
338  if (Use->isMachineOpcode()) {
339    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
340    if (MCID.mayStore())
341      return true;
342    unsigned Opcode = MCID.getOpcode();
343    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
344      return true;
345    // vmlx feeding into another vmlx. We actually want to unfold
346    // the use later in the MLxExpansion pass. e.g.
347    // vmla
348    // vmla (stall 8 cycles)
349    //
350    // vmul (5 cycles)
351    // vadd (5 cycles)
352    // vmla
353    // This adds up to about 18 - 19 cycles.
354    //
355    // vmla
356    // vmul (stall 4 cycles)
357    // vadd adds up to about 14 cycles.
358    return TII->isFpMLxInstruction(Opcode);
359  }
360
361  return false;
362}
363
364bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
365                                            ARM_AM::ShiftOpc ShOpcVal,
366                                            unsigned ShAmt) {
367  if (!Subtarget->isCortexA9())
368    return true;
369  if (Shift.hasOneUse())
370    return true;
371  // R << 2 is free.
372  return ShOpcVal == ARM_AM::lsl && ShAmt == 2;
373}
374
375bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
376                                              SDValue &BaseReg,
377                                              SDValue &Opc,
378                                              bool CheckProfitability) {
379  if (DisableShifterOp)
380    return false;
381
382  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
383
384  // Don't match base register only case. That is matched to a separate
385  // lower complexity pattern with explicit register operand.
386  if (ShOpcVal == ARM_AM::no_shift) return false;
387
388  BaseReg = N.getOperand(0);
389  unsigned ShImmVal = 0;
390  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
391  if (!RHS) return false;
392  ShImmVal = RHS->getZExtValue() & 31;
393  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
394                                  MVT::i32);
395  return true;
396}
397
398bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
399                                              SDValue &BaseReg,
400                                              SDValue &ShReg,
401                                              SDValue &Opc,
402                                              bool CheckProfitability) {
403  if (DisableShifterOp)
404    return false;
405
406  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
407
408  // Don't match base register only case. That is matched to a separate
409  // lower complexity pattern with explicit register operand.
410  if (ShOpcVal == ARM_AM::no_shift) return false;
411
412  BaseReg = N.getOperand(0);
413  unsigned ShImmVal = 0;
414  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
415  if (RHS) return false;
416
417  ShReg = N.getOperand(1);
418  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
419    return false;
420  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
421                                  MVT::i32);
422  return true;
423}
424
425
426bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
427                                          SDValue &Base,
428                                          SDValue &OffImm) {
429  // Match simple R + imm12 operands.
430
431  // Base only.
432  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
433      !CurDAG->isBaseWithConstantOffset(N)) {
434    if (N.getOpcode() == ISD::FrameIndex) {
435      // Match frame index.
436      int FI = cast<FrameIndexSDNode>(N)->getIndex();
437      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
438      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
439      return true;
440    }
441
442    if (N.getOpcode() == ARMISD::Wrapper &&
443        !(Subtarget->useMovt() &&
444                     N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
445      Base = N.getOperand(0);
446    } else
447      Base = N;
448    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
449    return true;
450  }
451
452  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
453    int RHSC = (int)RHS->getZExtValue();
454    if (N.getOpcode() == ISD::SUB)
455      RHSC = -RHSC;
456
457    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
458      Base   = N.getOperand(0);
459      if (Base.getOpcode() == ISD::FrameIndex) {
460        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
461        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
462      }
463      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
464      return true;
465    }
466  }
467
468  // Base only.
469  Base = N;
470  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
471  return true;
472}
473
474
475
476bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
477                                      SDValue &Opc) {
478  if (N.getOpcode() == ISD::MUL &&
479      (!Subtarget->isCortexA9() || N.hasOneUse())) {
480    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
481      // X * [3,5,9] -> X + X * [2,4,8] etc.
482      int RHSC = (int)RHS->getZExtValue();
483      if (RHSC & 1) {
484        RHSC = RHSC & ~1;
485        ARM_AM::AddrOpc AddSub = ARM_AM::add;
486        if (RHSC < 0) {
487          AddSub = ARM_AM::sub;
488          RHSC = - RHSC;
489        }
490        if (isPowerOf2_32(RHSC)) {
491          unsigned ShAmt = Log2_32(RHSC);
492          Base = Offset = N.getOperand(0);
493          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
494                                                            ARM_AM::lsl),
495                                          MVT::i32);
496          return true;
497        }
498      }
499    }
500  }
501
502  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
503      // ISD::OR that is equivalent to an ISD::ADD.
504      !CurDAG->isBaseWithConstantOffset(N))
505    return false;
506
507  // Leave simple R +/- imm12 operands for LDRi12
508  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
509    int RHSC;
510    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
511                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
512      return false;
513  }
514
515  if (Subtarget->isCortexA9() && !N.hasOneUse())
516    // Compute R +/- (R << N) and reuse it.
517    return false;
518
519  // Otherwise this is R +/- [possibly shifted] R.
520  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
521  ARM_AM::ShiftOpc ShOpcVal =
522    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
523  unsigned ShAmt = 0;
524
525  Base   = N.getOperand(0);
526  Offset = N.getOperand(1);
527
528  if (ShOpcVal != ARM_AM::no_shift) {
529    // Check to see if the RHS of the shift is a constant, if not, we can't fold
530    // it.
531    if (ConstantSDNode *Sh =
532           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
533      ShAmt = Sh->getZExtValue();
534      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
535        Offset = N.getOperand(1).getOperand(0);
536      else {
537        ShAmt = 0;
538        ShOpcVal = ARM_AM::no_shift;
539      }
540    } else {
541      ShOpcVal = ARM_AM::no_shift;
542    }
543  }
544
545  // Try matching (R shl C) + (R).
546  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
547      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
548    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
549    if (ShOpcVal != ARM_AM::no_shift) {
550      // Check to see if the RHS of the shift is a constant, if not, we can't
551      // fold it.
552      if (ConstantSDNode *Sh =
553          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
554        ShAmt = Sh->getZExtValue();
555        if (!Subtarget->isCortexA9() ||
556            (N.hasOneUse() &&
557             isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt))) {
558          Offset = N.getOperand(0).getOperand(0);
559          Base = N.getOperand(1);
560        } else {
561          ShAmt = 0;
562          ShOpcVal = ARM_AM::no_shift;
563        }
564      } else {
565        ShOpcVal = ARM_AM::no_shift;
566      }
567    }
568  }
569
570  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
571                                  MVT::i32);
572  return true;
573}
574
575
576
577
578//-----
579
580AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
581                                                     SDValue &Base,
582                                                     SDValue &Offset,
583                                                     SDValue &Opc) {
584  if (N.getOpcode() == ISD::MUL &&
585      (!Subtarget->isCortexA9() || N.hasOneUse())) {
586    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
587      // X * [3,5,9] -> X + X * [2,4,8] etc.
588      int RHSC = (int)RHS->getZExtValue();
589      if (RHSC & 1) {
590        RHSC = RHSC & ~1;
591        ARM_AM::AddrOpc AddSub = ARM_AM::add;
592        if (RHSC < 0) {
593          AddSub = ARM_AM::sub;
594          RHSC = - RHSC;
595        }
596        if (isPowerOf2_32(RHSC)) {
597          unsigned ShAmt = Log2_32(RHSC);
598          Base = Offset = N.getOperand(0);
599          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
600                                                            ARM_AM::lsl),
601                                          MVT::i32);
602          return AM2_SHOP;
603        }
604      }
605    }
606  }
607
608  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
609      // ISD::OR that is equivalent to an ADD.
610      !CurDAG->isBaseWithConstantOffset(N)) {
611    Base = N;
612    if (N.getOpcode() == ISD::FrameIndex) {
613      int FI = cast<FrameIndexSDNode>(N)->getIndex();
614      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
615    } else if (N.getOpcode() == ARMISD::Wrapper &&
616               !(Subtarget->useMovt() &&
617                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
618      Base = N.getOperand(0);
619    }
620    Offset = CurDAG->getRegister(0, MVT::i32);
621    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
622                                                      ARM_AM::no_shift),
623                                    MVT::i32);
624    return AM2_BASE;
625  }
626
627  // Match simple R +/- imm12 operands.
628  if (N.getOpcode() != ISD::SUB) {
629    int RHSC;
630    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
631                                -0x1000+1, 0x1000, RHSC)) { // 12 bits.
632      Base = N.getOperand(0);
633      if (Base.getOpcode() == ISD::FrameIndex) {
634        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
635        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
636      }
637      Offset = CurDAG->getRegister(0, MVT::i32);
638
639      ARM_AM::AddrOpc AddSub = ARM_AM::add;
640      if (RHSC < 0) {
641        AddSub = ARM_AM::sub;
642        RHSC = - RHSC;
643      }
644      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
645                                                        ARM_AM::no_shift),
646                                      MVT::i32);
647      return AM2_BASE;
648    }
649  }
650
651  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
652    // Compute R +/- (R << N) and reuse it.
653    Base = N;
654    Offset = CurDAG->getRegister(0, MVT::i32);
655    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
656                                                      ARM_AM::no_shift),
657                                    MVT::i32);
658    return AM2_BASE;
659  }
660
661  // Otherwise this is R +/- [possibly shifted] R.
662  ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
663  ARM_AM::ShiftOpc ShOpcVal =
664    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
665  unsigned ShAmt = 0;
666
667  Base   = N.getOperand(0);
668  Offset = N.getOperand(1);
669
670  if (ShOpcVal != ARM_AM::no_shift) {
671    // Check to see if the RHS of the shift is a constant, if not, we can't fold
672    // it.
673    if (ConstantSDNode *Sh =
674           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
675      ShAmt = Sh->getZExtValue();
676      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
677        Offset = N.getOperand(1).getOperand(0);
678      else {
679        ShAmt = 0;
680        ShOpcVal = ARM_AM::no_shift;
681      }
682    } else {
683      ShOpcVal = ARM_AM::no_shift;
684    }
685  }
686
687  // Try matching (R shl C) + (R).
688  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
689      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
690    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
691    if (ShOpcVal != ARM_AM::no_shift) {
692      // Check to see if the RHS of the shift is a constant, if not, we can't
693      // fold it.
694      if (ConstantSDNode *Sh =
695          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
696        ShAmt = Sh->getZExtValue();
697        if (!Subtarget->isCortexA9() ||
698            (N.hasOneUse() &&
699             isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt))) {
700          Offset = N.getOperand(0).getOperand(0);
701          Base = N.getOperand(1);
702        } else {
703          ShAmt = 0;
704          ShOpcVal = ARM_AM::no_shift;
705        }
706      } else {
707        ShOpcVal = ARM_AM::no_shift;
708      }
709    }
710  }
711
712  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
713                                  MVT::i32);
714  return AM2_SHOP;
715}
716
717bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
718                                            SDValue &Offset, SDValue &Opc) {
719  unsigned Opcode = Op->getOpcode();
720  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
721    ? cast<LoadSDNode>(Op)->getAddressingMode()
722    : cast<StoreSDNode>(Op)->getAddressingMode();
723  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
724    ? ARM_AM::add : ARM_AM::sub;
725  int Val;
726  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
727    Offset = CurDAG->getRegister(0, MVT::i32);
728    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
729                                                      ARM_AM::no_shift),
730                                    MVT::i32);
731    return true;
732  }
733
734  Offset = N;
735  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
736  unsigned ShAmt = 0;
737  if (ShOpcVal != ARM_AM::no_shift) {
738    // Check to see if the RHS of the shift is a constant, if not, we can't fold
739    // it.
740    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
741      ShAmt = Sh->getZExtValue();
742      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
743        Offset = N.getOperand(0);
744      else {
745        ShAmt = 0;
746        ShOpcVal = ARM_AM::no_shift;
747      }
748    } else {
749      ShOpcVal = ARM_AM::no_shift;
750    }
751  }
752
753  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
754                                  MVT::i32);
755  return true;
756}
757
758
759bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
760                                      SDValue &Base, SDValue &Offset,
761                                      SDValue &Opc) {
762  if (N.getOpcode() == ISD::SUB) {
763    // X - C  is canonicalize to X + -C, no need to handle it here.
764    Base = N.getOperand(0);
765    Offset = N.getOperand(1);
766    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
767    return true;
768  }
769
770  if (!CurDAG->isBaseWithConstantOffset(N)) {
771    Base = N;
772    if (N.getOpcode() == ISD::FrameIndex) {
773      int FI = cast<FrameIndexSDNode>(N)->getIndex();
774      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
775    }
776    Offset = CurDAG->getRegister(0, MVT::i32);
777    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
778    return true;
779  }
780
781  // If the RHS is +/- imm8, fold into addr mode.
782  int RHSC;
783  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
784                              -256 + 1, 256, RHSC)) { // 8 bits.
785    Base = N.getOperand(0);
786    if (Base.getOpcode() == ISD::FrameIndex) {
787      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
788      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
789    }
790    Offset = CurDAG->getRegister(0, MVT::i32);
791
792    ARM_AM::AddrOpc AddSub = ARM_AM::add;
793    if (RHSC < 0) {
794      AddSub = ARM_AM::sub;
795      RHSC = -RHSC;
796    }
797    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
798    return true;
799  }
800
801  Base = N.getOperand(0);
802  Offset = N.getOperand(1);
803  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
804  return true;
805}
806
807bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
808                                            SDValue &Offset, SDValue &Opc) {
809  unsigned Opcode = Op->getOpcode();
810  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
811    ? cast<LoadSDNode>(Op)->getAddressingMode()
812    : cast<StoreSDNode>(Op)->getAddressingMode();
813  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
814    ? ARM_AM::add : ARM_AM::sub;
815  int Val;
816  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
817    Offset = CurDAG->getRegister(0, MVT::i32);
818    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
819    return true;
820  }
821
822  Offset = N;
823  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
824  return true;
825}
826
827bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
828                                      SDValue &Base, SDValue &Offset) {
829  if (!CurDAG->isBaseWithConstantOffset(N)) {
830    Base = N;
831    if (N.getOpcode() == ISD::FrameIndex) {
832      int FI = cast<FrameIndexSDNode>(N)->getIndex();
833      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
834    } else if (N.getOpcode() == ARMISD::Wrapper &&
835               !(Subtarget->useMovt() &&
836                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
837      Base = N.getOperand(0);
838    }
839    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
840                                       MVT::i32);
841    return true;
842  }
843
844  // If the RHS is +/- imm8, fold into addr mode.
845  int RHSC;
846  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
847                              -256 + 1, 256, RHSC)) {
848    Base = N.getOperand(0);
849    if (Base.getOpcode() == ISD::FrameIndex) {
850      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
851      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
852    }
853
854    ARM_AM::AddrOpc AddSub = ARM_AM::add;
855    if (RHSC < 0) {
856      AddSub = ARM_AM::sub;
857      RHSC = -RHSC;
858    }
859    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
860                                       MVT::i32);
861    return true;
862  }
863
864  Base = N;
865  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
866                                     MVT::i32);
867  return true;
868}
869
870bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
871                                      SDValue &Align) {
872  Addr = N;
873
874  unsigned Alignment = 0;
875  if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
876    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
877    // The maximum alignment is equal to the memory size being referenced.
878    unsigned LSNAlign = LSN->getAlignment();
879    unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
880    if (LSNAlign > MemSize && MemSize > 1)
881      Alignment = MemSize;
882  } else {
883    // All other uses of addrmode6 are for intrinsics.  For now just record
884    // the raw alignment value; it will be refined later based on the legal
885    // alignment operands for the intrinsic.
886    Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
887  }
888
889  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
890  return true;
891}
892
893bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
894                                            SDValue &Offset) {
895  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
896  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
897  if (AM != ISD::POST_INC)
898    return false;
899  Offset = N;
900  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
901    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
902      Offset = CurDAG->getRegister(0, MVT::i32);
903  }
904  return true;
905}
906
907bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
908                                       SDValue &Offset, SDValue &Label) {
909  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
910    Offset = N.getOperand(0);
911    SDValue N1 = N.getOperand(1);
912    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
913                                      MVT::i32);
914    return true;
915  }
916
917  return false;
918}
919
920
921//===----------------------------------------------------------------------===//
922//                         Thumb Addressing Modes
923//===----------------------------------------------------------------------===//
924
925bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
926                                            SDValue &Base, SDValue &Offset){
927  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
928    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
929    if (!NC || !NC->isNullValue())
930      return false;
931
932    Base = Offset = N;
933    return true;
934  }
935
936  Base = N.getOperand(0);
937  Offset = N.getOperand(1);
938  return true;
939}
940
941bool
942ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
943                                       SDValue &Offset, unsigned Scale) {
944  if (Scale == 4) {
945    SDValue TmpBase, TmpOffImm;
946    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
947      return false;  // We want to select tLDRspi / tSTRspi instead.
948
949    if (N.getOpcode() == ARMISD::Wrapper &&
950        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
951      return false;  // We want to select tLDRpci instead.
952  }
953
954  if (!CurDAG->isBaseWithConstantOffset(N))
955    return false;
956
957  // Thumb does not have [sp, r] address mode.
958  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
959  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
960  if ((LHSR && LHSR->getReg() == ARM::SP) ||
961      (RHSR && RHSR->getReg() == ARM::SP))
962    return false;
963
964  // FIXME: Why do we explicitly check for a match here and then return false?
965  // Presumably to allow something else to match, but shouldn't this be
966  // documented?
967  int RHSC;
968  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
969    return false;
970
971  Base = N.getOperand(0);
972  Offset = N.getOperand(1);
973  return true;
974}
975
976bool
977ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
978                                          SDValue &Base,
979                                          SDValue &Offset) {
980  return SelectThumbAddrModeRI(N, Base, Offset, 1);
981}
982
983bool
984ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
985                                          SDValue &Base,
986                                          SDValue &Offset) {
987  return SelectThumbAddrModeRI(N, Base, Offset, 2);
988}
989
990bool
991ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
992                                          SDValue &Base,
993                                          SDValue &Offset) {
994  return SelectThumbAddrModeRI(N, Base, Offset, 4);
995}
996
997bool
998ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
999                                          SDValue &Base, SDValue &OffImm) {
1000  if (Scale == 4) {
1001    SDValue TmpBase, TmpOffImm;
1002    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1003      return false;  // We want to select tLDRspi / tSTRspi instead.
1004
1005    if (N.getOpcode() == ARMISD::Wrapper &&
1006        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1007      return false;  // We want to select tLDRpci instead.
1008  }
1009
1010  if (!CurDAG->isBaseWithConstantOffset(N)) {
1011    if (N.getOpcode() == ARMISD::Wrapper &&
1012        !(Subtarget->useMovt() &&
1013          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1014      Base = N.getOperand(0);
1015    } else {
1016      Base = N;
1017    }
1018
1019    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1020    return true;
1021  }
1022
1023  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1024  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1025  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1026      (RHSR && RHSR->getReg() == ARM::SP)) {
1027    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1028    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1029    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1030    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1031
1032    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1033    if (LHSC != 0 || RHSC != 0) return false;
1034
1035    Base = N;
1036    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1037    return true;
1038  }
1039
1040  // If the RHS is + imm5 * scale, fold into addr mode.
1041  int RHSC;
1042  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1043    Base = N.getOperand(0);
1044    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1045    return true;
1046  }
1047
1048  Base = N.getOperand(0);
1049  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1050  return true;
1051}
1052
1053bool
1054ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1055                                           SDValue &OffImm) {
1056  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1057}
1058
1059bool
1060ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1061                                           SDValue &OffImm) {
1062  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1063}
1064
1065bool
1066ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1067                                           SDValue &OffImm) {
1068  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1069}
1070
1071bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1072                                            SDValue &Base, SDValue &OffImm) {
1073  if (N.getOpcode() == ISD::FrameIndex) {
1074    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1075    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1076    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1077    return true;
1078  }
1079
1080  if (!CurDAG->isBaseWithConstantOffset(N))
1081    return false;
1082
1083  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1084  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1085      (LHSR && LHSR->getReg() == ARM::SP)) {
1086    // If the RHS is + imm8 * scale, fold into addr mode.
1087    int RHSC;
1088    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1089      Base = N.getOperand(0);
1090      if (Base.getOpcode() == ISD::FrameIndex) {
1091        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1092        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1093      }
1094      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1095      return true;
1096    }
1097  }
1098
1099  return false;
1100}
1101
1102
1103//===----------------------------------------------------------------------===//
1104//                        Thumb 2 Addressing Modes
1105//===----------------------------------------------------------------------===//
1106
1107
1108bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1109                                                SDValue &Opc) {
1110  if (DisableShifterOp)
1111    return false;
1112
1113  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1114
1115  // Don't match base register only case. That is matched to a separate
1116  // lower complexity pattern with explicit register operand.
1117  if (ShOpcVal == ARM_AM::no_shift) return false;
1118
1119  BaseReg = N.getOperand(0);
1120  unsigned ShImmVal = 0;
1121  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1122    ShImmVal = RHS->getZExtValue() & 31;
1123    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1124    return true;
1125  }
1126
1127  return false;
1128}
1129
1130bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1131                                            SDValue &Base, SDValue &OffImm) {
1132  // Match simple R + imm12 operands.
1133
1134  // Base only.
1135  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1136      !CurDAG->isBaseWithConstantOffset(N)) {
1137    if (N.getOpcode() == ISD::FrameIndex) {
1138      // Match frame index.
1139      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1140      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1141      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1142      return true;
1143    }
1144
1145    if (N.getOpcode() == ARMISD::Wrapper &&
1146               !(Subtarget->useMovt() &&
1147                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1148      Base = N.getOperand(0);
1149      if (Base.getOpcode() == ISD::TargetConstantPool)
1150        return false;  // We want to select t2LDRpci instead.
1151    } else
1152      Base = N;
1153    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1154    return true;
1155  }
1156
1157  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1158    if (SelectT2AddrModeImm8(N, Base, OffImm))
1159      // Let t2LDRi8 handle (R - imm8).
1160      return false;
1161
1162    int RHSC = (int)RHS->getZExtValue();
1163    if (N.getOpcode() == ISD::SUB)
1164      RHSC = -RHSC;
1165
1166    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1167      Base   = N.getOperand(0);
1168      if (Base.getOpcode() == ISD::FrameIndex) {
1169        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1170        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1171      }
1172      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1173      return true;
1174    }
1175  }
1176
1177  // Base only.
1178  Base = N;
1179  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1180  return true;
1181}
1182
1183bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1184                                           SDValue &Base, SDValue &OffImm) {
1185  // Match simple R - imm8 operands.
1186  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1187      !CurDAG->isBaseWithConstantOffset(N))
1188    return false;
1189
1190  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1191    int RHSC = (int)RHS->getSExtValue();
1192    if (N.getOpcode() == ISD::SUB)
1193      RHSC = -RHSC;
1194
1195    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1196      Base = N.getOperand(0);
1197      if (Base.getOpcode() == ISD::FrameIndex) {
1198        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1199        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1200      }
1201      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1202      return true;
1203    }
1204  }
1205
1206  return false;
1207}
1208
1209bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1210                                                 SDValue &OffImm){
1211  unsigned Opcode = Op->getOpcode();
1212  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1213    ? cast<LoadSDNode>(Op)->getAddressingMode()
1214    : cast<StoreSDNode>(Op)->getAddressingMode();
1215  int RHSC;
1216  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1217    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1218      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1219      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1220    return true;
1221  }
1222
1223  return false;
1224}
1225
1226bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1227                                            SDValue &Base,
1228                                            SDValue &OffReg, SDValue &ShImm) {
1229  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1230  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1231    return false;
1232
1233  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1234  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1235    int RHSC = (int)RHS->getZExtValue();
1236    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1237      return false;
1238    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1239      return false;
1240  }
1241
1242  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
1243    // Compute R + (R << [1,2,3]) and reuse it.
1244    Base = N;
1245    return false;
1246  }
1247
1248  // Look for (R + R) or (R + (R << [1,2,3])).
1249  unsigned ShAmt = 0;
1250  Base   = N.getOperand(0);
1251  OffReg = N.getOperand(1);
1252
1253  // Swap if it is ((R << c) + R).
1254  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1255  if (ShOpcVal != ARM_AM::lsl) {
1256    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1257    if (ShOpcVal == ARM_AM::lsl)
1258      std::swap(Base, OffReg);
1259  }
1260
1261  if (ShOpcVal == ARM_AM::lsl) {
1262    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1263    // it.
1264    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1265      ShAmt = Sh->getZExtValue();
1266      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1267        OffReg = OffReg.getOperand(0);
1268      else {
1269        ShAmt = 0;
1270        ShOpcVal = ARM_AM::no_shift;
1271      }
1272    } else {
1273      ShOpcVal = ARM_AM::no_shift;
1274    }
1275  }
1276
1277  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1278
1279  return true;
1280}
1281
1282//===--------------------------------------------------------------------===//
1283
1284/// getAL - Returns a ARMCC::AL immediate node.
1285static inline SDValue getAL(SelectionDAG *CurDAG) {
1286  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1287}
1288
1289SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1290  LoadSDNode *LD = cast<LoadSDNode>(N);
1291  ISD::MemIndexedMode AM = LD->getAddressingMode();
1292  if (AM == ISD::UNINDEXED)
1293    return NULL;
1294
1295  EVT LoadedVT = LD->getMemoryVT();
1296  SDValue Offset, AMOpc;
1297  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1298  unsigned Opcode = 0;
1299  bool Match = false;
1300  if (LoadedVT == MVT::i32 &&
1301      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
1302    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
1303    Match = true;
1304  } else if (LoadedVT == MVT::i16 &&
1305             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1306    Match = true;
1307    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1308      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1309      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1310  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1311    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1312      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1313        Match = true;
1314        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1315      }
1316    } else {
1317      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
1318        Match = true;
1319        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
1320      }
1321    }
1322  }
1323
1324  if (Match) {
1325    SDValue Chain = LD->getChain();
1326    SDValue Base = LD->getBasePtr();
1327    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1328                     CurDAG->getRegister(0, MVT::i32), Chain };
1329    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1330                                  MVT::Other, Ops, 6);
1331  }
1332
1333  return NULL;
1334}
1335
1336SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1337  LoadSDNode *LD = cast<LoadSDNode>(N);
1338  ISD::MemIndexedMode AM = LD->getAddressingMode();
1339  if (AM == ISD::UNINDEXED)
1340    return NULL;
1341
1342  EVT LoadedVT = LD->getMemoryVT();
1343  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1344  SDValue Offset;
1345  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1346  unsigned Opcode = 0;
1347  bool Match = false;
1348  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1349    switch (LoadedVT.getSimpleVT().SimpleTy) {
1350    case MVT::i32:
1351      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1352      break;
1353    case MVT::i16:
1354      if (isSExtLd)
1355        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1356      else
1357        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1358      break;
1359    case MVT::i8:
1360    case MVT::i1:
1361      if (isSExtLd)
1362        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1363      else
1364        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1365      break;
1366    default:
1367      return NULL;
1368    }
1369    Match = true;
1370  }
1371
1372  if (Match) {
1373    SDValue Chain = LD->getChain();
1374    SDValue Base = LD->getBasePtr();
1375    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1376                     CurDAG->getRegister(0, MVT::i32), Chain };
1377    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1378                                  MVT::Other, Ops, 5);
1379  }
1380
1381  return NULL;
1382}
1383
1384/// PairSRegs - Form a D register from a pair of S registers.
1385///
1386SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
1387  DebugLoc dl = V0.getNode()->getDebugLoc();
1388  SDValue RegClass =
1389    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1390  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1391  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1392  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1393  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1394}
1395
1396/// PairDRegs - Form a quad register from a pair of D registers.
1397///
1398SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
1399  DebugLoc dl = V0.getNode()->getDebugLoc();
1400  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1401  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1402  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1403  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1404  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1405}
1406
1407/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
1408///
1409SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
1410  DebugLoc dl = V0.getNode()->getDebugLoc();
1411  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1412  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1413  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1414  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1415  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1416}
1417
1418/// QuadSRegs - Form 4 consecutive S registers.
1419///
1420SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
1421                                   SDValue V2, SDValue V3) {
1422  DebugLoc dl = V0.getNode()->getDebugLoc();
1423  SDValue RegClass =
1424    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1425  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1426  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1427  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1428  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1429  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1430                                    V2, SubReg2, V3, SubReg3 };
1431  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1432}
1433
1434/// QuadDRegs - Form 4 consecutive D registers.
1435///
1436SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
1437                                   SDValue V2, SDValue V3) {
1438  DebugLoc dl = V0.getNode()->getDebugLoc();
1439  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1440  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1441  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1442  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1443  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1444  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1445                                    V2, SubReg2, V3, SubReg3 };
1446  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1447}
1448
1449/// QuadQRegs - Form 4 consecutive Q registers.
1450///
1451SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
1452                                   SDValue V2, SDValue V3) {
1453  DebugLoc dl = V0.getNode()->getDebugLoc();
1454  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1455  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1456  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1457  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1458  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1459  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1460                                    V2, SubReg2, V3, SubReg3 };
1461  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1462}
1463
1464/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1465/// of a NEON VLD or VST instruction.  The supported values depend on the
1466/// number of registers being loaded.
1467SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1468                                       bool is64BitVector) {
1469  unsigned NumRegs = NumVecs;
1470  if (!is64BitVector && NumVecs < 3)
1471    NumRegs *= 2;
1472
1473  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1474  if (Alignment >= 32 && NumRegs == 4)
1475    Alignment = 32;
1476  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1477    Alignment = 16;
1478  else if (Alignment >= 8)
1479    Alignment = 8;
1480  else
1481    Alignment = 0;
1482
1483  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1484}
1485
1486SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1487                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1488                                   unsigned *QOpcodes1) {
1489  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1490  DebugLoc dl = N->getDebugLoc();
1491
1492  SDValue MemAddr, Align;
1493  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1494  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1495    return NULL;
1496
1497  SDValue Chain = N->getOperand(0);
1498  EVT VT = N->getValueType(0);
1499  bool is64BitVector = VT.is64BitVector();
1500  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1501
1502  unsigned OpcodeIndex;
1503  switch (VT.getSimpleVT().SimpleTy) {
1504  default: llvm_unreachable("unhandled vld type");
1505    // Double-register operations:
1506  case MVT::v8i8:  OpcodeIndex = 0; break;
1507  case MVT::v4i16: OpcodeIndex = 1; break;
1508  case MVT::v2f32:
1509  case MVT::v2i32: OpcodeIndex = 2; break;
1510  case MVT::v1i64: OpcodeIndex = 3; break;
1511    // Quad-register operations:
1512  case MVT::v16i8: OpcodeIndex = 0; break;
1513  case MVT::v8i16: OpcodeIndex = 1; break;
1514  case MVT::v4f32:
1515  case MVT::v4i32: OpcodeIndex = 2; break;
1516  case MVT::v2i64: OpcodeIndex = 3;
1517    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1518    break;
1519  }
1520
1521  EVT ResTy;
1522  if (NumVecs == 1)
1523    ResTy = VT;
1524  else {
1525    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1526    if (!is64BitVector)
1527      ResTyElts *= 2;
1528    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1529  }
1530  std::vector<EVT> ResTys;
1531  ResTys.push_back(ResTy);
1532  if (isUpdating)
1533    ResTys.push_back(MVT::i32);
1534  ResTys.push_back(MVT::Other);
1535
1536  SDValue Pred = getAL(CurDAG);
1537  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1538  SDNode *VLd;
1539  SmallVector<SDValue, 7> Ops;
1540
1541  // Double registers and VLD1/VLD2 quad registers are directly supported.
1542  if (is64BitVector || NumVecs <= 2) {
1543    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1544                    QOpcodes0[OpcodeIndex]);
1545    Ops.push_back(MemAddr);
1546    Ops.push_back(Align);
1547    if (isUpdating) {
1548      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1549      Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1550    }
1551    Ops.push_back(Pred);
1552    Ops.push_back(Reg0);
1553    Ops.push_back(Chain);
1554    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1555
1556  } else {
1557    // Otherwise, quad registers are loaded with two separate instructions,
1558    // where one loads the even registers and the other loads the odd registers.
1559    EVT AddrTy = MemAddr.getValueType();
1560
1561    // Load the even subregs.  This is always an updating load, so that it
1562    // provides the address to the second load for the odd subregs.
1563    SDValue ImplDef =
1564      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1565    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1566    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1567                                          ResTy, AddrTy, MVT::Other, OpsA, 7);
1568    Chain = SDValue(VLdA, 2);
1569
1570    // Load the odd subregs.
1571    Ops.push_back(SDValue(VLdA, 1));
1572    Ops.push_back(Align);
1573    if (isUpdating) {
1574      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1575      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1576             "only constant post-increment update allowed for VLD3/4");
1577      (void)Inc;
1578      Ops.push_back(Reg0);
1579    }
1580    Ops.push_back(SDValue(VLdA, 0));
1581    Ops.push_back(Pred);
1582    Ops.push_back(Reg0);
1583    Ops.push_back(Chain);
1584    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1585                                 Ops.data(), Ops.size());
1586  }
1587
1588  // Transfer memoperands.
1589  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1590  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1591  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1592
1593  if (NumVecs == 1)
1594    return VLd;
1595
1596  // Extract out the subregisters.
1597  SDValue SuperReg = SDValue(VLd, 0);
1598  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1599         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1600  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1601  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1602    ReplaceUses(SDValue(N, Vec),
1603                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1604  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1605  if (isUpdating)
1606    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1607  return NULL;
1608}
1609
1610SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1611                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1612                                   unsigned *QOpcodes1) {
1613  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1614  DebugLoc dl = N->getDebugLoc();
1615
1616  SDValue MemAddr, Align;
1617  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1618  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1619  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1620    return NULL;
1621
1622  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1623  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1624
1625  SDValue Chain = N->getOperand(0);
1626  EVT VT = N->getOperand(Vec0Idx).getValueType();
1627  bool is64BitVector = VT.is64BitVector();
1628  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1629
1630  unsigned OpcodeIndex;
1631  switch (VT.getSimpleVT().SimpleTy) {
1632  default: llvm_unreachable("unhandled vst type");
1633    // Double-register operations:
1634  case MVT::v8i8:  OpcodeIndex = 0; break;
1635  case MVT::v4i16: OpcodeIndex = 1; break;
1636  case MVT::v2f32:
1637  case MVT::v2i32: OpcodeIndex = 2; break;
1638  case MVT::v1i64: OpcodeIndex = 3; break;
1639    // Quad-register operations:
1640  case MVT::v16i8: OpcodeIndex = 0; break;
1641  case MVT::v8i16: OpcodeIndex = 1; break;
1642  case MVT::v4f32:
1643  case MVT::v4i32: OpcodeIndex = 2; break;
1644  case MVT::v2i64: OpcodeIndex = 3;
1645    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1646    break;
1647  }
1648
1649  std::vector<EVT> ResTys;
1650  if (isUpdating)
1651    ResTys.push_back(MVT::i32);
1652  ResTys.push_back(MVT::Other);
1653
1654  SDValue Pred = getAL(CurDAG);
1655  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1656  SmallVector<SDValue, 7> Ops;
1657
1658  // Double registers and VST1/VST2 quad registers are directly supported.
1659  if (is64BitVector || NumVecs <= 2) {
1660    SDValue SrcReg;
1661    if (NumVecs == 1) {
1662      SrcReg = N->getOperand(Vec0Idx);
1663    } else if (is64BitVector) {
1664      // Form a REG_SEQUENCE to force register allocation.
1665      SDValue V0 = N->getOperand(Vec0Idx + 0);
1666      SDValue V1 = N->getOperand(Vec0Idx + 1);
1667      if (NumVecs == 2)
1668        SrcReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1669      else {
1670        SDValue V2 = N->getOperand(Vec0Idx + 2);
1671        // If it's a vst3, form a quad D-register and leave the last part as
1672        // an undef.
1673        SDValue V3 = (NumVecs == 3)
1674          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1675          : N->getOperand(Vec0Idx + 3);
1676        SrcReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1677      }
1678    } else {
1679      // Form a QQ register.
1680      SDValue Q0 = N->getOperand(Vec0Idx);
1681      SDValue Q1 = N->getOperand(Vec0Idx + 1);
1682      SrcReg = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1683    }
1684
1685    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1686                    QOpcodes0[OpcodeIndex]);
1687    Ops.push_back(MemAddr);
1688    Ops.push_back(Align);
1689    if (isUpdating) {
1690      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1691      Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1692    }
1693    Ops.push_back(SrcReg);
1694    Ops.push_back(Pred);
1695    Ops.push_back(Reg0);
1696    Ops.push_back(Chain);
1697    SDNode *VSt =
1698      CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1699
1700    // Transfer memoperands.
1701    cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
1702
1703    return VSt;
1704  }
1705
1706  // Otherwise, quad registers are stored with two separate instructions,
1707  // where one stores the even registers and the other stores the odd registers.
1708
1709  // Form the QQQQ REG_SEQUENCE.
1710  SDValue V0 = N->getOperand(Vec0Idx + 0);
1711  SDValue V1 = N->getOperand(Vec0Idx + 1);
1712  SDValue V2 = N->getOperand(Vec0Idx + 2);
1713  SDValue V3 = (NumVecs == 3)
1714    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1715    : N->getOperand(Vec0Idx + 3);
1716  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1717
1718  // Store the even D registers.  This is always an updating store, so that it
1719  // provides the address to the second store for the odd subregs.
1720  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
1721  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1722                                        MemAddr.getValueType(),
1723                                        MVT::Other, OpsA, 7);
1724  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
1725  Chain = SDValue(VStA, 1);
1726
1727  // Store the odd D registers.
1728  Ops.push_back(SDValue(VStA, 0));
1729  Ops.push_back(Align);
1730  if (isUpdating) {
1731    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1732    assert(isa<ConstantSDNode>(Inc.getNode()) &&
1733           "only constant post-increment update allowed for VST3/4");
1734    (void)Inc;
1735    Ops.push_back(Reg0);
1736  }
1737  Ops.push_back(RegSeq);
1738  Ops.push_back(Pred);
1739  Ops.push_back(Reg0);
1740  Ops.push_back(Chain);
1741  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1742                                        Ops.data(), Ops.size());
1743  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
1744  return VStB;
1745}
1746
1747SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1748                                         bool isUpdating, unsigned NumVecs,
1749                                         unsigned *DOpcodes,
1750                                         unsigned *QOpcodes) {
1751  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1752  DebugLoc dl = N->getDebugLoc();
1753
1754  SDValue MemAddr, Align;
1755  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1756  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1757  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1758    return NULL;
1759
1760  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1761  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1762
1763  SDValue Chain = N->getOperand(0);
1764  unsigned Lane =
1765    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
1766  EVT VT = N->getOperand(Vec0Idx).getValueType();
1767  bool is64BitVector = VT.is64BitVector();
1768
1769  unsigned Alignment = 0;
1770  if (NumVecs != 3) {
1771    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1772    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1773    if (Alignment > NumBytes)
1774      Alignment = NumBytes;
1775    if (Alignment < 8 && Alignment < NumBytes)
1776      Alignment = 0;
1777    // Alignment must be a power of two; make sure of that.
1778    Alignment = (Alignment & -Alignment);
1779    if (Alignment == 1)
1780      Alignment = 0;
1781  }
1782  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1783
1784  unsigned OpcodeIndex;
1785  switch (VT.getSimpleVT().SimpleTy) {
1786  default: llvm_unreachable("unhandled vld/vst lane type");
1787    // Double-register operations:
1788  case MVT::v8i8:  OpcodeIndex = 0; break;
1789  case MVT::v4i16: OpcodeIndex = 1; break;
1790  case MVT::v2f32:
1791  case MVT::v2i32: OpcodeIndex = 2; break;
1792    // Quad-register operations:
1793  case MVT::v8i16: OpcodeIndex = 0; break;
1794  case MVT::v4f32:
1795  case MVT::v4i32: OpcodeIndex = 1; break;
1796  }
1797
1798  std::vector<EVT> ResTys;
1799  if (IsLoad) {
1800    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1801    if (!is64BitVector)
1802      ResTyElts *= 2;
1803    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
1804                                      MVT::i64, ResTyElts));
1805  }
1806  if (isUpdating)
1807    ResTys.push_back(MVT::i32);
1808  ResTys.push_back(MVT::Other);
1809
1810  SDValue Pred = getAL(CurDAG);
1811  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1812
1813  SmallVector<SDValue, 8> Ops;
1814  Ops.push_back(MemAddr);
1815  Ops.push_back(Align);
1816  if (isUpdating) {
1817    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1818    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1819  }
1820
1821  SDValue SuperReg;
1822  SDValue V0 = N->getOperand(Vec0Idx + 0);
1823  SDValue V1 = N->getOperand(Vec0Idx + 1);
1824  if (NumVecs == 2) {
1825    if (is64BitVector)
1826      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1827    else
1828      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1829  } else {
1830    SDValue V2 = N->getOperand(Vec0Idx + 2);
1831    SDValue V3 = (NumVecs == 3)
1832      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1833      : N->getOperand(Vec0Idx + 3);
1834    if (is64BitVector)
1835      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1836    else
1837      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1838  }
1839  Ops.push_back(SuperReg);
1840  Ops.push_back(getI32Imm(Lane));
1841  Ops.push_back(Pred);
1842  Ops.push_back(Reg0);
1843  Ops.push_back(Chain);
1844
1845  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1846                                  QOpcodes[OpcodeIndex]);
1847  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys,
1848                                         Ops.data(), Ops.size());
1849  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
1850  if (!IsLoad)
1851    return VLdLn;
1852
1853  // Extract the subregisters.
1854  SuperReg = SDValue(VLdLn, 0);
1855  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1856         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1857  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1858  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1859    ReplaceUses(SDValue(N, Vec),
1860                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1861  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
1862  if (isUpdating)
1863    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
1864  return NULL;
1865}
1866
1867SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
1868                                      unsigned NumVecs, unsigned *Opcodes) {
1869  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
1870  DebugLoc dl = N->getDebugLoc();
1871
1872  SDValue MemAddr, Align;
1873  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
1874    return NULL;
1875
1876  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1877  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1878
1879  SDValue Chain = N->getOperand(0);
1880  EVT VT = N->getValueType(0);
1881
1882  unsigned Alignment = 0;
1883  if (NumVecs != 3) {
1884    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1885    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1886    if (Alignment > NumBytes)
1887      Alignment = NumBytes;
1888    if (Alignment < 8 && Alignment < NumBytes)
1889      Alignment = 0;
1890    // Alignment must be a power of two; make sure of that.
1891    Alignment = (Alignment & -Alignment);
1892    if (Alignment == 1)
1893      Alignment = 0;
1894  }
1895  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1896
1897  unsigned OpcodeIndex;
1898  switch (VT.getSimpleVT().SimpleTy) {
1899  default: llvm_unreachable("unhandled vld-dup type");
1900  case MVT::v8i8:  OpcodeIndex = 0; break;
1901  case MVT::v4i16: OpcodeIndex = 1; break;
1902  case MVT::v2f32:
1903  case MVT::v2i32: OpcodeIndex = 2; break;
1904  }
1905
1906  SDValue Pred = getAL(CurDAG);
1907  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1908  SDValue SuperReg;
1909  unsigned Opc = Opcodes[OpcodeIndex];
1910  SmallVector<SDValue, 6> Ops;
1911  Ops.push_back(MemAddr);
1912  Ops.push_back(Align);
1913  if (isUpdating) {
1914    SDValue Inc = N->getOperand(2);
1915    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1916  }
1917  Ops.push_back(Pred);
1918  Ops.push_back(Reg0);
1919  Ops.push_back(Chain);
1920
1921  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1922  std::vector<EVT> ResTys;
1923  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
1924  if (isUpdating)
1925    ResTys.push_back(MVT::i32);
1926  ResTys.push_back(MVT::Other);
1927  SDNode *VLdDup =
1928    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1929  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
1930  SuperReg = SDValue(VLdDup, 0);
1931
1932  // Extract the subregisters.
1933  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1934  unsigned SubIdx = ARM::dsub_0;
1935  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1936    ReplaceUses(SDValue(N, Vec),
1937                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
1938  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
1939  if (isUpdating)
1940    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
1941  return NULL;
1942}
1943
1944SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
1945                                    unsigned Opc) {
1946  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
1947  DebugLoc dl = N->getDebugLoc();
1948  EVT VT = N->getValueType(0);
1949  unsigned FirstTblReg = IsExt ? 2 : 1;
1950
1951  // Form a REG_SEQUENCE to force register allocation.
1952  SDValue RegSeq;
1953  SDValue V0 = N->getOperand(FirstTblReg + 0);
1954  SDValue V1 = N->getOperand(FirstTblReg + 1);
1955  if (NumVecs == 2)
1956    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
1957  else {
1958    SDValue V2 = N->getOperand(FirstTblReg + 2);
1959    // If it's a vtbl3, form a quad D-register and leave the last part as
1960    // an undef.
1961    SDValue V3 = (NumVecs == 3)
1962      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1963      : N->getOperand(FirstTblReg + 3);
1964    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1965  }
1966
1967  SmallVector<SDValue, 6> Ops;
1968  if (IsExt)
1969    Ops.push_back(N->getOperand(1));
1970  Ops.push_back(RegSeq);
1971  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
1972  Ops.push_back(getAL(CurDAG)); // predicate
1973  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
1974  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
1975}
1976
1977SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1978                                                     bool isSigned) {
1979  if (!Subtarget->hasV6T2Ops())
1980    return NULL;
1981
1982  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1983    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1984
1985
1986  // For unsigned extracts, check for a shift right and mask
1987  unsigned And_imm = 0;
1988  if (N->getOpcode() == ISD::AND) {
1989    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1990
1991      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1992      if (And_imm & (And_imm + 1))
1993        return NULL;
1994
1995      unsigned Srl_imm = 0;
1996      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1997                                Srl_imm)) {
1998        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1999
2000        unsigned Width = CountTrailingOnes_32(And_imm);
2001        unsigned LSB = Srl_imm;
2002        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2003        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2004                          CurDAG->getTargetConstant(LSB, MVT::i32),
2005                          CurDAG->getTargetConstant(Width, MVT::i32),
2006          getAL(CurDAG), Reg0 };
2007        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2008      }
2009    }
2010    return NULL;
2011  }
2012
2013  // Otherwise, we're looking for a shift of a shift
2014  unsigned Shl_imm = 0;
2015  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2016    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2017    unsigned Srl_imm = 0;
2018    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2019      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2020      unsigned Width = 32 - Srl_imm;
2021      int LSB = Srl_imm - Shl_imm;
2022      if (LSB < 0)
2023        return NULL;
2024      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2025      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2026                        CurDAG->getTargetConstant(LSB, MVT::i32),
2027                        CurDAG->getTargetConstant(Width, MVT::i32),
2028                        getAL(CurDAG), Reg0 };
2029      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2030    }
2031  }
2032  return NULL;
2033}
2034
2035SDNode *ARMDAGToDAGISel::
2036SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2037                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2038  SDValue CPTmp0;
2039  SDValue CPTmp1;
2040  if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
2041    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
2042    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
2043    unsigned Opc = 0;
2044    switch (SOShOp) {
2045    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
2046    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
2047    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
2048    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
2049    default:
2050      llvm_unreachable("Unknown so_reg opcode!");
2051      break;
2052    }
2053    SDValue SOShImm =
2054      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
2055    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2056    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
2057    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
2058  }
2059  return 0;
2060}
2061
2062SDNode *ARMDAGToDAGISel::
2063SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2064                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2065  SDValue CPTmp0;
2066  SDValue CPTmp1;
2067  SDValue CPTmp2;
2068  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
2069    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2070    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag };
2071    return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6);
2072  }
2073
2074  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
2075    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2076    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
2077    return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7);
2078  }
2079  return 0;
2080}
2081
2082SDNode *ARMDAGToDAGISel::
2083SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2084                  ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2085  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2086  if (!T)
2087    return 0;
2088
2089  unsigned Opc = 0;
2090  unsigned TrueImm = T->getZExtValue();
2091  if (is_t2_so_imm(TrueImm)) {
2092    Opc = ARM::t2MOVCCi;
2093  } else if (TrueImm <= 0xffff) {
2094    Opc = ARM::t2MOVCCi16;
2095  } else if (is_t2_so_imm_not(TrueImm)) {
2096    TrueImm = ~TrueImm;
2097    Opc = ARM::t2MVNCCi;
2098  } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) {
2099    // Large immediate.
2100    Opc = ARM::t2MOVCCi32imm;
2101  }
2102
2103  if (Opc) {
2104    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2105    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2106    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2107    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2108  }
2109
2110  return 0;
2111}
2112
2113SDNode *ARMDAGToDAGISel::
2114SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2115                   ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2116  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2117  if (!T)
2118    return 0;
2119
2120  unsigned Opc = 0;
2121  unsigned TrueImm = T->getZExtValue();
2122  bool isSoImm = is_so_imm(TrueImm);
2123  if (isSoImm) {
2124    Opc = ARM::MOVCCi;
2125  } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) {
2126    Opc = ARM::MOVCCi16;
2127  } else if (is_so_imm_not(TrueImm)) {
2128    TrueImm = ~TrueImm;
2129    Opc = ARM::MVNCCi;
2130  } else if (TrueVal.getNode()->hasOneUse() &&
2131             (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) {
2132    // Large immediate.
2133    Opc = ARM::MOVCCi32imm;
2134  }
2135
2136  if (Opc) {
2137    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2138    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2139    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2140    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2141  }
2142
2143  return 0;
2144}
2145
2146SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
2147  EVT VT = N->getValueType(0);
2148  SDValue FalseVal = N->getOperand(0);
2149  SDValue TrueVal  = N->getOperand(1);
2150  SDValue CC = N->getOperand(2);
2151  SDValue CCR = N->getOperand(3);
2152  SDValue InFlag = N->getOperand(4);
2153  assert(CC.getOpcode() == ISD::Constant);
2154  assert(CCR.getOpcode() == ISD::Register);
2155  ARMCC::CondCodes CCVal =
2156    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
2157
2158  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
2159    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2160    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2161    // Pattern complexity = 18  cost = 1  size = 0
2162    SDValue CPTmp0;
2163    SDValue CPTmp1;
2164    SDValue CPTmp2;
2165    if (Subtarget->isThumb()) {
2166      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
2167                                        CCVal, CCR, InFlag);
2168      if (!Res)
2169        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
2170                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2171      if (Res)
2172        return Res;
2173    } else {
2174      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
2175                                         CCVal, CCR, InFlag);
2176      if (!Res)
2177        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
2178                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2179      if (Res)
2180        return Res;
2181    }
2182
2183    // Pattern: (ARMcmov:i32 GPR:i32:$false,
2184    //             (imm:i32)<<P:Pred_so_imm>>:$true,
2185    //             (imm:i32):$cc)
2186    // Emits: (MOVCCi:i32 GPR:i32:$false,
2187    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
2188    // Pattern complexity = 10  cost = 1  size = 0
2189    if (Subtarget->isThumb()) {
2190      SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal,
2191                                        CCVal, CCR, InFlag);
2192      if (!Res)
2193        Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal,
2194                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2195      if (Res)
2196        return Res;
2197    } else {
2198      SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal,
2199                                         CCVal, CCR, InFlag);
2200      if (!Res)
2201        Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal,
2202                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2203      if (Res)
2204        return Res;
2205    }
2206  }
2207
2208  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2209  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2210  // Pattern complexity = 6  cost = 1  size = 0
2211  //
2212  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2213  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2214  // Pattern complexity = 6  cost = 11  size = 0
2215  //
2216  // Also VMOVScc and VMOVDcc.
2217  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
2218  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
2219  unsigned Opc = 0;
2220  switch (VT.getSimpleVT().SimpleTy) {
2221  default: assert(false && "Illegal conditional move type!");
2222    break;
2223  case MVT::i32:
2224    Opc = Subtarget->isThumb()
2225      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
2226      : ARM::MOVCCr;
2227    break;
2228  case MVT::f32:
2229    Opc = ARM::VMOVScc;
2230    break;
2231  case MVT::f64:
2232    Opc = ARM::VMOVDcc;
2233    break;
2234  }
2235  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2236}
2237
2238SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2239  // The only time a CONCAT_VECTORS operation can have legal types is when
2240  // two 64-bit vectors are concatenated to a 128-bit vector.
2241  EVT VT = N->getValueType(0);
2242  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2243    llvm_unreachable("unexpected CONCAT_VECTORS");
2244  return PairDRegs(VT, N->getOperand(0), N->getOperand(1));
2245}
2246
2247SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2248  DebugLoc dl = N->getDebugLoc();
2249
2250  if (N->isMachineOpcode())
2251    return NULL;   // Already selected.
2252
2253  switch (N->getOpcode()) {
2254  default: break;
2255  case ISD::Constant: {
2256    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2257    bool UseCP = true;
2258    if (Subtarget->hasThumb2())
2259      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2260      // be done with MOV + MOVT, at worst.
2261      UseCP = 0;
2262    else {
2263      if (Subtarget->isThumb()) {
2264        UseCP = (Val > 255 &&                          // MOV
2265                 ~Val > 255 &&                         // MOV + MVN
2266                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
2267      } else
2268        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
2269                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
2270                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
2271    }
2272
2273    if (UseCP) {
2274      SDValue CPIdx =
2275        CurDAG->getTargetConstantPool(ConstantInt::get(
2276                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
2277                                      TLI.getPointerTy());
2278
2279      SDNode *ResNode;
2280      if (Subtarget->isThumb1Only()) {
2281        SDValue Pred = getAL(CurDAG);
2282        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2283        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2284        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2285                                         Ops, 4);
2286      } else {
2287        SDValue Ops[] = {
2288          CPIdx,
2289          CurDAG->getTargetConstant(0, MVT::i32),
2290          getAL(CurDAG),
2291          CurDAG->getRegister(0, MVT::i32),
2292          CurDAG->getEntryNode()
2293        };
2294        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2295                                       Ops, 5);
2296      }
2297      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2298      return NULL;
2299    }
2300
2301    // Other cases are autogenerated.
2302    break;
2303  }
2304  case ISD::FrameIndex: {
2305    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2306    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2307    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
2308    if (Subtarget->isThumb1Only()) {
2309      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
2310                                  CurDAG->getTargetConstant(0, MVT::i32));
2311    } else {
2312      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2313                      ARM::t2ADDri : ARM::ADDri);
2314      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2315                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2316                        CurDAG->getRegister(0, MVT::i32) };
2317      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2318    }
2319  }
2320  case ISD::SRL:
2321    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2322      return I;
2323    break;
2324  case ISD::SRA:
2325    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2326      return I;
2327    break;
2328  case ISD::MUL:
2329    if (Subtarget->isThumb1Only())
2330      break;
2331    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2332      unsigned RHSV = C->getZExtValue();
2333      if (!RHSV) break;
2334      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2335        unsigned ShImm = Log2_32(RHSV-1);
2336        if (ShImm >= 32)
2337          break;
2338        SDValue V = N->getOperand(0);
2339        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2340        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2341        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2342        if (Subtarget->isThumb()) {
2343          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2344          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
2345        } else {
2346          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2347          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7);
2348        }
2349      }
2350      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2351        unsigned ShImm = Log2_32(RHSV+1);
2352        if (ShImm >= 32)
2353          break;
2354        SDValue V = N->getOperand(0);
2355        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2356        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2357        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2358        if (Subtarget->isThumb()) {
2359          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2360          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
2361        } else {
2362          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2363          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7);
2364        }
2365      }
2366    }
2367    break;
2368  case ISD::AND: {
2369    // Check for unsigned bitfield extract
2370    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2371      return I;
2372
2373    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2374    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2375    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2376    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2377    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2378    EVT VT = N->getValueType(0);
2379    if (VT != MVT::i32)
2380      break;
2381    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2382      ? ARM::t2MOVTi16
2383      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2384    if (!Opc)
2385      break;
2386    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2387    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2388    if (!N1C)
2389      break;
2390    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2391      SDValue N2 = N0.getOperand(1);
2392      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2393      if (!N2C)
2394        break;
2395      unsigned N1CVal = N1C->getZExtValue();
2396      unsigned N2CVal = N2C->getZExtValue();
2397      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2398          (N1CVal & 0xffffU) == 0xffffU &&
2399          (N2CVal & 0xffffU) == 0x0U) {
2400        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2401                                                  MVT::i32);
2402        SDValue Ops[] = { N0.getOperand(0), Imm16,
2403                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2404        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
2405      }
2406    }
2407    break;
2408  }
2409  case ARMISD::VMOVRRD:
2410    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2411                                  N->getOperand(0), getAL(CurDAG),
2412                                  CurDAG->getRegister(0, MVT::i32));
2413  case ISD::UMUL_LOHI: {
2414    if (Subtarget->isThumb1Only())
2415      break;
2416    if (Subtarget->isThumb()) {
2417      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2418                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2419                        CurDAG->getRegister(0, MVT::i32) };
2420      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
2421    } else {
2422      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2423                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2424                        CurDAG->getRegister(0, MVT::i32) };
2425      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2426                                    ARM::UMULL : ARM::UMULLv5,
2427                                    dl, MVT::i32, MVT::i32, Ops, 5);
2428    }
2429  }
2430  case ISD::SMUL_LOHI: {
2431    if (Subtarget->isThumb1Only())
2432      break;
2433    if (Subtarget->isThumb()) {
2434      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2435                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2436      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
2437    } else {
2438      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2439                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2440                        CurDAG->getRegister(0, MVT::i32) };
2441      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2442                                    ARM::SMULL : ARM::SMULLv5,
2443                                    dl, MVT::i32, MVT::i32, Ops, 5);
2444    }
2445  }
2446  case ISD::LOAD: {
2447    SDNode *ResNode = 0;
2448    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2449      ResNode = SelectT2IndexedLoad(N);
2450    else
2451      ResNode = SelectARMIndexedLoad(N);
2452    if (ResNode)
2453      return ResNode;
2454    // Other cases are autogenerated.
2455    break;
2456  }
2457  case ARMISD::BRCOND: {
2458    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2459    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2460    // Pattern complexity = 6  cost = 1  size = 0
2461
2462    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2463    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2464    // Pattern complexity = 6  cost = 1  size = 0
2465
2466    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2467    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2468    // Pattern complexity = 6  cost = 1  size = 0
2469
2470    unsigned Opc = Subtarget->isThumb() ?
2471      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2472    SDValue Chain = N->getOperand(0);
2473    SDValue N1 = N->getOperand(1);
2474    SDValue N2 = N->getOperand(2);
2475    SDValue N3 = N->getOperand(3);
2476    SDValue InFlag = N->getOperand(4);
2477    assert(N1.getOpcode() == ISD::BasicBlock);
2478    assert(N2.getOpcode() == ISD::Constant);
2479    assert(N3.getOpcode() == ISD::Register);
2480
2481    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2482                               cast<ConstantSDNode>(N2)->getZExtValue()),
2483                               MVT::i32);
2484    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2485    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2486                                             MVT::Glue, Ops, 5);
2487    Chain = SDValue(ResNode, 0);
2488    if (N->getNumValues() == 2) {
2489      InFlag = SDValue(ResNode, 1);
2490      ReplaceUses(SDValue(N, 1), InFlag);
2491    }
2492    ReplaceUses(SDValue(N, 0),
2493                SDValue(Chain.getNode(), Chain.getResNo()));
2494    return NULL;
2495  }
2496  case ARMISD::CMOV:
2497    return SelectCMOVOp(N);
2498  case ARMISD::VZIP: {
2499    unsigned Opc = 0;
2500    EVT VT = N->getValueType(0);
2501    switch (VT.getSimpleVT().SimpleTy) {
2502    default: return NULL;
2503    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2504    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2505    case MVT::v2f32:
2506    case MVT::v2i32: Opc = ARM::VZIPd32; break;
2507    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2508    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2509    case MVT::v4f32:
2510    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2511    }
2512    SDValue Pred = getAL(CurDAG);
2513    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2514    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2515    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2516  }
2517  case ARMISD::VUZP: {
2518    unsigned Opc = 0;
2519    EVT VT = N->getValueType(0);
2520    switch (VT.getSimpleVT().SimpleTy) {
2521    default: return NULL;
2522    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2523    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2524    case MVT::v2f32:
2525    case MVT::v2i32: Opc = ARM::VUZPd32; break;
2526    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2527    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2528    case MVT::v4f32:
2529    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2530    }
2531    SDValue Pred = getAL(CurDAG);
2532    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2533    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2534    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2535  }
2536  case ARMISD::VTRN: {
2537    unsigned Opc = 0;
2538    EVT VT = N->getValueType(0);
2539    switch (VT.getSimpleVT().SimpleTy) {
2540    default: return NULL;
2541    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2542    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2543    case MVT::v2f32:
2544    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2545    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2546    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2547    case MVT::v4f32:
2548    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2549    }
2550    SDValue Pred = getAL(CurDAG);
2551    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2552    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2553    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2554  }
2555  case ARMISD::BUILD_VECTOR: {
2556    EVT VecVT = N->getValueType(0);
2557    EVT EltVT = VecVT.getVectorElementType();
2558    unsigned NumElts = VecVT.getVectorNumElements();
2559    if (EltVT == MVT::f64) {
2560      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2561      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
2562    }
2563    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2564    if (NumElts == 2)
2565      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
2566    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2567    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
2568                     N->getOperand(2), N->getOperand(3));
2569  }
2570
2571  case ARMISD::VLD2DUP: {
2572    unsigned Opcodes[] = { ARM::VLD2DUPd8Pseudo, ARM::VLD2DUPd16Pseudo,
2573                           ARM::VLD2DUPd32Pseudo };
2574    return SelectVLDDup(N, false, 2, Opcodes);
2575  }
2576
2577  case ARMISD::VLD3DUP: {
2578    unsigned Opcodes[] = { ARM::VLD3DUPd8Pseudo, ARM::VLD3DUPd16Pseudo,
2579                           ARM::VLD3DUPd32Pseudo };
2580    return SelectVLDDup(N, false, 3, Opcodes);
2581  }
2582
2583  case ARMISD::VLD4DUP: {
2584    unsigned Opcodes[] = { ARM::VLD4DUPd8Pseudo, ARM::VLD4DUPd16Pseudo,
2585                           ARM::VLD4DUPd32Pseudo };
2586    return SelectVLDDup(N, false, 4, Opcodes);
2587  }
2588
2589  case ARMISD::VLD2DUP_UPD: {
2590    unsigned Opcodes[] = { ARM::VLD2DUPd8Pseudo_UPD, ARM::VLD2DUPd16Pseudo_UPD,
2591                           ARM::VLD2DUPd32Pseudo_UPD };
2592    return SelectVLDDup(N, true, 2, Opcodes);
2593  }
2594
2595  case ARMISD::VLD3DUP_UPD: {
2596    unsigned Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, ARM::VLD3DUPd16Pseudo_UPD,
2597                           ARM::VLD3DUPd32Pseudo_UPD };
2598    return SelectVLDDup(N, true, 3, Opcodes);
2599  }
2600
2601  case ARMISD::VLD4DUP_UPD: {
2602    unsigned Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, ARM::VLD4DUPd16Pseudo_UPD,
2603                           ARM::VLD4DUPd32Pseudo_UPD };
2604    return SelectVLDDup(N, true, 4, Opcodes);
2605  }
2606
2607  case ARMISD::VLD1_UPD: {
2608    unsigned DOpcodes[] = { ARM::VLD1d8_UPD, ARM::VLD1d16_UPD,
2609                            ARM::VLD1d32_UPD, ARM::VLD1d64_UPD };
2610    unsigned QOpcodes[] = { ARM::VLD1q8Pseudo_UPD, ARM::VLD1q16Pseudo_UPD,
2611                            ARM::VLD1q32Pseudo_UPD, ARM::VLD1q64Pseudo_UPD };
2612    return SelectVLD(N, true, 1, DOpcodes, QOpcodes, 0);
2613  }
2614
2615  case ARMISD::VLD2_UPD: {
2616    unsigned DOpcodes[] = { ARM::VLD2d8Pseudo_UPD, ARM::VLD2d16Pseudo_UPD,
2617                            ARM::VLD2d32Pseudo_UPD, ARM::VLD1q64Pseudo_UPD };
2618    unsigned QOpcodes[] = { ARM::VLD2q8Pseudo_UPD, ARM::VLD2q16Pseudo_UPD,
2619                            ARM::VLD2q32Pseudo_UPD };
2620    return SelectVLD(N, true, 2, DOpcodes, QOpcodes, 0);
2621  }
2622
2623  case ARMISD::VLD3_UPD: {
2624    unsigned DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, ARM::VLD3d16Pseudo_UPD,
2625                            ARM::VLD3d32Pseudo_UPD, ARM::VLD1d64TPseudo_UPD };
2626    unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2627                             ARM::VLD3q16Pseudo_UPD,
2628                             ARM::VLD3q32Pseudo_UPD };
2629    unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2630                             ARM::VLD3q16oddPseudo_UPD,
2631                             ARM::VLD3q32oddPseudo_UPD };
2632    return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2633  }
2634
2635  case ARMISD::VLD4_UPD: {
2636    unsigned DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD,
2637                            ARM::VLD4d32Pseudo_UPD, ARM::VLD1d64QPseudo_UPD };
2638    unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2639                             ARM::VLD4q16Pseudo_UPD,
2640                             ARM::VLD4q32Pseudo_UPD };
2641    unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2642                             ARM::VLD4q16oddPseudo_UPD,
2643                             ARM::VLD4q32oddPseudo_UPD };
2644    return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2645  }
2646
2647  case ARMISD::VLD2LN_UPD: {
2648    unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, ARM::VLD2LNd16Pseudo_UPD,
2649                            ARM::VLD2LNd32Pseudo_UPD };
2650    unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2651                            ARM::VLD2LNq32Pseudo_UPD };
2652    return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2653  }
2654
2655  case ARMISD::VLD3LN_UPD: {
2656    unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, ARM::VLD3LNd16Pseudo_UPD,
2657                            ARM::VLD3LNd32Pseudo_UPD };
2658    unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2659                            ARM::VLD3LNq32Pseudo_UPD };
2660    return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2661  }
2662
2663  case ARMISD::VLD4LN_UPD: {
2664    unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, ARM::VLD4LNd16Pseudo_UPD,
2665                            ARM::VLD4LNd32Pseudo_UPD };
2666    unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
2667                            ARM::VLD4LNq32Pseudo_UPD };
2668    return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
2669  }
2670
2671  case ARMISD::VST1_UPD: {
2672    unsigned DOpcodes[] = { ARM::VST1d8_UPD, ARM::VST1d16_UPD,
2673                            ARM::VST1d32_UPD, ARM::VST1d64_UPD };
2674    unsigned QOpcodes[] = { ARM::VST1q8Pseudo_UPD, ARM::VST1q16Pseudo_UPD,
2675                            ARM::VST1q32Pseudo_UPD, ARM::VST1q64Pseudo_UPD };
2676    return SelectVST(N, true, 1, DOpcodes, QOpcodes, 0);
2677  }
2678
2679  case ARMISD::VST2_UPD: {
2680    unsigned DOpcodes[] = { ARM::VST2d8Pseudo_UPD, ARM::VST2d16Pseudo_UPD,
2681                            ARM::VST2d32Pseudo_UPD, ARM::VST1q64Pseudo_UPD };
2682    unsigned QOpcodes[] = { ARM::VST2q8Pseudo_UPD, ARM::VST2q16Pseudo_UPD,
2683                            ARM::VST2q32Pseudo_UPD };
2684    return SelectVST(N, true, 2, DOpcodes, QOpcodes, 0);
2685  }
2686
2687  case ARMISD::VST3_UPD: {
2688    unsigned DOpcodes[] = { ARM::VST3d8Pseudo_UPD, ARM::VST3d16Pseudo_UPD,
2689                            ARM::VST3d32Pseudo_UPD, ARM::VST1d64TPseudo_UPD };
2690    unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2691                             ARM::VST3q16Pseudo_UPD,
2692                             ARM::VST3q32Pseudo_UPD };
2693    unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2694                             ARM::VST3q16oddPseudo_UPD,
2695                             ARM::VST3q32oddPseudo_UPD };
2696    return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2697  }
2698
2699  case ARMISD::VST4_UPD: {
2700    unsigned DOpcodes[] = { ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD,
2701                            ARM::VST4d32Pseudo_UPD, ARM::VST1d64QPseudo_UPD };
2702    unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2703                             ARM::VST4q16Pseudo_UPD,
2704                             ARM::VST4q32Pseudo_UPD };
2705    unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2706                             ARM::VST4q16oddPseudo_UPD,
2707                             ARM::VST4q32oddPseudo_UPD };
2708    return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2709  }
2710
2711  case ARMISD::VST2LN_UPD: {
2712    unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, ARM::VST2LNd16Pseudo_UPD,
2713                            ARM::VST2LNd32Pseudo_UPD };
2714    unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
2715                            ARM::VST2LNq32Pseudo_UPD };
2716    return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
2717  }
2718
2719  case ARMISD::VST3LN_UPD: {
2720    unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, ARM::VST3LNd16Pseudo_UPD,
2721                            ARM::VST3LNd32Pseudo_UPD };
2722    unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
2723                            ARM::VST3LNq32Pseudo_UPD };
2724    return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
2725  }
2726
2727  case ARMISD::VST4LN_UPD: {
2728    unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, ARM::VST4LNd16Pseudo_UPD,
2729                            ARM::VST4LNd32Pseudo_UPD };
2730    unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
2731                            ARM::VST4LNq32Pseudo_UPD };
2732    return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
2733  }
2734
2735  case ISD::INTRINSIC_VOID:
2736  case ISD::INTRINSIC_W_CHAIN: {
2737    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2738    switch (IntNo) {
2739    default:
2740      break;
2741
2742    case Intrinsic::arm_ldrexd: {
2743      SDValue MemAddr = N->getOperand(2);
2744      DebugLoc dl = N->getDebugLoc();
2745      SDValue Chain = N->getOperand(0);
2746
2747      unsigned NewOpc = ARM::LDREXD;
2748      if (Subtarget->isThumb() && Subtarget->hasThumb2())
2749        NewOpc = ARM::t2LDREXD;
2750
2751      // arm_ldrexd returns a i64 value in {i32, i32}
2752      std::vector<EVT> ResTys;
2753      ResTys.push_back(MVT::i32);
2754      ResTys.push_back(MVT::i32);
2755      ResTys.push_back(MVT::Other);
2756
2757      // place arguments in the right order
2758      SmallVector<SDValue, 7> Ops;
2759      Ops.push_back(MemAddr);
2760      Ops.push_back(getAL(CurDAG));
2761      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2762      Ops.push_back(Chain);
2763      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
2764                                          Ops.size());
2765      // Transfer memoperands.
2766      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2767      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2768      cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
2769
2770      // Until there's support for specifing explicit register constraints
2771      // like the use of even/odd register pair, hardcode ldrexd to always
2772      // use the pair [R0, R1] to hold the load result.
2773      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R0,
2774                                   SDValue(Ld, 0), SDValue(0,0));
2775      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R1,
2776                                   SDValue(Ld, 1), Chain.getValue(1));
2777
2778      // Remap uses.
2779      SDValue Glue = Chain.getValue(1);
2780      if (!SDValue(N, 0).use_empty()) {
2781        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2782                                                ARM::R0, MVT::i32, Glue);
2783        Glue = Result.getValue(2);
2784        ReplaceUses(SDValue(N, 0), Result);
2785      }
2786      if (!SDValue(N, 1).use_empty()) {
2787        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2788                                                ARM::R1, MVT::i32, Glue);
2789        Glue = Result.getValue(2);
2790        ReplaceUses(SDValue(N, 1), Result);
2791      }
2792
2793      ReplaceUses(SDValue(N, 2), SDValue(Ld, 2));
2794      return NULL;
2795    }
2796
2797    case Intrinsic::arm_strexd: {
2798      DebugLoc dl = N->getDebugLoc();
2799      SDValue Chain = N->getOperand(0);
2800      SDValue Val0 = N->getOperand(2);
2801      SDValue Val1 = N->getOperand(3);
2802      SDValue MemAddr = N->getOperand(4);
2803
2804      // Until there's support for specifing explicit register constraints
2805      // like the use of even/odd register pair, hardcode strexd to always
2806      // use the pair [R2, R3] to hold the i64 (i32, i32) value to be stored.
2807      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R2, Val0,
2808                                   SDValue(0, 0));
2809      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R3, Val1, Chain.getValue(1));
2810
2811      SDValue Glue = Chain.getValue(1);
2812      Val0 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2813                                    ARM::R2, MVT::i32, Glue);
2814      Glue = Val0.getValue(1);
2815      Val1 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
2816                                    ARM::R3, MVT::i32, Glue);
2817
2818      // Store exclusive double return a i32 value which is the return status
2819      // of the issued store.
2820      std::vector<EVT> ResTys;
2821      ResTys.push_back(MVT::i32);
2822      ResTys.push_back(MVT::Other);
2823
2824      // place arguments in the right order
2825      SmallVector<SDValue, 7> Ops;
2826      Ops.push_back(Val0);
2827      Ops.push_back(Val1);
2828      Ops.push_back(MemAddr);
2829      Ops.push_back(getAL(CurDAG));
2830      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2831      Ops.push_back(Chain);
2832
2833      unsigned NewOpc = ARM::STREXD;
2834      if (Subtarget->isThumb() && Subtarget->hasThumb2())
2835        NewOpc = ARM::t2STREXD;
2836
2837      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
2838                                          Ops.size());
2839      // Transfer memoperands.
2840      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2841      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2842      cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
2843
2844      return St;
2845    }
2846
2847    case Intrinsic::arm_neon_vld1: {
2848      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
2849                              ARM::VLD1d32, ARM::VLD1d64 };
2850      unsigned QOpcodes[] = { ARM::VLD1q8Pseudo, ARM::VLD1q16Pseudo,
2851                              ARM::VLD1q32Pseudo, ARM::VLD1q64Pseudo };
2852      return SelectVLD(N, false, 1, DOpcodes, QOpcodes, 0);
2853    }
2854
2855    case Intrinsic::arm_neon_vld2: {
2856      unsigned DOpcodes[] = { ARM::VLD2d8Pseudo, ARM::VLD2d16Pseudo,
2857                              ARM::VLD2d32Pseudo, ARM::VLD1q64Pseudo };
2858      unsigned QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
2859                              ARM::VLD2q32Pseudo };
2860      return SelectVLD(N, false, 2, DOpcodes, QOpcodes, 0);
2861    }
2862
2863    case Intrinsic::arm_neon_vld3: {
2864      unsigned DOpcodes[] = { ARM::VLD3d8Pseudo, ARM::VLD3d16Pseudo,
2865                              ARM::VLD3d32Pseudo, ARM::VLD1d64TPseudo };
2866      unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2867                               ARM::VLD3q16Pseudo_UPD,
2868                               ARM::VLD3q32Pseudo_UPD };
2869      unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo,
2870                               ARM::VLD3q16oddPseudo,
2871                               ARM::VLD3q32oddPseudo };
2872      return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
2873    }
2874
2875    case Intrinsic::arm_neon_vld4: {
2876      unsigned DOpcodes[] = { ARM::VLD4d8Pseudo, ARM::VLD4d16Pseudo,
2877                              ARM::VLD4d32Pseudo, ARM::VLD1d64QPseudo };
2878      unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2879                               ARM::VLD4q16Pseudo_UPD,
2880                               ARM::VLD4q32Pseudo_UPD };
2881      unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo,
2882                               ARM::VLD4q16oddPseudo,
2883                               ARM::VLD4q32oddPseudo };
2884      return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
2885    }
2886
2887    case Intrinsic::arm_neon_vld2lane: {
2888      unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo, ARM::VLD2LNd16Pseudo,
2889                              ARM::VLD2LNd32Pseudo };
2890      unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo, ARM::VLD2LNq32Pseudo };
2891      return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
2892    }
2893
2894    case Intrinsic::arm_neon_vld3lane: {
2895      unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo, ARM::VLD3LNd16Pseudo,
2896                              ARM::VLD3LNd32Pseudo };
2897      unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo, ARM::VLD3LNq32Pseudo };
2898      return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
2899    }
2900
2901    case Intrinsic::arm_neon_vld4lane: {
2902      unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo, ARM::VLD4LNd16Pseudo,
2903                              ARM::VLD4LNd32Pseudo };
2904      unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo, ARM::VLD4LNq32Pseudo };
2905      return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
2906    }
2907
2908    case Intrinsic::arm_neon_vst1: {
2909      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2910                              ARM::VST1d32, ARM::VST1d64 };
2911      unsigned QOpcodes[] = { ARM::VST1q8Pseudo, ARM::VST1q16Pseudo,
2912                              ARM::VST1q32Pseudo, ARM::VST1q64Pseudo };
2913      return SelectVST(N, false, 1, DOpcodes, QOpcodes, 0);
2914    }
2915
2916    case Intrinsic::arm_neon_vst2: {
2917      unsigned DOpcodes[] = { ARM::VST2d8Pseudo, ARM::VST2d16Pseudo,
2918                              ARM::VST2d32Pseudo, ARM::VST1q64Pseudo };
2919      unsigned QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
2920                              ARM::VST2q32Pseudo };
2921      return SelectVST(N, false, 2, DOpcodes, QOpcodes, 0);
2922    }
2923
2924    case Intrinsic::arm_neon_vst3: {
2925      unsigned DOpcodes[] = { ARM::VST3d8Pseudo, ARM::VST3d16Pseudo,
2926                              ARM::VST3d32Pseudo, ARM::VST1d64TPseudo };
2927      unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2928                               ARM::VST3q16Pseudo_UPD,
2929                               ARM::VST3q32Pseudo_UPD };
2930      unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo,
2931                               ARM::VST3q16oddPseudo,
2932                               ARM::VST3q32oddPseudo };
2933      return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
2934    }
2935
2936    case Intrinsic::arm_neon_vst4: {
2937      unsigned DOpcodes[] = { ARM::VST4d8Pseudo, ARM::VST4d16Pseudo,
2938                              ARM::VST4d32Pseudo, ARM::VST1d64QPseudo };
2939      unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2940                               ARM::VST4q16Pseudo_UPD,
2941                               ARM::VST4q32Pseudo_UPD };
2942      unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo,
2943                               ARM::VST4q16oddPseudo,
2944                               ARM::VST4q32oddPseudo };
2945      return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
2946    }
2947
2948    case Intrinsic::arm_neon_vst2lane: {
2949      unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo, ARM::VST2LNd16Pseudo,
2950                              ARM::VST2LNd32Pseudo };
2951      unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo, ARM::VST2LNq32Pseudo };
2952      return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
2953    }
2954
2955    case Intrinsic::arm_neon_vst3lane: {
2956      unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo, ARM::VST3LNd16Pseudo,
2957                              ARM::VST3LNd32Pseudo };
2958      unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo, ARM::VST3LNq32Pseudo };
2959      return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
2960    }
2961
2962    case Intrinsic::arm_neon_vst4lane: {
2963      unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo, ARM::VST4LNd16Pseudo,
2964                              ARM::VST4LNd32Pseudo };
2965      unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo, ARM::VST4LNq32Pseudo };
2966      return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
2967    }
2968    }
2969    break;
2970  }
2971
2972  case ISD::INTRINSIC_WO_CHAIN: {
2973    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2974    switch (IntNo) {
2975    default:
2976      break;
2977
2978    case Intrinsic::arm_neon_vtbl2:
2979      return SelectVTBL(N, false, 2, ARM::VTBL2Pseudo);
2980    case Intrinsic::arm_neon_vtbl3:
2981      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
2982    case Intrinsic::arm_neon_vtbl4:
2983      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
2984
2985    case Intrinsic::arm_neon_vtbx2:
2986      return SelectVTBL(N, true, 2, ARM::VTBX2Pseudo);
2987    case Intrinsic::arm_neon_vtbx3:
2988      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
2989    case Intrinsic::arm_neon_vtbx4:
2990      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
2991    }
2992    break;
2993  }
2994
2995  case ARMISD::VTBL1: {
2996    DebugLoc dl = N->getDebugLoc();
2997    EVT VT = N->getValueType(0);
2998    SmallVector<SDValue, 6> Ops;
2999
3000    Ops.push_back(N->getOperand(0));
3001    Ops.push_back(N->getOperand(1));
3002    Ops.push_back(getAL(CurDAG));                    // Predicate
3003    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3004    return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops.data(), Ops.size());
3005  }
3006  case ARMISD::VTBL2: {
3007    DebugLoc dl = N->getDebugLoc();
3008    EVT VT = N->getValueType(0);
3009
3010    // Form a REG_SEQUENCE to force register allocation.
3011    SDValue V0 = N->getOperand(0);
3012    SDValue V1 = N->getOperand(1);
3013    SDValue RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
3014
3015    SmallVector<SDValue, 6> Ops;
3016    Ops.push_back(RegSeq);
3017    Ops.push_back(N->getOperand(2));
3018    Ops.push_back(getAL(CurDAG));                    // Predicate
3019    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3020    return CurDAG->getMachineNode(ARM::VTBL2Pseudo, dl, VT,
3021                                  Ops.data(), Ops.size());
3022  }
3023
3024  case ISD::CONCAT_VECTORS:
3025    return SelectConcatVector(N);
3026  }
3027
3028  return SelectCode(N);
3029}
3030
3031bool ARMDAGToDAGISel::
3032SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
3033                             std::vector<SDValue> &OutOps) {
3034  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
3035  // Require the address to be in a register.  That is safe for all ARM
3036  // variants and it is hard to do anything much smarter without knowing
3037  // how the operand is used.
3038  OutOps.push_back(Op);
3039  return false;
3040}
3041
3042/// createARMISelDag - This pass converts a legalized DAG into a
3043/// ARM-specific DAG, ready for instruction scheduling.
3044///
3045FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3046                                     CodeGenOpt::Level OptLevel) {
3047  return new ARMDAGToDAGISel(TM, OptLevel);
3048}
3049