NVPTXISelDAGToDAG.cpp revision fe16848601bdde6e3a5e0860199169dd171222a4 
1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the NVPTX target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "NVPTXISelDAGToDAG.h"
15#include "llvm/IR/GlobalValue.h"
16#include "llvm/IR/Instructions.h"
17#include "llvm/Support/CommandLine.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
20#include "llvm/Support/raw_ostream.h"
21#include "llvm/Target/TargetIntrinsicInfo.h"
22
23#undef DEBUG_TYPE
24#define DEBUG_TYPE "nvptx-isel"
25
26using namespace llvm;
27
28static cl::opt<int>
29FMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden,
30                 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
31                          " 1: do it  2: do it aggressively"),
32                 cl::init(2));
33
34static cl::opt<int> UsePrecDivF32(
35    "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
36    cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
37             " IEEE Compliant F32 div.rnd if avaiable."),
38    cl::init(2));
39
40static cl::opt<bool>
41UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
42          cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
43          cl::init(true));
44
45static cl::opt<bool>
46FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
47           cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
48           cl::init(false));
49
50
51/// createNVPTXISelDag - This pass converts a legalized DAG into a
52/// NVPTX-specific DAG, ready for instruction scheduling.
53FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
54                                       llvm::CodeGenOpt::Level OptLevel) {
55  return new NVPTXDAGToDAGISel(TM, OptLevel);
56}
57
58NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
59                                     CodeGenOpt::Level OptLevel)
60    : SelectionDAGISel(tm, OptLevel),
61      Subtarget(tm.getSubtarget<NVPTXSubtarget>()) {
62
63  doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1);
64  doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1);
65  doFMAF32AGG =
66      (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2);
67  doFMAF64AGG =
68      (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2);
69
70  allowFMA = (FMAContractLevel >= 1);
71
72  doMulWide = (OptLevel > 0);
73}
74
75int NVPTXDAGToDAGISel::getDivF32Level() const {
76  if (UsePrecDivF32.getNumOccurrences() > 0) {
77    // If nvptx-prec-div32=N is used on the command-line, always honor it
78    return UsePrecDivF32;
79  } else {
80    // Otherwise, use div.approx if fast math is enabled
81    if (TM.Options.UnsafeFPMath)
82      return 0;
83    else
84      return 2;
85  }
86}
87
88bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
89  if (UsePrecSqrtF32.getNumOccurrences() > 0) {
90    // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
91    return UsePrecSqrtF32;
92  } else {
93    // Otherwise, use sqrt.approx if fast math is enabled
94    if (TM.Options.UnsafeFPMath)
95      return false;
96    else
97      return true;
98  }
99}
100
101bool NVPTXDAGToDAGISel::useF32FTZ() const {
102  if (FtzEnabled.getNumOccurrences() > 0) {
103    // If nvptx-f32ftz is used on the command-line, always honor it
104    return FtzEnabled;
105  } else {
106    const Function *F = MF->getFunction();
107    // Otherwise, check for an nvptx-f32ftz attribute on the function
108    if (F->hasFnAttribute("nvptx-f32ftz"))
109      return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex,
110                                              "nvptx-f32ftz")
111                                              .getValueAsString() == "true");
112    else
113      return false;
114  }
115}
116
117/// Select - Select instructions not customized! Used for
118/// expanded, promoted and normal instructions.
119SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
120
121  if (N->isMachineOpcode()) {
122    N->setNodeId(-1);
123    return NULL; // Already selected.
124  }
125
126  SDNode *ResNode = NULL;
127  switch (N->getOpcode()) {
128  case ISD::LOAD:
129    ResNode = SelectLoad(N);
130    break;
131  case ISD::STORE:
132    ResNode = SelectStore(N);
133    break;
134  case NVPTXISD::LoadV2:
135  case NVPTXISD::LoadV4:
136    ResNode = SelectLoadVector(N);
137    break;
138  case NVPTXISD::LDGV2:
139  case NVPTXISD::LDGV4:
140  case NVPTXISD::LDUV2:
141  case NVPTXISD::LDUV4:
142    ResNode = SelectLDGLDUVector(N);
143    break;
144  case NVPTXISD::StoreV2:
145  case NVPTXISD::StoreV4:
146    ResNode = SelectStoreVector(N);
147    break;
148  case NVPTXISD::LoadParam:
149  case NVPTXISD::LoadParamV2:
150  case NVPTXISD::LoadParamV4:
151    ResNode = SelectLoadParam(N);
152    break;
153  case NVPTXISD::StoreRetval:
154  case NVPTXISD::StoreRetvalV2:
155  case NVPTXISD::StoreRetvalV4:
156    ResNode = SelectStoreRetval(N);
157    break;
158  case NVPTXISD::StoreParam:
159  case NVPTXISD::StoreParamV2:
160  case NVPTXISD::StoreParamV4:
161  case NVPTXISD::StoreParamS32:
162  case NVPTXISD::StoreParamU32:
163    ResNode = SelectStoreParam(N);
164    break;
165  default:
166    break;
167  }
168  if (ResNode)
169    return ResNode;
170  return SelectCode(N);
171}
172
173static unsigned int getCodeAddrSpace(MemSDNode *N,
174                                     const NVPTXSubtarget &Subtarget) {
175  const Value *Src = N->getSrcValue();
176
177  if (!Src)
178    return NVPTX::PTXLdStInstCode::GENERIC;
179
180  if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
181    switch (PT->getAddressSpace()) {
182    case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
183    case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
184    case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
185    case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
186    case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
187    case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
188    default: break;
189    }
190  }
191  return NVPTX::PTXLdStInstCode::GENERIC;
192}
193
194SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
195  SDLoc dl(N);
196  LoadSDNode *LD = cast<LoadSDNode>(N);
197  EVT LoadedVT = LD->getMemoryVT();
198  SDNode *NVPTXLD = NULL;
199
200  // do not support pre/post inc/dec
201  if (LD->isIndexed())
202    return NULL;
203
204  if (!LoadedVT.isSimple())
205    return NULL;
206
207  // Address Space Setting
208  unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget);
209
210  // Volatile Setting
211  // - .volatile is only availalble for .global and .shared
212  bool isVolatile = LD->isVolatile();
213  if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
214      codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
215      codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
216    isVolatile = false;
217
218  // Vector Setting
219  MVT SimpleVT = LoadedVT.getSimpleVT();
220  unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
221  if (SimpleVT.isVector()) {
222    unsigned num = SimpleVT.getVectorNumElements();
223    if (num == 2)
224      vecType = NVPTX::PTXLdStInstCode::V2;
225    else if (num == 4)
226      vecType = NVPTX::PTXLdStInstCode::V4;
227    else
228      return NULL;
229  }
230
231  // Type Setting: fromType + fromTypeWidth
232  //
233  // Sign   : ISD::SEXTLOAD
234  // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
235  //          type is integer
236  // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
237  MVT ScalarVT = SimpleVT.getScalarType();
238  // Read at least 8 bits (predicates are stored as 8-bit values)
239  unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
240  unsigned int fromType;
241  if ((LD->getExtensionType() == ISD::SEXTLOAD))
242    fromType = NVPTX::PTXLdStInstCode::Signed;
243  else if (ScalarVT.isFloatingPoint())
244    fromType = NVPTX::PTXLdStInstCode::Float;
245  else
246    fromType = NVPTX::PTXLdStInstCode::Unsigned;
247
248  // Create the machine instruction DAG
249  SDValue Chain = N->getOperand(0);
250  SDValue N1 = N->getOperand(1);
251  SDValue Addr;
252  SDValue Offset, Base;
253  unsigned Opcode;
254  MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
255
256  if (SelectDirectAddr(N1, Addr)) {
257    switch (TargetVT) {
258    case MVT::i8:
259      Opcode = NVPTX::LD_i8_avar;
260      break;
261    case MVT::i16:
262      Opcode = NVPTX::LD_i16_avar;
263      break;
264    case MVT::i32:
265      Opcode = NVPTX::LD_i32_avar;
266      break;
267    case MVT::i64:
268      Opcode = NVPTX::LD_i64_avar;
269      break;
270    case MVT::f32:
271      Opcode = NVPTX::LD_f32_avar;
272      break;
273    case MVT::f64:
274      Opcode = NVPTX::LD_f64_avar;
275      break;
276    default:
277      return NULL;
278    }
279    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
280                      getI32Imm(vecType), getI32Imm(fromType),
281                      getI32Imm(fromTypeWidth), Addr, Chain };
282    NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
283  } else if (Subtarget.is64Bit()
284                 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
285                 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
286    switch (TargetVT) {
287    case MVT::i8:
288      Opcode = NVPTX::LD_i8_asi;
289      break;
290    case MVT::i16:
291      Opcode = NVPTX::LD_i16_asi;
292      break;
293    case MVT::i32:
294      Opcode = NVPTX::LD_i32_asi;
295      break;
296    case MVT::i64:
297      Opcode = NVPTX::LD_i64_asi;
298      break;
299    case MVT::f32:
300      Opcode = NVPTX::LD_f32_asi;
301      break;
302    case MVT::f64:
303      Opcode = NVPTX::LD_f64_asi;
304      break;
305    default:
306      return NULL;
307    }
308    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
309                      getI32Imm(vecType), getI32Imm(fromType),
310                      getI32Imm(fromTypeWidth), Base, Offset, Chain };
311    NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
312  } else if (Subtarget.is64Bit()
313                 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
314                 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
315    if (Subtarget.is64Bit()) {
316      switch (TargetVT) {
317      case MVT::i8:
318        Opcode = NVPTX::LD_i8_ari_64;
319        break;
320      case MVT::i16:
321        Opcode = NVPTX::LD_i16_ari_64;
322        break;
323      case MVT::i32:
324        Opcode = NVPTX::LD_i32_ari_64;
325        break;
326      case MVT::i64:
327        Opcode = NVPTX::LD_i64_ari_64;
328        break;
329      case MVT::f32:
330        Opcode = NVPTX::LD_f32_ari_64;
331        break;
332      case MVT::f64:
333        Opcode = NVPTX::LD_f64_ari_64;
334        break;
335      default:
336        return NULL;
337      }
338    } else {
339      switch (TargetVT) {
340      case MVT::i8:
341        Opcode = NVPTX::LD_i8_ari;
342        break;
343      case MVT::i16:
344        Opcode = NVPTX::LD_i16_ari;
345        break;
346      case MVT::i32:
347        Opcode = NVPTX::LD_i32_ari;
348        break;
349      case MVT::i64:
350        Opcode = NVPTX::LD_i64_ari;
351        break;
352      case MVT::f32:
353        Opcode = NVPTX::LD_f32_ari;
354        break;
355      case MVT::f64:
356        Opcode = NVPTX::LD_f64_ari;
357        break;
358      default:
359        return NULL;
360      }
361    }
362    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
363                      getI32Imm(vecType), getI32Imm(fromType),
364                      getI32Imm(fromTypeWidth), Base, Offset, Chain };
365    NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
366  } else {
367    if (Subtarget.is64Bit()) {
368      switch (TargetVT) {
369      case MVT::i8:
370        Opcode = NVPTX::LD_i8_areg_64;
371        break;
372      case MVT::i16:
373        Opcode = NVPTX::LD_i16_areg_64;
374        break;
375      case MVT::i32:
376        Opcode = NVPTX::LD_i32_areg_64;
377        break;
378      case MVT::i64:
379        Opcode = NVPTX::LD_i64_areg_64;
380        break;
381      case MVT::f32:
382        Opcode = NVPTX::LD_f32_areg_64;
383        break;
384      case MVT::f64:
385        Opcode = NVPTX::LD_f64_areg_64;
386        break;
387      default:
388        return NULL;
389      }
390    } else {
391      switch (TargetVT) {
392      case MVT::i8:
393        Opcode = NVPTX::LD_i8_areg;
394        break;
395      case MVT::i16:
396        Opcode = NVPTX::LD_i16_areg;
397        break;
398      case MVT::i32:
399        Opcode = NVPTX::LD_i32_areg;
400        break;
401      case MVT::i64:
402        Opcode = NVPTX::LD_i64_areg;
403        break;
404      case MVT::f32:
405        Opcode = NVPTX::LD_f32_areg;
406        break;
407      case MVT::f64:
408        Opcode = NVPTX::LD_f64_areg;
409        break;
410      default:
411        return NULL;
412      }
413    }
414    SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
415                      getI32Imm(vecType), getI32Imm(fromType),
416                      getI32Imm(fromTypeWidth), N1, Chain };
417    NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
418  }
419
420  if (NVPTXLD != NULL) {
421    MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
422    MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
423    cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
424  }
425
426  return NVPTXLD;
427}
428
429SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
430
431  SDValue Chain = N->getOperand(0);
432  SDValue Op1 = N->getOperand(1);
433  SDValue Addr, Offset, Base;
434  unsigned Opcode;
435  SDLoc DL(N);
436  SDNode *LD;
437  MemSDNode *MemSD = cast<MemSDNode>(N);
438  EVT LoadedVT = MemSD->getMemoryVT();
439
440  if (!LoadedVT.isSimple())
441    return NULL;
442
443  // Address Space Setting
444  unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
445
446  // Volatile Setting
447  // - .volatile is only availalble for .global and .shared
448  bool IsVolatile = MemSD->isVolatile();
449  if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
450      CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
451      CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
452    IsVolatile = false;
453
454  // Vector Setting
455  MVT SimpleVT = LoadedVT.getSimpleVT();
456
457  // Type Setting: fromType + fromTypeWidth
458  //
459  // Sign   : ISD::SEXTLOAD
460  // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
461  //          type is integer
462  // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
463  MVT ScalarVT = SimpleVT.getScalarType();
464  // Read at least 8 bits (predicates are stored as 8-bit values)
465  unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
466  unsigned int FromType;
467  // The last operand holds the original LoadSDNode::getExtensionType() value
468  unsigned ExtensionType = cast<ConstantSDNode>(
469      N->getOperand(N->getNumOperands() - 1))->getZExtValue();
470  if (ExtensionType == ISD::SEXTLOAD)
471    FromType = NVPTX::PTXLdStInstCode::Signed;
472  else if (ScalarVT.isFloatingPoint())
473    FromType = NVPTX::PTXLdStInstCode::Float;
474  else
475    FromType = NVPTX::PTXLdStInstCode::Unsigned;
476
477  unsigned VecType;
478
479  switch (N->getOpcode()) {
480  case NVPTXISD::LoadV2:
481    VecType = NVPTX::PTXLdStInstCode::V2;
482    break;
483  case NVPTXISD::LoadV4:
484    VecType = NVPTX::PTXLdStInstCode::V4;
485    break;
486  default:
487    return NULL;
488  }
489
490  EVT EltVT = N->getValueType(0);
491
492  if (SelectDirectAddr(Op1, Addr)) {
493    switch (N->getOpcode()) {
494    default:
495      return NULL;
496    case NVPTXISD::LoadV2:
497      switch (EltVT.getSimpleVT().SimpleTy) {
498      default:
499        return NULL;
500      case MVT::i8:
501        Opcode = NVPTX::LDV_i8_v2_avar;
502        break;
503      case MVT::i16:
504        Opcode = NVPTX::LDV_i16_v2_avar;
505        break;
506      case MVT::i32:
507        Opcode = NVPTX::LDV_i32_v2_avar;
508        break;
509      case MVT::i64:
510        Opcode = NVPTX::LDV_i64_v2_avar;
511        break;
512      case MVT::f32:
513        Opcode = NVPTX::LDV_f32_v2_avar;
514        break;
515      case MVT::f64:
516        Opcode = NVPTX::LDV_f64_v2_avar;
517        break;
518      }
519      break;
520    case NVPTXISD::LoadV4:
521      switch (EltVT.getSimpleVT().SimpleTy) {
522      default:
523        return NULL;
524      case MVT::i8:
525        Opcode = NVPTX::LDV_i8_v4_avar;
526        break;
527      case MVT::i16:
528        Opcode = NVPTX::LDV_i16_v4_avar;
529        break;
530      case MVT::i32:
531        Opcode = NVPTX::LDV_i32_v4_avar;
532        break;
533      case MVT::f32:
534        Opcode = NVPTX::LDV_f32_v4_avar;
535        break;
536      }
537      break;
538    }
539
540    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
541                      getI32Imm(VecType), getI32Imm(FromType),
542                      getI32Imm(FromTypeWidth), Addr, Chain };
543    LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
544  } else if (Subtarget.is64Bit()
545                 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
546                 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
547    switch (N->getOpcode()) {
548    default:
549      return NULL;
550    case NVPTXISD::LoadV2:
551      switch (EltVT.getSimpleVT().SimpleTy) {
552      default:
553        return NULL;
554      case MVT::i8:
555        Opcode = NVPTX::LDV_i8_v2_asi;
556        break;
557      case MVT::i16:
558        Opcode = NVPTX::LDV_i16_v2_asi;
559        break;
560      case MVT::i32:
561        Opcode = NVPTX::LDV_i32_v2_asi;
562        break;
563      case MVT::i64:
564        Opcode = NVPTX::LDV_i64_v2_asi;
565        break;
566      case MVT::f32:
567        Opcode = NVPTX::LDV_f32_v2_asi;
568        break;
569      case MVT::f64:
570        Opcode = NVPTX::LDV_f64_v2_asi;
571        break;
572      }
573      break;
574    case NVPTXISD::LoadV4:
575      switch (EltVT.getSimpleVT().SimpleTy) {
576      default:
577        return NULL;
578      case MVT::i8:
579        Opcode = NVPTX::LDV_i8_v4_asi;
580        break;
581      case MVT::i16:
582        Opcode = NVPTX::LDV_i16_v4_asi;
583        break;
584      case MVT::i32:
585        Opcode = NVPTX::LDV_i32_v4_asi;
586        break;
587      case MVT::f32:
588        Opcode = NVPTX::LDV_f32_v4_asi;
589        break;
590      }
591      break;
592    }
593
594    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
595                      getI32Imm(VecType), getI32Imm(FromType),
596                      getI32Imm(FromTypeWidth), Base, Offset, Chain };
597    LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
598  } else if (Subtarget.is64Bit()
599                 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
600                 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
601    if (Subtarget.is64Bit()) {
602      switch (N->getOpcode()) {
603      default:
604        return NULL;
605      case NVPTXISD::LoadV2:
606        switch (EltVT.getSimpleVT().SimpleTy) {
607        default:
608          return NULL;
609        case MVT::i8:
610          Opcode = NVPTX::LDV_i8_v2_ari_64;
611          break;
612        case MVT::i16:
613          Opcode = NVPTX::LDV_i16_v2_ari_64;
614          break;
615        case MVT::i32:
616          Opcode = NVPTX::LDV_i32_v2_ari_64;
617          break;
618        case MVT::i64:
619          Opcode = NVPTX::LDV_i64_v2_ari_64;
620          break;
621        case MVT::f32:
622          Opcode = NVPTX::LDV_f32_v2_ari_64;
623          break;
624        case MVT::f64:
625          Opcode = NVPTX::LDV_f64_v2_ari_64;
626          break;
627        }
628        break;
629      case NVPTXISD::LoadV4:
630        switch (EltVT.getSimpleVT().SimpleTy) {
631        default:
632          return NULL;
633        case MVT::i8:
634          Opcode = NVPTX::LDV_i8_v4_ari_64;
635          break;
636        case MVT::i16:
637          Opcode = NVPTX::LDV_i16_v4_ari_64;
638          break;
639        case MVT::i32:
640          Opcode = NVPTX::LDV_i32_v4_ari_64;
641          break;
642        case MVT::f32:
643          Opcode = NVPTX::LDV_f32_v4_ari_64;
644          break;
645        }
646        break;
647      }
648    } else {
649      switch (N->getOpcode()) {
650      default:
651        return NULL;
652      case NVPTXISD::LoadV2:
653        switch (EltVT.getSimpleVT().SimpleTy) {
654        default:
655          return NULL;
656        case MVT::i8:
657          Opcode = NVPTX::LDV_i8_v2_ari;
658          break;
659        case MVT::i16:
660          Opcode = NVPTX::LDV_i16_v2_ari;
661          break;
662        case MVT::i32:
663          Opcode = NVPTX::LDV_i32_v2_ari;
664          break;
665        case MVT::i64:
666          Opcode = NVPTX::LDV_i64_v2_ari;
667          break;
668        case MVT::f32:
669          Opcode = NVPTX::LDV_f32_v2_ari;
670          break;
671        case MVT::f64:
672          Opcode = NVPTX::LDV_f64_v2_ari;
673          break;
674        }
675        break;
676      case NVPTXISD::LoadV4:
677        switch (EltVT.getSimpleVT().SimpleTy) {
678        default:
679          return NULL;
680        case MVT::i8:
681          Opcode = NVPTX::LDV_i8_v4_ari;
682          break;
683        case MVT::i16:
684          Opcode = NVPTX::LDV_i16_v4_ari;
685          break;
686        case MVT::i32:
687          Opcode = NVPTX::LDV_i32_v4_ari;
688          break;
689        case MVT::f32:
690          Opcode = NVPTX::LDV_f32_v4_ari;
691          break;
692        }
693        break;
694      }
695    }
696
697    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
698                      getI32Imm(VecType), getI32Imm(FromType),
699                      getI32Imm(FromTypeWidth), Base, Offset, Chain };
700
701    LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
702  } else {
703    if (Subtarget.is64Bit()) {
704      switch (N->getOpcode()) {
705      default:
706        return NULL;
707      case NVPTXISD::LoadV2:
708        switch (EltVT.getSimpleVT().SimpleTy) {
709        default:
710          return NULL;
711        case MVT::i8:
712          Opcode = NVPTX::LDV_i8_v2_areg_64;
713          break;
714        case MVT::i16:
715          Opcode = NVPTX::LDV_i16_v2_areg_64;
716          break;
717        case MVT::i32:
718          Opcode = NVPTX::LDV_i32_v2_areg_64;
719          break;
720        case MVT::i64:
721          Opcode = NVPTX::LDV_i64_v2_areg_64;
722          break;
723        case MVT::f32:
724          Opcode = NVPTX::LDV_f32_v2_areg_64;
725          break;
726        case MVT::f64:
727          Opcode = NVPTX::LDV_f64_v2_areg_64;
728          break;
729        }
730        break;
731      case NVPTXISD::LoadV4:
732        switch (EltVT.getSimpleVT().SimpleTy) {
733        default:
734          return NULL;
735        case MVT::i8:
736          Opcode = NVPTX::LDV_i8_v4_areg_64;
737          break;
738        case MVT::i16:
739          Opcode = NVPTX::LDV_i16_v4_areg_64;
740          break;
741        case MVT::i32:
742          Opcode = NVPTX::LDV_i32_v4_areg_64;
743          break;
744        case MVT::f32:
745          Opcode = NVPTX::LDV_f32_v4_areg_64;
746          break;
747        }
748        break;
749      }
750    } else {
751      switch (N->getOpcode()) {
752      default:
753        return NULL;
754      case NVPTXISD::LoadV2:
755        switch (EltVT.getSimpleVT().SimpleTy) {
756        default:
757          return NULL;
758        case MVT::i8:
759          Opcode = NVPTX::LDV_i8_v2_areg;
760          break;
761        case MVT::i16:
762          Opcode = NVPTX::LDV_i16_v2_areg;
763          break;
764        case MVT::i32:
765          Opcode = NVPTX::LDV_i32_v2_areg;
766          break;
767        case MVT::i64:
768          Opcode = NVPTX::LDV_i64_v2_areg;
769          break;
770        case MVT::f32:
771          Opcode = NVPTX::LDV_f32_v2_areg;
772          break;
773        case MVT::f64:
774          Opcode = NVPTX::LDV_f64_v2_areg;
775          break;
776        }
777        break;
778      case NVPTXISD::LoadV4:
779        switch (EltVT.getSimpleVT().SimpleTy) {
780        default:
781          return NULL;
782        case MVT::i8:
783          Opcode = NVPTX::LDV_i8_v4_areg;
784          break;
785        case MVT::i16:
786          Opcode = NVPTX::LDV_i16_v4_areg;
787          break;
788        case MVT::i32:
789          Opcode = NVPTX::LDV_i32_v4_areg;
790          break;
791        case MVT::f32:
792          Opcode = NVPTX::LDV_f32_v4_areg;
793          break;
794        }
795        break;
796      }
797    }
798
799    SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
800                      getI32Imm(VecType), getI32Imm(FromType),
801                      getI32Imm(FromTypeWidth), Op1, Chain };
802    LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
803  }
804
805  MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
806  MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
807  cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
808
809  return LD;
810}
811
812SDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) {
813
814  SDValue Chain = N->getOperand(0);
815  SDValue Op1 = N->getOperand(1);
816  unsigned Opcode;
817  SDLoc DL(N);
818  SDNode *LD;
819  MemSDNode *Mem = cast<MemSDNode>(N);
820  SDValue Base, Offset, Addr;
821
822  EVT EltVT = Mem->getMemoryVT().getVectorElementType();
823
824  if (SelectDirectAddr(Op1, Addr)) {
825    switch (N->getOpcode()) {
826    default:
827      return NULL;
828    case NVPTXISD::LDGV2:
829      switch (EltVT.getSimpleVT().SimpleTy) {
830      default:
831        return NULL;
832      case MVT::i8:
833        Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
834        break;
835      case MVT::i16:
836        Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
837        break;
838      case MVT::i32:
839        Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
840        break;
841      case MVT::i64:
842        Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
843        break;
844      case MVT::f32:
845        Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
846        break;
847      case MVT::f64:
848        Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
849        break;
850      }
851      break;
852    case NVPTXISD::LDUV2:
853      switch (EltVT.getSimpleVT().SimpleTy) {
854      default:
855        return NULL;
856      case MVT::i8:
857        Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
858        break;
859      case MVT::i16:
860        Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
861        break;
862      case MVT::i32:
863        Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
864        break;
865      case MVT::i64:
866        Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
867        break;
868      case MVT::f32:
869        Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
870        break;
871      case MVT::f64:
872        Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
873        break;
874      }
875      break;
876    case NVPTXISD::LDGV4:
877      switch (EltVT.getSimpleVT().SimpleTy) {
878      default:
879        return NULL;
880      case MVT::i8:
881        Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
882        break;
883      case MVT::i16:
884        Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
885        break;
886      case MVT::i32:
887        Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
888        break;
889      case MVT::f32:
890        Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
891        break;
892      }
893      break;
894    case NVPTXISD::LDUV4:
895      switch (EltVT.getSimpleVT().SimpleTy) {
896      default:
897        return NULL;
898      case MVT::i8:
899        Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
900        break;
901      case MVT::i16:
902        Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
903        break;
904      case MVT::i32:
905        Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
906        break;
907      case MVT::f32:
908        Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
909        break;
910      }
911      break;
912    }
913
914    SDValue Ops[] = { Addr, Chain };
915    LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
916                                ArrayRef<SDValue>(Ops, 2));
917  } else if (Subtarget.is64Bit()
918                 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
919                 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
920    if (Subtarget.is64Bit()) {
921      switch (N->getOpcode()) {
922      default:
923        return NULL;
924      case NVPTXISD::LDGV2:
925        switch (EltVT.getSimpleVT().SimpleTy) {
926        default:
927          return NULL;
928        case MVT::i8:
929          Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
930          break;
931        case MVT::i16:
932          Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
933          break;
934        case MVT::i32:
935          Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
936          break;
937        case MVT::i64:
938          Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
939          break;
940        case MVT::f32:
941          Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
942          break;
943        case MVT::f64:
944          Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
945          break;
946        }
947        break;
948      case NVPTXISD::LDUV2:
949        switch (EltVT.getSimpleVT().SimpleTy) {
950        default:
951          return NULL;
952        case MVT::i8:
953          Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
954          break;
955        case MVT::i16:
956          Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
957          break;
958        case MVT::i32:
959          Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
960          break;
961        case MVT::i64:
962          Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
963          break;
964        case MVT::f32:
965          Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
966          break;
967        case MVT::f64:
968          Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
969          break;
970        }
971        break;
972      case NVPTXISD::LDGV4:
973        switch (EltVT.getSimpleVT().SimpleTy) {
974        default:
975          return NULL;
976        case MVT::i8:
977          Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
978          break;
979        case MVT::i16:
980          Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
981          break;
982        case MVT::i32:
983          Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
984          break;
985        case MVT::f32:
986          Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
987          break;
988        }
989        break;
990      case NVPTXISD::LDUV4:
991        switch (EltVT.getSimpleVT().SimpleTy) {
992        default:
993          return NULL;
994        case MVT::i8:
995          Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
996          break;
997        case MVT::i16:
998          Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
999          break;
1000        case MVT::i32:
1001          Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1002          break;
1003        case MVT::f32:
1004          Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1005          break;
1006        }
1007        break;
1008      }
1009    } else {
1010      switch (N->getOpcode()) {
1011      default:
1012        return NULL;
1013      case NVPTXISD::LDGV2:
1014        switch (EltVT.getSimpleVT().SimpleTy) {
1015        default:
1016          return NULL;
1017        case MVT::i8:
1018          Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1019          break;
1020        case MVT::i16:
1021          Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1022          break;
1023        case MVT::i32:
1024          Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1025          break;
1026        case MVT::i64:
1027          Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1028          break;
1029        case MVT::f32:
1030          Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1031          break;
1032        case MVT::f64:
1033          Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1034          break;
1035        }
1036        break;
1037      case NVPTXISD::LDUV2:
1038        switch (EltVT.getSimpleVT().SimpleTy) {
1039        default:
1040          return NULL;
1041        case MVT::i8:
1042          Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1043          break;
1044        case MVT::i16:
1045          Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1046          break;
1047        case MVT::i32:
1048          Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1049          break;
1050        case MVT::i64:
1051          Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1052          break;
1053        case MVT::f32:
1054          Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1055          break;
1056        case MVT::f64:
1057          Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1058          break;
1059        }
1060        break;
1061      case NVPTXISD::LDGV4:
1062        switch (EltVT.getSimpleVT().SimpleTy) {
1063        default:
1064          return NULL;
1065        case MVT::i8:
1066          Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1067          break;
1068        case MVT::i16:
1069          Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1070          break;
1071        case MVT::i32:
1072          Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1073          break;
1074        case MVT::f32:
1075          Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1076          break;
1077        }
1078        break;
1079      case NVPTXISD::LDUV4:
1080        switch (EltVT.getSimpleVT().SimpleTy) {
1081        default:
1082          return NULL;
1083        case MVT::i8:
1084          Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1085          break;
1086        case MVT::i16:
1087          Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1088          break;
1089        case MVT::i32:
1090          Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1091          break;
1092        case MVT::f32:
1093          Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1094          break;
1095        }
1096        break;
1097      }
1098    }
1099
1100    SDValue Ops[] = { Base, Offset, Chain };
1101
1102    LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
1103                                ArrayRef<SDValue>(Ops, 3));
1104  } else {
1105    if (Subtarget.is64Bit()) {
1106      switch (N->getOpcode()) {
1107      default:
1108        return NULL;
1109      case NVPTXISD::LDGV2:
1110        switch (EltVT.getSimpleVT().SimpleTy) {
1111        default:
1112          return NULL;
1113        case MVT::i8:
1114          Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1115          break;
1116        case MVT::i16:
1117          Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1118          break;
1119        case MVT::i32:
1120          Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1121          break;
1122        case MVT::i64:
1123          Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1124          break;
1125        case MVT::f32:
1126          Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1127          break;
1128        case MVT::f64:
1129          Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1130          break;
1131        }
1132        break;
1133      case NVPTXISD::LDUV2:
1134        switch (EltVT.getSimpleVT().SimpleTy) {
1135        default:
1136          return NULL;
1137        case MVT::i8:
1138          Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1139          break;
1140        case MVT::i16:
1141          Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1142          break;
1143        case MVT::i32:
1144          Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1145          break;
1146        case MVT::i64:
1147          Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1148          break;
1149        case MVT::f32:
1150          Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1151          break;
1152        case MVT::f64:
1153          Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1154          break;
1155        }
1156        break;
1157      case NVPTXISD::LDGV4:
1158        switch (EltVT.getSimpleVT().SimpleTy) {
1159        default:
1160          return NULL;
1161        case MVT::i8:
1162          Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1163          break;
1164        case MVT::i16:
1165          Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1166          break;
1167        case MVT::i32:
1168          Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1169          break;
1170        case MVT::f32:
1171          Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1172          break;
1173        }
1174        break;
1175      case NVPTXISD::LDUV4:
1176        switch (EltVT.getSimpleVT().SimpleTy) {
1177        default:
1178          return NULL;
1179        case MVT::i8:
1180          Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1181          break;
1182        case MVT::i16:
1183          Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1184          break;
1185        case MVT::i32:
1186          Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1187          break;
1188        case MVT::f32:
1189          Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1190          break;
1191        }
1192        break;
1193      }
1194    } else {
1195      switch (N->getOpcode()) {
1196      default:
1197        return NULL;
1198      case NVPTXISD::LDGV2:
1199        switch (EltVT.getSimpleVT().SimpleTy) {
1200        default:
1201          return NULL;
1202        case MVT::i8:
1203          Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1204          break;
1205        case MVT::i16:
1206          Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1207          break;
1208        case MVT::i32:
1209          Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1210          break;
1211        case MVT::i64:
1212          Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1213          break;
1214        case MVT::f32:
1215          Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1216          break;
1217        case MVT::f64:
1218          Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1219          break;
1220        }
1221        break;
1222      case NVPTXISD::LDUV2:
1223        switch (EltVT.getSimpleVT().SimpleTy) {
1224        default:
1225          return NULL;
1226        case MVT::i8:
1227          Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1228          break;
1229        case MVT::i16:
1230          Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1231          break;
1232        case MVT::i32:
1233          Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1234          break;
1235        case MVT::i64:
1236          Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1237          break;
1238        case MVT::f32:
1239          Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1240          break;
1241        case MVT::f64:
1242          Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1243          break;
1244        }
1245        break;
1246      case NVPTXISD::LDGV4:
1247        switch (EltVT.getSimpleVT().SimpleTy) {
1248        default:
1249          return NULL;
1250        case MVT::i8:
1251          Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1252          break;
1253        case MVT::i16:
1254          Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1255          break;
1256        case MVT::i32:
1257          Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1258          break;
1259        case MVT::f32:
1260          Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1261          break;
1262        }
1263        break;
1264      case NVPTXISD::LDUV4:
1265        switch (EltVT.getSimpleVT().SimpleTy) {
1266        default:
1267          return NULL;
1268        case MVT::i8:
1269          Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1270          break;
1271        case MVT::i16:
1272          Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1273          break;
1274        case MVT::i32:
1275          Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1276          break;
1277        case MVT::f32:
1278          Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1279          break;
1280        }
1281        break;
1282      }
1283    }
1284
1285    SDValue Ops[] = { Op1, Chain };
1286    LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
1287                                ArrayRef<SDValue>(Ops, 2));
1288  }
1289
1290  MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1291  MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1292  cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1293
1294  return LD;
1295}
1296
1297SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
1298  SDLoc dl(N);
1299  StoreSDNode *ST = cast<StoreSDNode>(N);
1300  EVT StoreVT = ST->getMemoryVT();
1301  SDNode *NVPTXST = NULL;
1302
1303  // do not support pre/post inc/dec
1304  if (ST->isIndexed())
1305    return NULL;
1306
1307  if (!StoreVT.isSimple())
1308    return NULL;
1309
1310  // Address Space Setting
1311  unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget);
1312
1313  // Volatile Setting
1314  // - .volatile is only availalble for .global and .shared
1315  bool isVolatile = ST->isVolatile();
1316  if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
1317      codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
1318      codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
1319    isVolatile = false;
1320
1321  // Vector Setting
1322  MVT SimpleVT = StoreVT.getSimpleVT();
1323  unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
1324  if (SimpleVT.isVector()) {
1325    unsigned num = SimpleVT.getVectorNumElements();
1326    if (num == 2)
1327      vecType = NVPTX::PTXLdStInstCode::V2;
1328    else if (num == 4)
1329      vecType = NVPTX::PTXLdStInstCode::V4;
1330    else
1331      return NULL;
1332  }
1333
1334  // Type Setting: toType + toTypeWidth
1335  // - for integer type, always use 'u'
1336  //
1337  MVT ScalarVT = SimpleVT.getScalarType();
1338  unsigned toTypeWidth = ScalarVT.getSizeInBits();
1339  unsigned int toType;
1340  if (ScalarVT.isFloatingPoint())
1341    toType = NVPTX::PTXLdStInstCode::Float;
1342  else
1343    toType = NVPTX::PTXLdStInstCode::Unsigned;
1344
1345  // Create the machine instruction DAG
1346  SDValue Chain = N->getOperand(0);
1347  SDValue N1 = N->getOperand(1);
1348  SDValue N2 = N->getOperand(2);
1349  SDValue Addr;
1350  SDValue Offset, Base;
1351  unsigned Opcode;
1352  MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
1353
1354  if (SelectDirectAddr(N2, Addr)) {
1355    switch (SourceVT) {
1356    case MVT::i8:
1357      Opcode = NVPTX::ST_i8_avar;
1358      break;
1359    case MVT::i16:
1360      Opcode = NVPTX::ST_i16_avar;
1361      break;
1362    case MVT::i32:
1363      Opcode = NVPTX::ST_i32_avar;
1364      break;
1365    case MVT::i64:
1366      Opcode = NVPTX::ST_i64_avar;
1367      break;
1368    case MVT::f32:
1369      Opcode = NVPTX::ST_f32_avar;
1370      break;
1371    case MVT::f64:
1372      Opcode = NVPTX::ST_f64_avar;
1373      break;
1374    default:
1375      return NULL;
1376    }
1377    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1378                      getI32Imm(vecType), getI32Imm(toType),
1379                      getI32Imm(toTypeWidth), Addr, Chain };
1380    NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1381  } else if (Subtarget.is64Bit()
1382                 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
1383                 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
1384    switch (SourceVT) {
1385    case MVT::i8:
1386      Opcode = NVPTX::ST_i8_asi;
1387      break;
1388    case MVT::i16:
1389      Opcode = NVPTX::ST_i16_asi;
1390      break;
1391    case MVT::i32:
1392      Opcode = NVPTX::ST_i32_asi;
1393      break;
1394    case MVT::i64:
1395      Opcode = NVPTX::ST_i64_asi;
1396      break;
1397    case MVT::f32:
1398      Opcode = NVPTX::ST_f32_asi;
1399      break;
1400    case MVT::f64:
1401      Opcode = NVPTX::ST_f64_asi;
1402      break;
1403    default:
1404      return NULL;
1405    }
1406    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1407                      getI32Imm(vecType), getI32Imm(toType),
1408                      getI32Imm(toTypeWidth), Base, Offset, Chain };
1409    NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1410  } else if (Subtarget.is64Bit()
1411                 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
1412                 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
1413    if (Subtarget.is64Bit()) {
1414      switch (SourceVT) {
1415      case MVT::i8:
1416        Opcode = NVPTX::ST_i8_ari_64;
1417        break;
1418      case MVT::i16:
1419        Opcode = NVPTX::ST_i16_ari_64;
1420        break;
1421      case MVT::i32:
1422        Opcode = NVPTX::ST_i32_ari_64;
1423        break;
1424      case MVT::i64:
1425        Opcode = NVPTX::ST_i64_ari_64;
1426        break;
1427      case MVT::f32:
1428        Opcode = NVPTX::ST_f32_ari_64;
1429        break;
1430      case MVT::f64:
1431        Opcode = NVPTX::ST_f64_ari_64;
1432        break;
1433      default:
1434        return NULL;
1435      }
1436    } else {
1437      switch (SourceVT) {
1438      case MVT::i8:
1439        Opcode = NVPTX::ST_i8_ari;
1440        break;
1441      case MVT::i16:
1442        Opcode = NVPTX::ST_i16_ari;
1443        break;
1444      case MVT::i32:
1445        Opcode = NVPTX::ST_i32_ari;
1446        break;
1447      case MVT::i64:
1448        Opcode = NVPTX::ST_i64_ari;
1449        break;
1450      case MVT::f32:
1451        Opcode = NVPTX::ST_f32_ari;
1452        break;
1453      case MVT::f64:
1454        Opcode = NVPTX::ST_f64_ari;
1455        break;
1456      default:
1457        return NULL;
1458      }
1459    }
1460    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1461                      getI32Imm(vecType), getI32Imm(toType),
1462                      getI32Imm(toTypeWidth), Base, Offset, Chain };
1463    NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1464  } else {
1465    if (Subtarget.is64Bit()) {
1466      switch (SourceVT) {
1467      case MVT::i8:
1468        Opcode = NVPTX::ST_i8_areg_64;
1469        break;
1470      case MVT::i16:
1471        Opcode = NVPTX::ST_i16_areg_64;
1472        break;
1473      case MVT::i32:
1474        Opcode = NVPTX::ST_i32_areg_64;
1475        break;
1476      case MVT::i64:
1477        Opcode = NVPTX::ST_i64_areg_64;
1478        break;
1479      case MVT::f32:
1480        Opcode = NVPTX::ST_f32_areg_64;
1481        break;
1482      case MVT::f64:
1483        Opcode = NVPTX::ST_f64_areg_64;
1484        break;
1485      default:
1486        return NULL;
1487      }
1488    } else {
1489      switch (SourceVT) {
1490      case MVT::i8:
1491        Opcode = NVPTX::ST_i8_areg;
1492        break;
1493      case MVT::i16:
1494        Opcode = NVPTX::ST_i16_areg;
1495        break;
1496      case MVT::i32:
1497        Opcode = NVPTX::ST_i32_areg;
1498        break;
1499      case MVT::i64:
1500        Opcode = NVPTX::ST_i64_areg;
1501        break;
1502      case MVT::f32:
1503        Opcode = NVPTX::ST_f32_areg;
1504        break;
1505      case MVT::f64:
1506        Opcode = NVPTX::ST_f64_areg;
1507        break;
1508      default:
1509        return NULL;
1510      }
1511    }
1512    SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1513                      getI32Imm(vecType), getI32Imm(toType),
1514                      getI32Imm(toTypeWidth), N2, Chain };
1515    NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1516  }
1517
1518  if (NVPTXST != NULL) {
1519    MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1520    MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1521    cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
1522  }
1523
1524  return NVPTXST;
1525}
1526
1527SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
1528  SDValue Chain = N->getOperand(0);
1529  SDValue Op1 = N->getOperand(1);
1530  SDValue Addr, Offset, Base;
1531  unsigned Opcode;
1532  SDLoc DL(N);
1533  SDNode *ST;
1534  EVT EltVT = Op1.getValueType();
1535  MemSDNode *MemSD = cast<MemSDNode>(N);
1536  EVT StoreVT = MemSD->getMemoryVT();
1537
1538  // Address Space Setting
1539  unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
1540
1541  if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
1542    report_fatal_error("Cannot store to pointer that points to constant "
1543                       "memory space");
1544  }
1545
1546  // Volatile Setting
1547  // - .volatile is only availalble for .global and .shared
1548  bool IsVolatile = MemSD->isVolatile();
1549  if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
1550      CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
1551      CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
1552    IsVolatile = false;
1553
1554  // Type Setting: toType + toTypeWidth
1555  // - for integer type, always use 'u'
1556  assert(StoreVT.isSimple() && "Store value is not simple");
1557  MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
1558  unsigned ToTypeWidth = ScalarVT.getSizeInBits();
1559  unsigned ToType;
1560  if (ScalarVT.isFloatingPoint())
1561    ToType = NVPTX::PTXLdStInstCode::Float;
1562  else
1563    ToType = NVPTX::PTXLdStInstCode::Unsigned;
1564
1565  SmallVector<SDValue, 12> StOps;
1566  SDValue N2;
1567  unsigned VecType;
1568
1569  switch (N->getOpcode()) {
1570  case NVPTXISD::StoreV2:
1571    VecType = NVPTX::PTXLdStInstCode::V2;
1572    StOps.push_back(N->getOperand(1));
1573    StOps.push_back(N->getOperand(2));
1574    N2 = N->getOperand(3);
1575    break;
1576  case NVPTXISD::StoreV4:
1577    VecType = NVPTX::PTXLdStInstCode::V4;
1578    StOps.push_back(N->getOperand(1));
1579    StOps.push_back(N->getOperand(2));
1580    StOps.push_back(N->getOperand(3));
1581    StOps.push_back(N->getOperand(4));
1582    N2 = N->getOperand(5);
1583    break;
1584  default:
1585    return NULL;
1586  }
1587
1588  StOps.push_back(getI32Imm(IsVolatile));
1589  StOps.push_back(getI32Imm(CodeAddrSpace));
1590  StOps.push_back(getI32Imm(VecType));
1591  StOps.push_back(getI32Imm(ToType));
1592  StOps.push_back(getI32Imm(ToTypeWidth));
1593
1594  if (SelectDirectAddr(N2, Addr)) {
1595    switch (N->getOpcode()) {
1596    default:
1597      return NULL;
1598    case NVPTXISD::StoreV2:
1599      switch (EltVT.getSimpleVT().SimpleTy) {
1600      default:
1601        return NULL;
1602      case MVT::i8:
1603        Opcode = NVPTX::STV_i8_v2_avar;
1604        break;
1605      case MVT::i16:
1606        Opcode = NVPTX::STV_i16_v2_avar;
1607        break;
1608      case MVT::i32:
1609        Opcode = NVPTX::STV_i32_v2_avar;
1610        break;
1611      case MVT::i64:
1612        Opcode = NVPTX::STV_i64_v2_avar;
1613        break;
1614      case MVT::f32:
1615        Opcode = NVPTX::STV_f32_v2_avar;
1616        break;
1617      case MVT::f64:
1618        Opcode = NVPTX::STV_f64_v2_avar;
1619        break;
1620      }
1621      break;
1622    case NVPTXISD::StoreV4:
1623      switch (EltVT.getSimpleVT().SimpleTy) {
1624      default:
1625        return NULL;
1626      case MVT::i8:
1627        Opcode = NVPTX::STV_i8_v4_avar;
1628        break;
1629      case MVT::i16:
1630        Opcode = NVPTX::STV_i16_v4_avar;
1631        break;
1632      case MVT::i32:
1633        Opcode = NVPTX::STV_i32_v4_avar;
1634        break;
1635      case MVT::f32:
1636        Opcode = NVPTX::STV_f32_v4_avar;
1637        break;
1638      }
1639      break;
1640    }
1641    StOps.push_back(Addr);
1642  } else if (Subtarget.is64Bit()
1643                 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
1644                 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
1645    switch (N->getOpcode()) {
1646    default:
1647      return NULL;
1648    case NVPTXISD::StoreV2:
1649      switch (EltVT.getSimpleVT().SimpleTy) {
1650      default:
1651        return NULL;
1652      case MVT::i8:
1653        Opcode = NVPTX::STV_i8_v2_asi;
1654        break;
1655      case MVT::i16:
1656        Opcode = NVPTX::STV_i16_v2_asi;
1657        break;
1658      case MVT::i32:
1659        Opcode = NVPTX::STV_i32_v2_asi;
1660        break;
1661      case MVT::i64:
1662        Opcode = NVPTX::STV_i64_v2_asi;
1663        break;
1664      case MVT::f32:
1665        Opcode = NVPTX::STV_f32_v2_asi;
1666        break;
1667      case MVT::f64:
1668        Opcode = NVPTX::STV_f64_v2_asi;
1669        break;
1670      }
1671      break;
1672    case NVPTXISD::StoreV4:
1673      switch (EltVT.getSimpleVT().SimpleTy) {
1674      default:
1675        return NULL;
1676      case MVT::i8:
1677        Opcode = NVPTX::STV_i8_v4_asi;
1678        break;
1679      case MVT::i16:
1680        Opcode = NVPTX::STV_i16_v4_asi;
1681        break;
1682      case MVT::i32:
1683        Opcode = NVPTX::STV_i32_v4_asi;
1684        break;
1685      case MVT::f32:
1686        Opcode = NVPTX::STV_f32_v4_asi;
1687        break;
1688      }
1689      break;
1690    }
1691    StOps.push_back(Base);
1692    StOps.push_back(Offset);
1693  } else if (Subtarget.is64Bit()
1694                 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
1695                 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
1696    if (Subtarget.is64Bit()) {
1697      switch (N->getOpcode()) {
1698      default:
1699        return NULL;
1700      case NVPTXISD::StoreV2:
1701        switch (EltVT.getSimpleVT().SimpleTy) {
1702        default:
1703          return NULL;
1704        case MVT::i8:
1705          Opcode = NVPTX::STV_i8_v2_ari_64;
1706          break;
1707        case MVT::i16:
1708          Opcode = NVPTX::STV_i16_v2_ari_64;
1709          break;
1710        case MVT::i32:
1711          Opcode = NVPTX::STV_i32_v2_ari_64;
1712          break;
1713        case MVT::i64:
1714          Opcode = NVPTX::STV_i64_v2_ari_64;
1715          break;
1716        case MVT::f32:
1717          Opcode = NVPTX::STV_f32_v2_ari_64;
1718          break;
1719        case MVT::f64:
1720          Opcode = NVPTX::STV_f64_v2_ari_64;
1721          break;
1722        }
1723        break;
1724      case NVPTXISD::StoreV4:
1725        switch (EltVT.getSimpleVT().SimpleTy) {
1726        default:
1727          return NULL;
1728        case MVT::i8:
1729          Opcode = NVPTX::STV_i8_v4_ari_64;
1730          break;
1731        case MVT::i16:
1732          Opcode = NVPTX::STV_i16_v4_ari_64;
1733          break;
1734        case MVT::i32:
1735          Opcode = NVPTX::STV_i32_v4_ari_64;
1736          break;
1737        case MVT::f32:
1738          Opcode = NVPTX::STV_f32_v4_ari_64;
1739          break;
1740        }
1741        break;
1742      }
1743    } else {
1744      switch (N->getOpcode()) {
1745      default:
1746        return NULL;
1747      case NVPTXISD::StoreV2:
1748        switch (EltVT.getSimpleVT().SimpleTy) {
1749        default:
1750          return NULL;
1751        case MVT::i8:
1752          Opcode = NVPTX::STV_i8_v2_ari;
1753          break;
1754        case MVT::i16:
1755          Opcode = NVPTX::STV_i16_v2_ari;
1756          break;
1757        case MVT::i32:
1758          Opcode = NVPTX::STV_i32_v2_ari;
1759          break;
1760        case MVT::i64:
1761          Opcode = NVPTX::STV_i64_v2_ari;
1762          break;
1763        case MVT::f32:
1764          Opcode = NVPTX::STV_f32_v2_ari;
1765          break;
1766        case MVT::f64:
1767          Opcode = NVPTX::STV_f64_v2_ari;
1768          break;
1769        }
1770        break;
1771      case NVPTXISD::StoreV4:
1772        switch (EltVT.getSimpleVT().SimpleTy) {
1773        default:
1774          return NULL;
1775        case MVT::i8:
1776          Opcode = NVPTX::STV_i8_v4_ari;
1777          break;
1778        case MVT::i16:
1779          Opcode = NVPTX::STV_i16_v4_ari;
1780          break;
1781        case MVT::i32:
1782          Opcode = NVPTX::STV_i32_v4_ari;
1783          break;
1784        case MVT::f32:
1785          Opcode = NVPTX::STV_f32_v4_ari;
1786          break;
1787        }
1788        break;
1789      }
1790    }
1791    StOps.push_back(Base);
1792    StOps.push_back(Offset);
1793  } else {
1794    if (Subtarget.is64Bit()) {
1795      switch (N->getOpcode()) {
1796      default:
1797        return NULL;
1798      case NVPTXISD::StoreV2:
1799        switch (EltVT.getSimpleVT().SimpleTy) {
1800        default:
1801          return NULL;
1802        case MVT::i8:
1803          Opcode = NVPTX::STV_i8_v2_areg_64;
1804          break;
1805        case MVT::i16:
1806          Opcode = NVPTX::STV_i16_v2_areg_64;
1807          break;
1808        case MVT::i32:
1809          Opcode = NVPTX::STV_i32_v2_areg_64;
1810          break;
1811        case MVT::i64:
1812          Opcode = NVPTX::STV_i64_v2_areg_64;
1813          break;
1814        case MVT::f32:
1815          Opcode = NVPTX::STV_f32_v2_areg_64;
1816          break;
1817        case MVT::f64:
1818          Opcode = NVPTX::STV_f64_v2_areg_64;
1819          break;
1820        }
1821        break;
1822      case NVPTXISD::StoreV4:
1823        switch (EltVT.getSimpleVT().SimpleTy) {
1824        default:
1825          return NULL;
1826        case MVT::i8:
1827          Opcode = NVPTX::STV_i8_v4_areg_64;
1828          break;
1829        case MVT::i16:
1830          Opcode = NVPTX::STV_i16_v4_areg_64;
1831          break;
1832        case MVT::i32:
1833          Opcode = NVPTX::STV_i32_v4_areg_64;
1834          break;
1835        case MVT::f32:
1836          Opcode = NVPTX::STV_f32_v4_areg_64;
1837          break;
1838        }
1839        break;
1840      }
1841    } else {
1842      switch (N->getOpcode()) {
1843      default:
1844        return NULL;
1845      case NVPTXISD::StoreV2:
1846        switch (EltVT.getSimpleVT().SimpleTy) {
1847        default:
1848          return NULL;
1849        case MVT::i8:
1850          Opcode = NVPTX::STV_i8_v2_areg;
1851          break;
1852        case MVT::i16:
1853          Opcode = NVPTX::STV_i16_v2_areg;
1854          break;
1855        case MVT::i32:
1856          Opcode = NVPTX::STV_i32_v2_areg;
1857          break;
1858        case MVT::i64:
1859          Opcode = NVPTX::STV_i64_v2_areg;
1860          break;
1861        case MVT::f32:
1862          Opcode = NVPTX::STV_f32_v2_areg;
1863          break;
1864        case MVT::f64:
1865          Opcode = NVPTX::STV_f64_v2_areg;
1866          break;
1867        }
1868        break;
1869      case NVPTXISD::StoreV4:
1870        switch (EltVT.getSimpleVT().SimpleTy) {
1871        default:
1872          return NULL;
1873        case MVT::i8:
1874          Opcode = NVPTX::STV_i8_v4_areg;
1875          break;
1876        case MVT::i16:
1877          Opcode = NVPTX::STV_i16_v4_areg;
1878          break;
1879        case MVT::i32:
1880          Opcode = NVPTX::STV_i32_v4_areg;
1881          break;
1882        case MVT::f32:
1883          Opcode = NVPTX::STV_f32_v4_areg;
1884          break;
1885        }
1886        break;
1887      }
1888    }
1889    StOps.push_back(N2);
1890  }
1891
1892  StOps.push_back(Chain);
1893
1894  ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
1895
1896  MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1897  MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1898  cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
1899
1900  return ST;
1901}
1902
1903SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
1904  SDValue Chain = Node->getOperand(0);
1905  SDValue Offset = Node->getOperand(2);
1906  SDValue Flag = Node->getOperand(3);
1907  SDLoc DL(Node);
1908  MemSDNode *Mem = cast<MemSDNode>(Node);
1909
1910  unsigned VecSize;
1911  switch (Node->getOpcode()) {
1912  default:
1913    return NULL;
1914  case NVPTXISD::LoadParam:
1915    VecSize = 1;
1916    break;
1917  case NVPTXISD::LoadParamV2:
1918    VecSize = 2;
1919    break;
1920  case NVPTXISD::LoadParamV4:
1921    VecSize = 4;
1922    break;
1923  }
1924
1925  EVT EltVT = Node->getValueType(0);
1926  EVT MemVT = Mem->getMemoryVT();
1927
1928  unsigned Opc = 0;
1929
1930  switch (VecSize) {
1931  default:
1932    return NULL;
1933  case 1:
1934    switch (MemVT.getSimpleVT().SimpleTy) {
1935    default:
1936      return NULL;
1937    case MVT::i1:
1938      Opc = NVPTX::LoadParamMemI8;
1939      break;
1940    case MVT::i8:
1941      Opc = NVPTX::LoadParamMemI8;
1942      break;
1943    case MVT::i16:
1944      Opc = NVPTX::LoadParamMemI16;
1945      break;
1946    case MVT::i32:
1947      Opc = NVPTX::LoadParamMemI32;
1948      break;
1949    case MVT::i64:
1950      Opc = NVPTX::LoadParamMemI64;
1951      break;
1952    case MVT::f32:
1953      Opc = NVPTX::LoadParamMemF32;
1954      break;
1955    case MVT::f64:
1956      Opc = NVPTX::LoadParamMemF64;
1957      break;
1958    }
1959    break;
1960  case 2:
1961    switch (MemVT.getSimpleVT().SimpleTy) {
1962    default:
1963      return NULL;
1964    case MVT::i1:
1965      Opc = NVPTX::LoadParamMemV2I8;
1966      break;
1967    case MVT::i8:
1968      Opc = NVPTX::LoadParamMemV2I8;
1969      break;
1970    case MVT::i16:
1971      Opc = NVPTX::LoadParamMemV2I16;
1972      break;
1973    case MVT::i32:
1974      Opc = NVPTX::LoadParamMemV2I32;
1975      break;
1976    case MVT::i64:
1977      Opc = NVPTX::LoadParamMemV2I64;
1978      break;
1979    case MVT::f32:
1980      Opc = NVPTX::LoadParamMemV2F32;
1981      break;
1982    case MVT::f64:
1983      Opc = NVPTX::LoadParamMemV2F64;
1984      break;
1985    }
1986    break;
1987  case 4:
1988    switch (MemVT.getSimpleVT().SimpleTy) {
1989    default:
1990      return NULL;
1991    case MVT::i1:
1992      Opc = NVPTX::LoadParamMemV4I8;
1993      break;
1994    case MVT::i8:
1995      Opc = NVPTX::LoadParamMemV4I8;
1996      break;
1997    case MVT::i16:
1998      Opc = NVPTX::LoadParamMemV4I16;
1999      break;
2000    case MVT::i32:
2001      Opc = NVPTX::LoadParamMemV4I32;
2002      break;
2003    case MVT::f32:
2004      Opc = NVPTX::LoadParamMemV4F32;
2005      break;
2006    }
2007    break;
2008  }
2009
2010  SDVTList VTs;
2011  if (VecSize == 1) {
2012    VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2013  } else if (VecSize == 2) {
2014    VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2015  } else {
2016    EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
2017    VTs = CurDAG->getVTList(&EVTs[0], 5);
2018  }
2019
2020  unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2021
2022  SmallVector<SDValue, 2> Ops;
2023  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2024  Ops.push_back(Chain);
2025  Ops.push_back(Flag);
2026
2027  SDNode *Ret =
2028      CurDAG->getMachineNode(Opc, DL, VTs, Ops);
2029  return Ret;
2030}
2031
2032SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2033  SDLoc DL(N);
2034  SDValue Chain = N->getOperand(0);
2035  SDValue Offset = N->getOperand(1);
2036  unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2037  MemSDNode *Mem = cast<MemSDNode>(N);
2038
2039  // How many elements do we have?
2040  unsigned NumElts = 1;
2041  switch (N->getOpcode()) {
2042  default:
2043    return NULL;
2044  case NVPTXISD::StoreRetval:
2045    NumElts = 1;
2046    break;
2047  case NVPTXISD::StoreRetvalV2:
2048    NumElts = 2;
2049    break;
2050  case NVPTXISD::StoreRetvalV4:
2051    NumElts = 4;
2052    break;
2053  }
2054
2055  // Build vector of operands
2056  SmallVector<SDValue, 6> Ops;
2057  for (unsigned i = 0; i < NumElts; ++i)
2058    Ops.push_back(N->getOperand(i + 2));
2059  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2060  Ops.push_back(Chain);
2061
2062  // Determine target opcode
2063  // If we have an i1, use an 8-bit store. The lowering code in
2064  // NVPTXISelLowering will have already emitted an upcast.
2065  unsigned Opcode = 0;
2066  switch (NumElts) {
2067  default:
2068    return NULL;
2069  case 1:
2070    switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2071    default:
2072      return NULL;
2073    case MVT::i1:
2074      Opcode = NVPTX::StoreRetvalI8;
2075      break;
2076    case MVT::i8:
2077      Opcode = NVPTX::StoreRetvalI8;
2078      break;
2079    case MVT::i16:
2080      Opcode = NVPTX::StoreRetvalI16;
2081      break;
2082    case MVT::i32:
2083      Opcode = NVPTX::StoreRetvalI32;
2084      break;
2085    case MVT::i64:
2086      Opcode = NVPTX::StoreRetvalI64;
2087      break;
2088    case MVT::f32:
2089      Opcode = NVPTX::StoreRetvalF32;
2090      break;
2091    case MVT::f64:
2092      Opcode = NVPTX::StoreRetvalF64;
2093      break;
2094    }
2095    break;
2096  case 2:
2097    switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2098    default:
2099      return NULL;
2100    case MVT::i1:
2101      Opcode = NVPTX::StoreRetvalV2I8;
2102      break;
2103    case MVT::i8:
2104      Opcode = NVPTX::StoreRetvalV2I8;
2105      break;
2106    case MVT::i16:
2107      Opcode = NVPTX::StoreRetvalV2I16;
2108      break;
2109    case MVT::i32:
2110      Opcode = NVPTX::StoreRetvalV2I32;
2111      break;
2112    case MVT::i64:
2113      Opcode = NVPTX::StoreRetvalV2I64;
2114      break;
2115    case MVT::f32:
2116      Opcode = NVPTX::StoreRetvalV2F32;
2117      break;
2118    case MVT::f64:
2119      Opcode = NVPTX::StoreRetvalV2F64;
2120      break;
2121    }
2122    break;
2123  case 4:
2124    switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2125    default:
2126      return NULL;
2127    case MVT::i1:
2128      Opcode = NVPTX::StoreRetvalV4I8;
2129      break;
2130    case MVT::i8:
2131      Opcode = NVPTX::StoreRetvalV4I8;
2132      break;
2133    case MVT::i16:
2134      Opcode = NVPTX::StoreRetvalV4I16;
2135      break;
2136    case MVT::i32:
2137      Opcode = NVPTX::StoreRetvalV4I32;
2138      break;
2139    case MVT::f32:
2140      Opcode = NVPTX::StoreRetvalV4F32;
2141      break;
2142    }
2143    break;
2144  }
2145
2146  SDNode *Ret =
2147      CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2148  MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2149  MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2150  cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2151
2152  return Ret;
2153}
2154
2155SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2156  SDLoc DL(N);
2157  SDValue Chain = N->getOperand(0);
2158  SDValue Param = N->getOperand(1);
2159  unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2160  SDValue Offset = N->getOperand(2);
2161  unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2162  MemSDNode *Mem = cast<MemSDNode>(N);
2163  SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2164
2165  // How many elements do we have?
2166  unsigned NumElts = 1;
2167  switch (N->getOpcode()) {
2168  default:
2169    return NULL;
2170  case NVPTXISD::StoreParamU32:
2171  case NVPTXISD::StoreParamS32:
2172  case NVPTXISD::StoreParam:
2173    NumElts = 1;
2174    break;
2175  case NVPTXISD::StoreParamV2:
2176    NumElts = 2;
2177    break;
2178  case NVPTXISD::StoreParamV4:
2179    NumElts = 4;
2180    break;
2181  }
2182
2183  // Build vector of operands
2184  SmallVector<SDValue, 8> Ops;
2185  for (unsigned i = 0; i < NumElts; ++i)
2186    Ops.push_back(N->getOperand(i + 3));
2187  Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2188  Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2189  Ops.push_back(Chain);
2190  Ops.push_back(Flag);
2191
2192  // Determine target opcode
2193  // If we have an i1, use an 8-bit store. The lowering code in
2194  // NVPTXISelLowering will have already emitted an upcast.
2195  unsigned Opcode = 0;
2196  switch (N->getOpcode()) {
2197  default:
2198    switch (NumElts) {
2199    default:
2200      return NULL;
2201    case 1:
2202      switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2203      default:
2204        return NULL;
2205      case MVT::i1:
2206        Opcode = NVPTX::StoreParamI8;
2207        break;
2208      case MVT::i8:
2209        Opcode = NVPTX::StoreParamI8;
2210        break;
2211      case MVT::i16:
2212        Opcode = NVPTX::StoreParamI16;
2213        break;
2214      case MVT::i32:
2215        Opcode = NVPTX::StoreParamI32;
2216        break;
2217      case MVT::i64:
2218        Opcode = NVPTX::StoreParamI64;
2219        break;
2220      case MVT::f32:
2221        Opcode = NVPTX::StoreParamF32;
2222        break;
2223      case MVT::f64:
2224        Opcode = NVPTX::StoreParamF64;
2225        break;
2226      }
2227      break;
2228    case 2:
2229      switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2230      default:
2231        return NULL;
2232      case MVT::i1:
2233        Opcode = NVPTX::StoreParamV2I8;
2234        break;
2235      case MVT::i8:
2236        Opcode = NVPTX::StoreParamV2I8;
2237        break;
2238      case MVT::i16:
2239        Opcode = NVPTX::StoreParamV2I16;
2240        break;
2241      case MVT::i32:
2242        Opcode = NVPTX::StoreParamV2I32;
2243        break;
2244      case MVT::i64:
2245        Opcode = NVPTX::StoreParamV2I64;
2246        break;
2247      case MVT::f32:
2248        Opcode = NVPTX::StoreParamV2F32;
2249        break;
2250      case MVT::f64:
2251        Opcode = NVPTX::StoreParamV2F64;
2252        break;
2253      }
2254      break;
2255    case 4:
2256      switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2257      default:
2258        return NULL;
2259      case MVT::i1:
2260        Opcode = NVPTX::StoreParamV4I8;
2261        break;
2262      case MVT::i8:
2263        Opcode = NVPTX::StoreParamV4I8;
2264        break;
2265      case MVT::i16:
2266        Opcode = NVPTX::StoreParamV4I16;
2267        break;
2268      case MVT::i32:
2269        Opcode = NVPTX::StoreParamV4I32;
2270        break;
2271      case MVT::f32:
2272        Opcode = NVPTX::StoreParamV4F32;
2273        break;
2274      }
2275      break;
2276    }
2277    break;
2278  // Special case: if we have a sign-extend/zero-extend node, insert the
2279  // conversion instruction first, and use that as the value operand to
2280  // the selected StoreParam node.
2281  case NVPTXISD::StoreParamU32: {
2282    Opcode = NVPTX::StoreParamI32;
2283    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2284                                                MVT::i32);
2285    SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2286                                         MVT::i32, Ops[0], CvtNone);
2287    Ops[0] = SDValue(Cvt, 0);
2288    break;
2289  }
2290  case NVPTXISD::StoreParamS32: {
2291    Opcode = NVPTX::StoreParamI32;
2292    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2293                                                MVT::i32);
2294    SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
2295                                         MVT::i32, Ops[0], CvtNone);
2296    Ops[0] = SDValue(Cvt, 0);
2297    break;
2298  }
2299  }
2300
2301  SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
2302  SDNode *Ret =
2303      CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
2304  MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2305  MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2306  cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2307
2308  return Ret;
2309}
2310
2311// SelectDirectAddr - Match a direct address for DAG.
2312// A direct address could be a globaladdress or externalsymbol.
2313bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
2314  // Return true if TGA or ES.
2315  if (N.getOpcode() == ISD::TargetGlobalAddress ||
2316      N.getOpcode() == ISD::TargetExternalSymbol) {
2317    Address = N;
2318    return true;
2319  }
2320  if (N.getOpcode() == NVPTXISD::Wrapper) {
2321    Address = N.getOperand(0);
2322    return true;
2323  }
2324  if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
2325    unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
2326    if (IID == Intrinsic::nvvm_ptr_gen_to_param)
2327      if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
2328        return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
2329  }
2330  return false;
2331}
2332
2333// symbol+offset
2334bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
2335    SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
2336  if (Addr.getOpcode() == ISD::ADD) {
2337    if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
2338      SDValue base = Addr.getOperand(0);
2339      if (SelectDirectAddr(base, Base)) {
2340        Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
2341        return true;
2342      }
2343    }
2344  }
2345  return false;
2346}
2347
2348// symbol+offset
2349bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
2350                                     SDValue &Base, SDValue &Offset) {
2351  return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
2352}
2353
2354// symbol+offset
2355bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
2356                                       SDValue &Base, SDValue &Offset) {
2357  return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
2358}
2359
2360// register+offset
2361bool NVPTXDAGToDAGISel::SelectADDRri_imp(
2362    SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
2363  if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
2364    Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
2365    Offset = CurDAG->getTargetConstant(0, mvt);
2366    return true;
2367  }
2368  if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
2369      Addr.getOpcode() == ISD::TargetGlobalAddress)
2370    return false; // direct calls.
2371
2372  if (Addr.getOpcode() == ISD::ADD) {
2373    if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
2374      return false;
2375    }
2376    if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
2377      if (FrameIndexSDNode *FIN =
2378              dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
2379        // Constant offset from frame ref.
2380        Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
2381      else
2382        Base = Addr.getOperand(0);
2383      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
2384      return true;
2385    }
2386  }
2387  return false;
2388}
2389
2390// register+offset
2391bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
2392                                     SDValue &Base, SDValue &Offset) {
2393  return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
2394}
2395
2396// register+offset
2397bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
2398                                       SDValue &Base, SDValue &Offset) {
2399  return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
2400}
2401
2402bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
2403                                                 unsigned int spN) const {
2404  const Value *Src = NULL;
2405  // Even though MemIntrinsicSDNode is a subclas of MemSDNode,
2406  // the classof() for MemSDNode does not include MemIntrinsicSDNode
2407  // (See SelectionDAGNodes.h). So we need to check for both.
2408  if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
2409    Src = mN->getSrcValue();
2410  } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) {
2411    Src = mN->getSrcValue();
2412  }
2413  if (!Src)
2414    return false;
2415  if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
2416    return (PT->getAddressSpace() == spN);
2417  return false;
2418}
2419
2420/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
2421/// inline asm expressions.
2422bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
2423    const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
2424  SDValue Op0, Op1;
2425  switch (ConstraintCode) {
2426  default:
2427    return true;
2428  case 'm': // memory
2429    if (SelectDirectAddr(Op, Op0)) {
2430      OutOps.push_back(Op0);
2431      OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
2432      return false;
2433    }
2434    if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
2435      OutOps.push_back(Op0);
2436      OutOps.push_back(Op1);
2437      return false;
2438    }
2439    break;
2440  }
2441  return true;
2442}
2443
2444// Return true if N is a undef or a constant.
2445// If N was undef, return a (i8imm 0) in Retval
2446// If N was imm, convert it to i8imm and return in Retval
2447// Note: The convert to i8imm is required, otherwise the
2448// pattern matcher inserts a bunch of IMOVi8rr to convert
2449// the imm to i8imm, and this causes instruction selection
2450// to fail.
2451bool NVPTXDAGToDAGISel::UndefOrImm(SDValue Op, SDValue N, SDValue &Retval) {
2452  if (!(N.getOpcode() == ISD::UNDEF) && !(N.getOpcode() == ISD::Constant))
2453    return false;
2454
2455  if (N.getOpcode() == ISD::UNDEF)
2456    Retval = CurDAG->getTargetConstant(0, MVT::i8);
2457  else {
2458    ConstantSDNode *cn = cast<ConstantSDNode>(N.getNode());
2459    unsigned retval = cn->getZExtValue();
2460    Retval = CurDAG->getTargetConstant(retval, MVT::i8);
2461  }
2462  return true;
2463}
2464