NVPTXISelDAGToDAG.cpp revision dce4a407a24b04eebc6a376f8e62b41aaa7b071f
1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines an instruction selector for the NVPTX target. 11// 12//===----------------------------------------------------------------------===// 13 14#include "NVPTXISelDAGToDAG.h" 15#include "llvm/IR/GlobalValue.h" 16#include "llvm/IR/Instructions.h" 17#include "llvm/Support/CommandLine.h" 18#include "llvm/Support/Debug.h" 19#include "llvm/Support/ErrorHandling.h" 20#include "llvm/Support/raw_ostream.h" 21#include "llvm/Target/TargetIntrinsicInfo.h" 22 23using namespace llvm; 24 25#define DEBUG_TYPE "nvptx-isel" 26 27static cl::opt<int> 28FMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden, 29 cl::desc("NVPTX Specific: FMA contraction (0: don't do it" 30 " 1: do it 2: do it aggressively"), 31 cl::init(2)); 32 33static cl::opt<int> UsePrecDivF32( 34 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, 35 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 36 " IEEE Compliant F32 div.rnd if avaiable."), 37 cl::init(2)); 38 39static cl::opt<bool> 40UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, 41 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 42 cl::init(true)); 43 44static cl::opt<bool> 45FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden, 46 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), 47 cl::init(false)); 48 49 50/// createNVPTXISelDag - This pass converts a legalized DAG into a 51/// NVPTX-specific DAG, ready for instruction scheduling. 52FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, 53 llvm::CodeGenOpt::Level OptLevel) { 54 return new NVPTXDAGToDAGISel(TM, OptLevel); 55} 56 57NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 58 CodeGenOpt::Level OptLevel) 59 : SelectionDAGISel(tm, OptLevel), 60 Subtarget(tm.getSubtarget<NVPTXSubtarget>()) { 61 62 doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1); 63 doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1); 64 doFMAF32AGG = 65 (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2); 66 doFMAF64AGG = 67 (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2); 68 69 allowFMA = (FMAContractLevel >= 1); 70 71 doMulWide = (OptLevel > 0); 72} 73 74int NVPTXDAGToDAGISel::getDivF32Level() const { 75 if (UsePrecDivF32.getNumOccurrences() > 0) { 76 // If nvptx-prec-div32=N is used on the command-line, always honor it 77 return UsePrecDivF32; 78 } else { 79 // Otherwise, use div.approx if fast math is enabled 80 if (TM.Options.UnsafeFPMath) 81 return 0; 82 else 83 return 2; 84 } 85} 86 87bool NVPTXDAGToDAGISel::usePrecSqrtF32() const { 88 if (UsePrecSqrtF32.getNumOccurrences() > 0) { 89 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it 90 return UsePrecSqrtF32; 91 } else { 92 // Otherwise, use sqrt.approx if fast math is enabled 93 if (TM.Options.UnsafeFPMath) 94 return false; 95 else 96 return true; 97 } 98} 99 100bool NVPTXDAGToDAGISel::useF32FTZ() const { 101 if (FtzEnabled.getNumOccurrences() > 0) { 102 // If nvptx-f32ftz is used on the command-line, always honor it 103 return FtzEnabled; 104 } else { 105 const Function *F = MF->getFunction(); 106 // Otherwise, check for an nvptx-f32ftz attribute on the function 107 if (F->hasFnAttribute("nvptx-f32ftz")) 108 return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex, 109 "nvptx-f32ftz") 110 .getValueAsString() == "true"); 111 else 112 return false; 113 } 114} 115 116/// Select - Select instructions not customized! Used for 117/// expanded, promoted and normal instructions. 118SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 119 120 if (N->isMachineOpcode()) { 121 N->setNodeId(-1); 122 return nullptr; // Already selected. 123 } 124 125 SDNode *ResNode = nullptr; 126 switch (N->getOpcode()) { 127 case ISD::LOAD: 128 ResNode = SelectLoad(N); 129 break; 130 case ISD::STORE: 131 ResNode = SelectStore(N); 132 break; 133 case NVPTXISD::LoadV2: 134 case NVPTXISD::LoadV4: 135 ResNode = SelectLoadVector(N); 136 break; 137 case NVPTXISD::LDGV2: 138 case NVPTXISD::LDGV4: 139 case NVPTXISD::LDUV2: 140 case NVPTXISD::LDUV4: 141 ResNode = SelectLDGLDUVector(N); 142 break; 143 case NVPTXISD::StoreV2: 144 case NVPTXISD::StoreV4: 145 ResNode = SelectStoreVector(N); 146 break; 147 case NVPTXISD::LoadParam: 148 case NVPTXISD::LoadParamV2: 149 case NVPTXISD::LoadParamV4: 150 ResNode = SelectLoadParam(N); 151 break; 152 case NVPTXISD::StoreRetval: 153 case NVPTXISD::StoreRetvalV2: 154 case NVPTXISD::StoreRetvalV4: 155 ResNode = SelectStoreRetval(N); 156 break; 157 case NVPTXISD::StoreParam: 158 case NVPTXISD::StoreParamV2: 159 case NVPTXISD::StoreParamV4: 160 case NVPTXISD::StoreParamS32: 161 case NVPTXISD::StoreParamU32: 162 ResNode = SelectStoreParam(N); 163 break; 164 case ISD::INTRINSIC_WO_CHAIN: 165 ResNode = SelectIntrinsicNoChain(N); 166 break; 167 case NVPTXISD::Tex1DFloatI32: 168 case NVPTXISD::Tex1DFloatFloat: 169 case NVPTXISD::Tex1DFloatFloatLevel: 170 case NVPTXISD::Tex1DFloatFloatGrad: 171 case NVPTXISD::Tex1DI32I32: 172 case NVPTXISD::Tex1DI32Float: 173 case NVPTXISD::Tex1DI32FloatLevel: 174 case NVPTXISD::Tex1DI32FloatGrad: 175 case NVPTXISD::Tex1DArrayFloatI32: 176 case NVPTXISD::Tex1DArrayFloatFloat: 177 case NVPTXISD::Tex1DArrayFloatFloatLevel: 178 case NVPTXISD::Tex1DArrayFloatFloatGrad: 179 case NVPTXISD::Tex1DArrayI32I32: 180 case NVPTXISD::Tex1DArrayI32Float: 181 case NVPTXISD::Tex1DArrayI32FloatLevel: 182 case NVPTXISD::Tex1DArrayI32FloatGrad: 183 case NVPTXISD::Tex2DFloatI32: 184 case NVPTXISD::Tex2DFloatFloat: 185 case NVPTXISD::Tex2DFloatFloatLevel: 186 case NVPTXISD::Tex2DFloatFloatGrad: 187 case NVPTXISD::Tex2DI32I32: 188 case NVPTXISD::Tex2DI32Float: 189 case NVPTXISD::Tex2DI32FloatLevel: 190 case NVPTXISD::Tex2DI32FloatGrad: 191 case NVPTXISD::Tex2DArrayFloatI32: 192 case NVPTXISD::Tex2DArrayFloatFloat: 193 case NVPTXISD::Tex2DArrayFloatFloatLevel: 194 case NVPTXISD::Tex2DArrayFloatFloatGrad: 195 case NVPTXISD::Tex2DArrayI32I32: 196 case NVPTXISD::Tex2DArrayI32Float: 197 case NVPTXISD::Tex2DArrayI32FloatLevel: 198 case NVPTXISD::Tex2DArrayI32FloatGrad: 199 case NVPTXISD::Tex3DFloatI32: 200 case NVPTXISD::Tex3DFloatFloat: 201 case NVPTXISD::Tex3DFloatFloatLevel: 202 case NVPTXISD::Tex3DFloatFloatGrad: 203 case NVPTXISD::Tex3DI32I32: 204 case NVPTXISD::Tex3DI32Float: 205 case NVPTXISD::Tex3DI32FloatLevel: 206 case NVPTXISD::Tex3DI32FloatGrad: 207 ResNode = SelectTextureIntrinsic(N); 208 break; 209 case NVPTXISD::Suld1DI8Trap: 210 case NVPTXISD::Suld1DI16Trap: 211 case NVPTXISD::Suld1DI32Trap: 212 case NVPTXISD::Suld1DV2I8Trap: 213 case NVPTXISD::Suld1DV2I16Trap: 214 case NVPTXISD::Suld1DV2I32Trap: 215 case NVPTXISD::Suld1DV4I8Trap: 216 case NVPTXISD::Suld1DV4I16Trap: 217 case NVPTXISD::Suld1DV4I32Trap: 218 case NVPTXISD::Suld1DArrayI8Trap: 219 case NVPTXISD::Suld1DArrayI16Trap: 220 case NVPTXISD::Suld1DArrayI32Trap: 221 case NVPTXISD::Suld1DArrayV2I8Trap: 222 case NVPTXISD::Suld1DArrayV2I16Trap: 223 case NVPTXISD::Suld1DArrayV2I32Trap: 224 case NVPTXISD::Suld1DArrayV4I8Trap: 225 case NVPTXISD::Suld1DArrayV4I16Trap: 226 case NVPTXISD::Suld1DArrayV4I32Trap: 227 case NVPTXISD::Suld2DI8Trap: 228 case NVPTXISD::Suld2DI16Trap: 229 case NVPTXISD::Suld2DI32Trap: 230 case NVPTXISD::Suld2DV2I8Trap: 231 case NVPTXISD::Suld2DV2I16Trap: 232 case NVPTXISD::Suld2DV2I32Trap: 233 case NVPTXISD::Suld2DV4I8Trap: 234 case NVPTXISD::Suld2DV4I16Trap: 235 case NVPTXISD::Suld2DV4I32Trap: 236 case NVPTXISD::Suld2DArrayI8Trap: 237 case NVPTXISD::Suld2DArrayI16Trap: 238 case NVPTXISD::Suld2DArrayI32Trap: 239 case NVPTXISD::Suld2DArrayV2I8Trap: 240 case NVPTXISD::Suld2DArrayV2I16Trap: 241 case NVPTXISD::Suld2DArrayV2I32Trap: 242 case NVPTXISD::Suld2DArrayV4I8Trap: 243 case NVPTXISD::Suld2DArrayV4I16Trap: 244 case NVPTXISD::Suld2DArrayV4I32Trap: 245 case NVPTXISD::Suld3DI8Trap: 246 case NVPTXISD::Suld3DI16Trap: 247 case NVPTXISD::Suld3DI32Trap: 248 case NVPTXISD::Suld3DV2I8Trap: 249 case NVPTXISD::Suld3DV2I16Trap: 250 case NVPTXISD::Suld3DV2I32Trap: 251 case NVPTXISD::Suld3DV4I8Trap: 252 case NVPTXISD::Suld3DV4I16Trap: 253 case NVPTXISD::Suld3DV4I32Trap: 254 ResNode = SelectSurfaceIntrinsic(N); 255 break; 256 case ISD::ADDRSPACECAST: 257 ResNode = SelectAddrSpaceCast(N); 258 break; 259 default: 260 break; 261 } 262 if (ResNode) 263 return ResNode; 264 return SelectCode(N); 265} 266 267static unsigned int getCodeAddrSpace(MemSDNode *N, 268 const NVPTXSubtarget &Subtarget) { 269 const Value *Src = N->getMemOperand()->getValue(); 270 271 if (!Src) 272 return NVPTX::PTXLdStInstCode::GENERIC; 273 274 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { 275 switch (PT->getAddressSpace()) { 276 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL; 277 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL; 278 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED; 279 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC; 280 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM; 281 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT; 282 default: break; 283 } 284 } 285 return NVPTX::PTXLdStInstCode::GENERIC; 286} 287 288SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) { 289 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 290 switch (IID) { 291 default: 292 return nullptr; 293 case Intrinsic::nvvm_texsurf_handle_internal: 294 return SelectTexSurfHandle(N); 295 } 296} 297 298SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) { 299 // Op 0 is the intrinsic ID 300 SDValue Wrapper = N->getOperand(1); 301 SDValue GlobalVal = Wrapper.getOperand(0); 302 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64, 303 GlobalVal); 304} 305 306SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) { 307 SDValue Src = N->getOperand(0); 308 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N); 309 unsigned SrcAddrSpace = CastN->getSrcAddressSpace(); 310 unsigned DstAddrSpace = CastN->getDestAddressSpace(); 311 312 assert(SrcAddrSpace != DstAddrSpace && 313 "addrspacecast must be between different address spaces"); 314 315 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) { 316 // Specific to generic 317 unsigned Opc; 318 switch (SrcAddrSpace) { 319 default: report_fatal_error("Bad address space in addrspacecast"); 320 case ADDRESS_SPACE_GLOBAL: 321 Opc = Subtarget.is64Bit() ? NVPTX::cvta_global_yes_64 322 : NVPTX::cvta_global_yes; 323 break; 324 case ADDRESS_SPACE_SHARED: 325 Opc = Subtarget.is64Bit() ? NVPTX::cvta_shared_yes_64 326 : NVPTX::cvta_shared_yes; 327 break; 328 case ADDRESS_SPACE_CONST: 329 Opc = Subtarget.is64Bit() ? NVPTX::cvta_const_yes_64 330 : NVPTX::cvta_const_yes; 331 break; 332 case ADDRESS_SPACE_LOCAL: 333 Opc = Subtarget.is64Bit() ? NVPTX::cvta_local_yes_64 334 : NVPTX::cvta_local_yes; 335 break; 336 } 337 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); 338 } else { 339 // Generic to specific 340 if (SrcAddrSpace != 0) 341 report_fatal_error("Cannot cast between two non-generic address spaces"); 342 unsigned Opc; 343 switch (DstAddrSpace) { 344 default: report_fatal_error("Bad address space in addrspacecast"); 345 case ADDRESS_SPACE_GLOBAL: 346 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_global_yes_64 347 : NVPTX::cvta_to_global_yes; 348 break; 349 case ADDRESS_SPACE_SHARED: 350 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_shared_yes_64 351 : NVPTX::cvta_to_shared_yes; 352 break; 353 case ADDRESS_SPACE_CONST: 354 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_const_yes_64 355 : NVPTX::cvta_to_const_yes; 356 break; 357 case ADDRESS_SPACE_LOCAL: 358 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_local_yes_64 359 : NVPTX::cvta_to_local_yes; 360 break; 361 } 362 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); 363 } 364} 365 366SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 367 SDLoc dl(N); 368 LoadSDNode *LD = cast<LoadSDNode>(N); 369 EVT LoadedVT = LD->getMemoryVT(); 370 SDNode *NVPTXLD = nullptr; 371 372 // do not support pre/post inc/dec 373 if (LD->isIndexed()) 374 return nullptr; 375 376 if (!LoadedVT.isSimple()) 377 return nullptr; 378 379 // Address Space Setting 380 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget); 381 382 // Volatile Setting 383 // - .volatile is only availalble for .global and .shared 384 bool isVolatile = LD->isVolatile(); 385 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 386 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 387 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 388 isVolatile = false; 389 390 // Vector Setting 391 MVT SimpleVT = LoadedVT.getSimpleVT(); 392 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 393 if (SimpleVT.isVector()) { 394 unsigned num = SimpleVT.getVectorNumElements(); 395 if (num == 2) 396 vecType = NVPTX::PTXLdStInstCode::V2; 397 else if (num == 4) 398 vecType = NVPTX::PTXLdStInstCode::V4; 399 else 400 return nullptr; 401 } 402 403 // Type Setting: fromType + fromTypeWidth 404 // 405 // Sign : ISD::SEXTLOAD 406 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 407 // type is integer 408 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 409 MVT ScalarVT = SimpleVT.getScalarType(); 410 // Read at least 8 bits (predicates are stored as 8-bit values) 411 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 412 unsigned int fromType; 413 if ((LD->getExtensionType() == ISD::SEXTLOAD)) 414 fromType = NVPTX::PTXLdStInstCode::Signed; 415 else if (ScalarVT.isFloatingPoint()) 416 fromType = NVPTX::PTXLdStInstCode::Float; 417 else 418 fromType = NVPTX::PTXLdStInstCode::Unsigned; 419 420 // Create the machine instruction DAG 421 SDValue Chain = N->getOperand(0); 422 SDValue N1 = N->getOperand(1); 423 SDValue Addr; 424 SDValue Offset, Base; 425 unsigned Opcode; 426 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy; 427 428 if (SelectDirectAddr(N1, Addr)) { 429 switch (TargetVT) { 430 case MVT::i8: 431 Opcode = NVPTX::LD_i8_avar; 432 break; 433 case MVT::i16: 434 Opcode = NVPTX::LD_i16_avar; 435 break; 436 case MVT::i32: 437 Opcode = NVPTX::LD_i32_avar; 438 break; 439 case MVT::i64: 440 Opcode = NVPTX::LD_i64_avar; 441 break; 442 case MVT::f32: 443 Opcode = NVPTX::LD_f32_avar; 444 break; 445 case MVT::f64: 446 Opcode = NVPTX::LD_f64_avar; 447 break; 448 default: 449 return nullptr; 450 } 451 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 452 getI32Imm(vecType), getI32Imm(fromType), 453 getI32Imm(fromTypeWidth), Addr, Chain }; 454 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 455 } else if (Subtarget.is64Bit() 456 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) 457 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { 458 switch (TargetVT) { 459 case MVT::i8: 460 Opcode = NVPTX::LD_i8_asi; 461 break; 462 case MVT::i16: 463 Opcode = NVPTX::LD_i16_asi; 464 break; 465 case MVT::i32: 466 Opcode = NVPTX::LD_i32_asi; 467 break; 468 case MVT::i64: 469 Opcode = NVPTX::LD_i64_asi; 470 break; 471 case MVT::f32: 472 Opcode = NVPTX::LD_f32_asi; 473 break; 474 case MVT::f64: 475 Opcode = NVPTX::LD_f64_asi; 476 break; 477 default: 478 return nullptr; 479 } 480 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 481 getI32Imm(vecType), getI32Imm(fromType), 482 getI32Imm(fromTypeWidth), Base, Offset, Chain }; 483 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 484 } else if (Subtarget.is64Bit() 485 ? SelectADDRri64(N1.getNode(), N1, Base, Offset) 486 : SelectADDRri(N1.getNode(), N1, Base, Offset)) { 487 if (Subtarget.is64Bit()) { 488 switch (TargetVT) { 489 case MVT::i8: 490 Opcode = NVPTX::LD_i8_ari_64; 491 break; 492 case MVT::i16: 493 Opcode = NVPTX::LD_i16_ari_64; 494 break; 495 case MVT::i32: 496 Opcode = NVPTX::LD_i32_ari_64; 497 break; 498 case MVT::i64: 499 Opcode = NVPTX::LD_i64_ari_64; 500 break; 501 case MVT::f32: 502 Opcode = NVPTX::LD_f32_ari_64; 503 break; 504 case MVT::f64: 505 Opcode = NVPTX::LD_f64_ari_64; 506 break; 507 default: 508 return nullptr; 509 } 510 } else { 511 switch (TargetVT) { 512 case MVT::i8: 513 Opcode = NVPTX::LD_i8_ari; 514 break; 515 case MVT::i16: 516 Opcode = NVPTX::LD_i16_ari; 517 break; 518 case MVT::i32: 519 Opcode = NVPTX::LD_i32_ari; 520 break; 521 case MVT::i64: 522 Opcode = NVPTX::LD_i64_ari; 523 break; 524 case MVT::f32: 525 Opcode = NVPTX::LD_f32_ari; 526 break; 527 case MVT::f64: 528 Opcode = NVPTX::LD_f64_ari; 529 break; 530 default: 531 return nullptr; 532 } 533 } 534 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 535 getI32Imm(vecType), getI32Imm(fromType), 536 getI32Imm(fromTypeWidth), Base, Offset, Chain }; 537 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 538 } else { 539 if (Subtarget.is64Bit()) { 540 switch (TargetVT) { 541 case MVT::i8: 542 Opcode = NVPTX::LD_i8_areg_64; 543 break; 544 case MVT::i16: 545 Opcode = NVPTX::LD_i16_areg_64; 546 break; 547 case MVT::i32: 548 Opcode = NVPTX::LD_i32_areg_64; 549 break; 550 case MVT::i64: 551 Opcode = NVPTX::LD_i64_areg_64; 552 break; 553 case MVT::f32: 554 Opcode = NVPTX::LD_f32_areg_64; 555 break; 556 case MVT::f64: 557 Opcode = NVPTX::LD_f64_areg_64; 558 break; 559 default: 560 return nullptr; 561 } 562 } else { 563 switch (TargetVT) { 564 case MVT::i8: 565 Opcode = NVPTX::LD_i8_areg; 566 break; 567 case MVT::i16: 568 Opcode = NVPTX::LD_i16_areg; 569 break; 570 case MVT::i32: 571 Opcode = NVPTX::LD_i32_areg; 572 break; 573 case MVT::i64: 574 Opcode = NVPTX::LD_i64_areg; 575 break; 576 case MVT::f32: 577 Opcode = NVPTX::LD_f32_areg; 578 break; 579 case MVT::f64: 580 Opcode = NVPTX::LD_f64_areg; 581 break; 582 default: 583 return nullptr; 584 } 585 } 586 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 587 getI32Imm(vecType), getI32Imm(fromType), 588 getI32Imm(fromTypeWidth), N1, Chain }; 589 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 590 } 591 592 if (NVPTXLD) { 593 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 594 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 595 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 596 } 597 598 return NVPTXLD; 599} 600 601SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { 602 603 SDValue Chain = N->getOperand(0); 604 SDValue Op1 = N->getOperand(1); 605 SDValue Addr, Offset, Base; 606 unsigned Opcode; 607 SDLoc DL(N); 608 SDNode *LD; 609 MemSDNode *MemSD = cast<MemSDNode>(N); 610 EVT LoadedVT = MemSD->getMemoryVT(); 611 612 if (!LoadedVT.isSimple()) 613 return nullptr; 614 615 // Address Space Setting 616 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 617 618 // Volatile Setting 619 // - .volatile is only availalble for .global and .shared 620 bool IsVolatile = MemSD->isVolatile(); 621 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 622 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 623 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 624 IsVolatile = false; 625 626 // Vector Setting 627 MVT SimpleVT = LoadedVT.getSimpleVT(); 628 629 // Type Setting: fromType + fromTypeWidth 630 // 631 // Sign : ISD::SEXTLOAD 632 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 633 // type is integer 634 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 635 MVT ScalarVT = SimpleVT.getScalarType(); 636 // Read at least 8 bits (predicates are stored as 8-bit values) 637 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 638 unsigned int FromType; 639 // The last operand holds the original LoadSDNode::getExtensionType() value 640 unsigned ExtensionType = cast<ConstantSDNode>( 641 N->getOperand(N->getNumOperands() - 1))->getZExtValue(); 642 if (ExtensionType == ISD::SEXTLOAD) 643 FromType = NVPTX::PTXLdStInstCode::Signed; 644 else if (ScalarVT.isFloatingPoint()) 645 FromType = NVPTX::PTXLdStInstCode::Float; 646 else 647 FromType = NVPTX::PTXLdStInstCode::Unsigned; 648 649 unsigned VecType; 650 651 switch (N->getOpcode()) { 652 case NVPTXISD::LoadV2: 653 VecType = NVPTX::PTXLdStInstCode::V2; 654 break; 655 case NVPTXISD::LoadV4: 656 VecType = NVPTX::PTXLdStInstCode::V4; 657 break; 658 default: 659 return nullptr; 660 } 661 662 EVT EltVT = N->getValueType(0); 663 664 if (SelectDirectAddr(Op1, Addr)) { 665 switch (N->getOpcode()) { 666 default: 667 return nullptr; 668 case NVPTXISD::LoadV2: 669 switch (EltVT.getSimpleVT().SimpleTy) { 670 default: 671 return nullptr; 672 case MVT::i8: 673 Opcode = NVPTX::LDV_i8_v2_avar; 674 break; 675 case MVT::i16: 676 Opcode = NVPTX::LDV_i16_v2_avar; 677 break; 678 case MVT::i32: 679 Opcode = NVPTX::LDV_i32_v2_avar; 680 break; 681 case MVT::i64: 682 Opcode = NVPTX::LDV_i64_v2_avar; 683 break; 684 case MVT::f32: 685 Opcode = NVPTX::LDV_f32_v2_avar; 686 break; 687 case MVT::f64: 688 Opcode = NVPTX::LDV_f64_v2_avar; 689 break; 690 } 691 break; 692 case NVPTXISD::LoadV4: 693 switch (EltVT.getSimpleVT().SimpleTy) { 694 default: 695 return nullptr; 696 case MVT::i8: 697 Opcode = NVPTX::LDV_i8_v4_avar; 698 break; 699 case MVT::i16: 700 Opcode = NVPTX::LDV_i16_v4_avar; 701 break; 702 case MVT::i32: 703 Opcode = NVPTX::LDV_i32_v4_avar; 704 break; 705 case MVT::f32: 706 Opcode = NVPTX::LDV_f32_v4_avar; 707 break; 708 } 709 break; 710 } 711 712 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 713 getI32Imm(VecType), getI32Imm(FromType), 714 getI32Imm(FromTypeWidth), Addr, Chain }; 715 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 716 } else if (Subtarget.is64Bit() 717 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 718 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 719 switch (N->getOpcode()) { 720 default: 721 return nullptr; 722 case NVPTXISD::LoadV2: 723 switch (EltVT.getSimpleVT().SimpleTy) { 724 default: 725 return nullptr; 726 case MVT::i8: 727 Opcode = NVPTX::LDV_i8_v2_asi; 728 break; 729 case MVT::i16: 730 Opcode = NVPTX::LDV_i16_v2_asi; 731 break; 732 case MVT::i32: 733 Opcode = NVPTX::LDV_i32_v2_asi; 734 break; 735 case MVT::i64: 736 Opcode = NVPTX::LDV_i64_v2_asi; 737 break; 738 case MVT::f32: 739 Opcode = NVPTX::LDV_f32_v2_asi; 740 break; 741 case MVT::f64: 742 Opcode = NVPTX::LDV_f64_v2_asi; 743 break; 744 } 745 break; 746 case NVPTXISD::LoadV4: 747 switch (EltVT.getSimpleVT().SimpleTy) { 748 default: 749 return nullptr; 750 case MVT::i8: 751 Opcode = NVPTX::LDV_i8_v4_asi; 752 break; 753 case MVT::i16: 754 Opcode = NVPTX::LDV_i16_v4_asi; 755 break; 756 case MVT::i32: 757 Opcode = NVPTX::LDV_i32_v4_asi; 758 break; 759 case MVT::f32: 760 Opcode = NVPTX::LDV_f32_v4_asi; 761 break; 762 } 763 break; 764 } 765 766 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 767 getI32Imm(VecType), getI32Imm(FromType), 768 getI32Imm(FromTypeWidth), Base, Offset, Chain }; 769 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 770 } else if (Subtarget.is64Bit() 771 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 772 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 773 if (Subtarget.is64Bit()) { 774 switch (N->getOpcode()) { 775 default: 776 return nullptr; 777 case NVPTXISD::LoadV2: 778 switch (EltVT.getSimpleVT().SimpleTy) { 779 default: 780 return nullptr; 781 case MVT::i8: 782 Opcode = NVPTX::LDV_i8_v2_ari_64; 783 break; 784 case MVT::i16: 785 Opcode = NVPTX::LDV_i16_v2_ari_64; 786 break; 787 case MVT::i32: 788 Opcode = NVPTX::LDV_i32_v2_ari_64; 789 break; 790 case MVT::i64: 791 Opcode = NVPTX::LDV_i64_v2_ari_64; 792 break; 793 case MVT::f32: 794 Opcode = NVPTX::LDV_f32_v2_ari_64; 795 break; 796 case MVT::f64: 797 Opcode = NVPTX::LDV_f64_v2_ari_64; 798 break; 799 } 800 break; 801 case NVPTXISD::LoadV4: 802 switch (EltVT.getSimpleVT().SimpleTy) { 803 default: 804 return nullptr; 805 case MVT::i8: 806 Opcode = NVPTX::LDV_i8_v4_ari_64; 807 break; 808 case MVT::i16: 809 Opcode = NVPTX::LDV_i16_v4_ari_64; 810 break; 811 case MVT::i32: 812 Opcode = NVPTX::LDV_i32_v4_ari_64; 813 break; 814 case MVT::f32: 815 Opcode = NVPTX::LDV_f32_v4_ari_64; 816 break; 817 } 818 break; 819 } 820 } else { 821 switch (N->getOpcode()) { 822 default: 823 return nullptr; 824 case NVPTXISD::LoadV2: 825 switch (EltVT.getSimpleVT().SimpleTy) { 826 default: 827 return nullptr; 828 case MVT::i8: 829 Opcode = NVPTX::LDV_i8_v2_ari; 830 break; 831 case MVT::i16: 832 Opcode = NVPTX::LDV_i16_v2_ari; 833 break; 834 case MVT::i32: 835 Opcode = NVPTX::LDV_i32_v2_ari; 836 break; 837 case MVT::i64: 838 Opcode = NVPTX::LDV_i64_v2_ari; 839 break; 840 case MVT::f32: 841 Opcode = NVPTX::LDV_f32_v2_ari; 842 break; 843 case MVT::f64: 844 Opcode = NVPTX::LDV_f64_v2_ari; 845 break; 846 } 847 break; 848 case NVPTXISD::LoadV4: 849 switch (EltVT.getSimpleVT().SimpleTy) { 850 default: 851 return nullptr; 852 case MVT::i8: 853 Opcode = NVPTX::LDV_i8_v4_ari; 854 break; 855 case MVT::i16: 856 Opcode = NVPTX::LDV_i16_v4_ari; 857 break; 858 case MVT::i32: 859 Opcode = NVPTX::LDV_i32_v4_ari; 860 break; 861 case MVT::f32: 862 Opcode = NVPTX::LDV_f32_v4_ari; 863 break; 864 } 865 break; 866 } 867 } 868 869 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 870 getI32Imm(VecType), getI32Imm(FromType), 871 getI32Imm(FromTypeWidth), Base, Offset, Chain }; 872 873 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 874 } else { 875 if (Subtarget.is64Bit()) { 876 switch (N->getOpcode()) { 877 default: 878 return nullptr; 879 case NVPTXISD::LoadV2: 880 switch (EltVT.getSimpleVT().SimpleTy) { 881 default: 882 return nullptr; 883 case MVT::i8: 884 Opcode = NVPTX::LDV_i8_v2_areg_64; 885 break; 886 case MVT::i16: 887 Opcode = NVPTX::LDV_i16_v2_areg_64; 888 break; 889 case MVT::i32: 890 Opcode = NVPTX::LDV_i32_v2_areg_64; 891 break; 892 case MVT::i64: 893 Opcode = NVPTX::LDV_i64_v2_areg_64; 894 break; 895 case MVT::f32: 896 Opcode = NVPTX::LDV_f32_v2_areg_64; 897 break; 898 case MVT::f64: 899 Opcode = NVPTX::LDV_f64_v2_areg_64; 900 break; 901 } 902 break; 903 case NVPTXISD::LoadV4: 904 switch (EltVT.getSimpleVT().SimpleTy) { 905 default: 906 return nullptr; 907 case MVT::i8: 908 Opcode = NVPTX::LDV_i8_v4_areg_64; 909 break; 910 case MVT::i16: 911 Opcode = NVPTX::LDV_i16_v4_areg_64; 912 break; 913 case MVT::i32: 914 Opcode = NVPTX::LDV_i32_v4_areg_64; 915 break; 916 case MVT::f32: 917 Opcode = NVPTX::LDV_f32_v4_areg_64; 918 break; 919 } 920 break; 921 } 922 } else { 923 switch (N->getOpcode()) { 924 default: 925 return nullptr; 926 case NVPTXISD::LoadV2: 927 switch (EltVT.getSimpleVT().SimpleTy) { 928 default: 929 return nullptr; 930 case MVT::i8: 931 Opcode = NVPTX::LDV_i8_v2_areg; 932 break; 933 case MVT::i16: 934 Opcode = NVPTX::LDV_i16_v2_areg; 935 break; 936 case MVT::i32: 937 Opcode = NVPTX::LDV_i32_v2_areg; 938 break; 939 case MVT::i64: 940 Opcode = NVPTX::LDV_i64_v2_areg; 941 break; 942 case MVT::f32: 943 Opcode = NVPTX::LDV_f32_v2_areg; 944 break; 945 case MVT::f64: 946 Opcode = NVPTX::LDV_f64_v2_areg; 947 break; 948 } 949 break; 950 case NVPTXISD::LoadV4: 951 switch (EltVT.getSimpleVT().SimpleTy) { 952 default: 953 return nullptr; 954 case MVT::i8: 955 Opcode = NVPTX::LDV_i8_v4_areg; 956 break; 957 case MVT::i16: 958 Opcode = NVPTX::LDV_i16_v4_areg; 959 break; 960 case MVT::i32: 961 Opcode = NVPTX::LDV_i32_v4_areg; 962 break; 963 case MVT::f32: 964 Opcode = NVPTX::LDV_f32_v4_areg; 965 break; 966 } 967 break; 968 } 969 } 970 971 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 972 getI32Imm(VecType), getI32Imm(FromType), 973 getI32Imm(FromTypeWidth), Op1, Chain }; 974 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 975 } 976 977 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 978 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 979 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 980 981 return LD; 982} 983 984SDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) { 985 986 SDValue Chain = N->getOperand(0); 987 SDValue Op1 = N->getOperand(1); 988 unsigned Opcode; 989 SDLoc DL(N); 990 SDNode *LD; 991 MemSDNode *Mem = cast<MemSDNode>(N); 992 SDValue Base, Offset, Addr; 993 994 EVT EltVT = Mem->getMemoryVT().getVectorElementType(); 995 996 if (SelectDirectAddr(Op1, Addr)) { 997 switch (N->getOpcode()) { 998 default: 999 return nullptr; 1000 case NVPTXISD::LDGV2: 1001 switch (EltVT.getSimpleVT().SimpleTy) { 1002 default: 1003 return nullptr; 1004 case MVT::i8: 1005 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; 1006 break; 1007 case MVT::i16: 1008 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; 1009 break; 1010 case MVT::i32: 1011 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar; 1012 break; 1013 case MVT::i64: 1014 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar; 1015 break; 1016 case MVT::f32: 1017 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar; 1018 break; 1019 case MVT::f64: 1020 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar; 1021 break; 1022 } 1023 break; 1024 case NVPTXISD::LDUV2: 1025 switch (EltVT.getSimpleVT().SimpleTy) { 1026 default: 1027 return nullptr; 1028 case MVT::i8: 1029 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; 1030 break; 1031 case MVT::i16: 1032 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; 1033 break; 1034 case MVT::i32: 1035 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar; 1036 break; 1037 case MVT::i64: 1038 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar; 1039 break; 1040 case MVT::f32: 1041 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar; 1042 break; 1043 case MVT::f64: 1044 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar; 1045 break; 1046 } 1047 break; 1048 case NVPTXISD::LDGV4: 1049 switch (EltVT.getSimpleVT().SimpleTy) { 1050 default: 1051 return nullptr; 1052 case MVT::i8: 1053 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; 1054 break; 1055 case MVT::i16: 1056 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar; 1057 break; 1058 case MVT::i32: 1059 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar; 1060 break; 1061 case MVT::f32: 1062 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar; 1063 break; 1064 } 1065 break; 1066 case NVPTXISD::LDUV4: 1067 switch (EltVT.getSimpleVT().SimpleTy) { 1068 default: 1069 return nullptr; 1070 case MVT::i8: 1071 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; 1072 break; 1073 case MVT::i16: 1074 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar; 1075 break; 1076 case MVT::i32: 1077 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar; 1078 break; 1079 case MVT::f32: 1080 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar; 1081 break; 1082 } 1083 break; 1084 } 1085 1086 SDValue Ops[] = { Addr, Chain }; 1087 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1088 } else if (Subtarget.is64Bit() 1089 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 1090 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 1091 if (Subtarget.is64Bit()) { 1092 switch (N->getOpcode()) { 1093 default: 1094 return nullptr; 1095 case NVPTXISD::LDGV2: 1096 switch (EltVT.getSimpleVT().SimpleTy) { 1097 default: 1098 return nullptr; 1099 case MVT::i8: 1100 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; 1101 break; 1102 case MVT::i16: 1103 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; 1104 break; 1105 case MVT::i32: 1106 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64; 1107 break; 1108 case MVT::i64: 1109 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64; 1110 break; 1111 case MVT::f32: 1112 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64; 1113 break; 1114 case MVT::f64: 1115 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64; 1116 break; 1117 } 1118 break; 1119 case NVPTXISD::LDUV2: 1120 switch (EltVT.getSimpleVT().SimpleTy) { 1121 default: 1122 return nullptr; 1123 case MVT::i8: 1124 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; 1125 break; 1126 case MVT::i16: 1127 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; 1128 break; 1129 case MVT::i32: 1130 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64; 1131 break; 1132 case MVT::i64: 1133 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64; 1134 break; 1135 case MVT::f32: 1136 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64; 1137 break; 1138 case MVT::f64: 1139 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64; 1140 break; 1141 } 1142 break; 1143 case NVPTXISD::LDGV4: 1144 switch (EltVT.getSimpleVT().SimpleTy) { 1145 default: 1146 return nullptr; 1147 case MVT::i8: 1148 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; 1149 break; 1150 case MVT::i16: 1151 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64; 1152 break; 1153 case MVT::i32: 1154 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64; 1155 break; 1156 case MVT::f32: 1157 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64; 1158 break; 1159 } 1160 break; 1161 case NVPTXISD::LDUV4: 1162 switch (EltVT.getSimpleVT().SimpleTy) { 1163 default: 1164 return nullptr; 1165 case MVT::i8: 1166 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; 1167 break; 1168 case MVT::i16: 1169 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64; 1170 break; 1171 case MVT::i32: 1172 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64; 1173 break; 1174 case MVT::f32: 1175 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64; 1176 break; 1177 } 1178 break; 1179 } 1180 } else { 1181 switch (N->getOpcode()) { 1182 default: 1183 return nullptr; 1184 case NVPTXISD::LDGV2: 1185 switch (EltVT.getSimpleVT().SimpleTy) { 1186 default: 1187 return nullptr; 1188 case MVT::i8: 1189 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; 1190 break; 1191 case MVT::i16: 1192 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; 1193 break; 1194 case MVT::i32: 1195 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32; 1196 break; 1197 case MVT::i64: 1198 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32; 1199 break; 1200 case MVT::f32: 1201 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32; 1202 break; 1203 case MVT::f64: 1204 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32; 1205 break; 1206 } 1207 break; 1208 case NVPTXISD::LDUV2: 1209 switch (EltVT.getSimpleVT().SimpleTy) { 1210 default: 1211 return nullptr; 1212 case MVT::i8: 1213 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; 1214 break; 1215 case MVT::i16: 1216 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; 1217 break; 1218 case MVT::i32: 1219 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32; 1220 break; 1221 case MVT::i64: 1222 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32; 1223 break; 1224 case MVT::f32: 1225 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32; 1226 break; 1227 case MVT::f64: 1228 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32; 1229 break; 1230 } 1231 break; 1232 case NVPTXISD::LDGV4: 1233 switch (EltVT.getSimpleVT().SimpleTy) { 1234 default: 1235 return nullptr; 1236 case MVT::i8: 1237 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; 1238 break; 1239 case MVT::i16: 1240 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32; 1241 break; 1242 case MVT::i32: 1243 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32; 1244 break; 1245 case MVT::f32: 1246 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32; 1247 break; 1248 } 1249 break; 1250 case NVPTXISD::LDUV4: 1251 switch (EltVT.getSimpleVT().SimpleTy) { 1252 default: 1253 return nullptr; 1254 case MVT::i8: 1255 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; 1256 break; 1257 case MVT::i16: 1258 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32; 1259 break; 1260 case MVT::i32: 1261 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32; 1262 break; 1263 case MVT::f32: 1264 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32; 1265 break; 1266 } 1267 break; 1268 } 1269 } 1270 1271 SDValue Ops[] = { Base, Offset, Chain }; 1272 1273 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1274 } else { 1275 if (Subtarget.is64Bit()) { 1276 switch (N->getOpcode()) { 1277 default: 1278 return nullptr; 1279 case NVPTXISD::LDGV2: 1280 switch (EltVT.getSimpleVT().SimpleTy) { 1281 default: 1282 return nullptr; 1283 case MVT::i8: 1284 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; 1285 break; 1286 case MVT::i16: 1287 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; 1288 break; 1289 case MVT::i32: 1290 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64; 1291 break; 1292 case MVT::i64: 1293 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64; 1294 break; 1295 case MVT::f32: 1296 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64; 1297 break; 1298 case MVT::f64: 1299 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64; 1300 break; 1301 } 1302 break; 1303 case NVPTXISD::LDUV2: 1304 switch (EltVT.getSimpleVT().SimpleTy) { 1305 default: 1306 return nullptr; 1307 case MVT::i8: 1308 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; 1309 break; 1310 case MVT::i16: 1311 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; 1312 break; 1313 case MVT::i32: 1314 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64; 1315 break; 1316 case MVT::i64: 1317 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64; 1318 break; 1319 case MVT::f32: 1320 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64; 1321 break; 1322 case MVT::f64: 1323 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64; 1324 break; 1325 } 1326 break; 1327 case NVPTXISD::LDGV4: 1328 switch (EltVT.getSimpleVT().SimpleTy) { 1329 default: 1330 return nullptr; 1331 case MVT::i8: 1332 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; 1333 break; 1334 case MVT::i16: 1335 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64; 1336 break; 1337 case MVT::i32: 1338 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64; 1339 break; 1340 case MVT::f32: 1341 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64; 1342 break; 1343 } 1344 break; 1345 case NVPTXISD::LDUV4: 1346 switch (EltVT.getSimpleVT().SimpleTy) { 1347 default: 1348 return nullptr; 1349 case MVT::i8: 1350 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; 1351 break; 1352 case MVT::i16: 1353 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64; 1354 break; 1355 case MVT::i32: 1356 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64; 1357 break; 1358 case MVT::f32: 1359 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64; 1360 break; 1361 } 1362 break; 1363 } 1364 } else { 1365 switch (N->getOpcode()) { 1366 default: 1367 return nullptr; 1368 case NVPTXISD::LDGV2: 1369 switch (EltVT.getSimpleVT().SimpleTy) { 1370 default: 1371 return nullptr; 1372 case MVT::i8: 1373 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; 1374 break; 1375 case MVT::i16: 1376 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; 1377 break; 1378 case MVT::i32: 1379 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32; 1380 break; 1381 case MVT::i64: 1382 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32; 1383 break; 1384 case MVT::f32: 1385 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32; 1386 break; 1387 case MVT::f64: 1388 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32; 1389 break; 1390 } 1391 break; 1392 case NVPTXISD::LDUV2: 1393 switch (EltVT.getSimpleVT().SimpleTy) { 1394 default: 1395 return nullptr; 1396 case MVT::i8: 1397 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; 1398 break; 1399 case MVT::i16: 1400 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; 1401 break; 1402 case MVT::i32: 1403 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32; 1404 break; 1405 case MVT::i64: 1406 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32; 1407 break; 1408 case MVT::f32: 1409 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32; 1410 break; 1411 case MVT::f64: 1412 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32; 1413 break; 1414 } 1415 break; 1416 case NVPTXISD::LDGV4: 1417 switch (EltVT.getSimpleVT().SimpleTy) { 1418 default: 1419 return nullptr; 1420 case MVT::i8: 1421 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; 1422 break; 1423 case MVT::i16: 1424 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32; 1425 break; 1426 case MVT::i32: 1427 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32; 1428 break; 1429 case MVT::f32: 1430 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32; 1431 break; 1432 } 1433 break; 1434 case NVPTXISD::LDUV4: 1435 switch (EltVT.getSimpleVT().SimpleTy) { 1436 default: 1437 return nullptr; 1438 case MVT::i8: 1439 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; 1440 break; 1441 case MVT::i16: 1442 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32; 1443 break; 1444 case MVT::i32: 1445 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32; 1446 break; 1447 case MVT::f32: 1448 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32; 1449 break; 1450 } 1451 break; 1452 } 1453 } 1454 1455 SDValue Ops[] = { Op1, Chain }; 1456 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1457 } 1458 1459 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1460 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1461 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1462 1463 return LD; 1464} 1465 1466SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 1467 SDLoc dl(N); 1468 StoreSDNode *ST = cast<StoreSDNode>(N); 1469 EVT StoreVT = ST->getMemoryVT(); 1470 SDNode *NVPTXST = nullptr; 1471 1472 // do not support pre/post inc/dec 1473 if (ST->isIndexed()) 1474 return nullptr; 1475 1476 if (!StoreVT.isSimple()) 1477 return nullptr; 1478 1479 // Address Space Setting 1480 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget); 1481 1482 // Volatile Setting 1483 // - .volatile is only availalble for .global and .shared 1484 bool isVolatile = ST->isVolatile(); 1485 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1486 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1487 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1488 isVolatile = false; 1489 1490 // Vector Setting 1491 MVT SimpleVT = StoreVT.getSimpleVT(); 1492 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 1493 if (SimpleVT.isVector()) { 1494 unsigned num = SimpleVT.getVectorNumElements(); 1495 if (num == 2) 1496 vecType = NVPTX::PTXLdStInstCode::V2; 1497 else if (num == 4) 1498 vecType = NVPTX::PTXLdStInstCode::V4; 1499 else 1500 return nullptr; 1501 } 1502 1503 // Type Setting: toType + toTypeWidth 1504 // - for integer type, always use 'u' 1505 // 1506 MVT ScalarVT = SimpleVT.getScalarType(); 1507 unsigned toTypeWidth = ScalarVT.getSizeInBits(); 1508 unsigned int toType; 1509 if (ScalarVT.isFloatingPoint()) 1510 toType = NVPTX::PTXLdStInstCode::Float; 1511 else 1512 toType = NVPTX::PTXLdStInstCode::Unsigned; 1513 1514 // Create the machine instruction DAG 1515 SDValue Chain = N->getOperand(0); 1516 SDValue N1 = N->getOperand(1); 1517 SDValue N2 = N->getOperand(2); 1518 SDValue Addr; 1519 SDValue Offset, Base; 1520 unsigned Opcode; 1521 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy; 1522 1523 if (SelectDirectAddr(N2, Addr)) { 1524 switch (SourceVT) { 1525 case MVT::i8: 1526 Opcode = NVPTX::ST_i8_avar; 1527 break; 1528 case MVT::i16: 1529 Opcode = NVPTX::ST_i16_avar; 1530 break; 1531 case MVT::i32: 1532 Opcode = NVPTX::ST_i32_avar; 1533 break; 1534 case MVT::i64: 1535 Opcode = NVPTX::ST_i64_avar; 1536 break; 1537 case MVT::f32: 1538 Opcode = NVPTX::ST_f32_avar; 1539 break; 1540 case MVT::f64: 1541 Opcode = NVPTX::ST_f64_avar; 1542 break; 1543 default: 1544 return nullptr; 1545 } 1546 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1547 getI32Imm(vecType), getI32Imm(toType), 1548 getI32Imm(toTypeWidth), Addr, Chain }; 1549 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1550 } else if (Subtarget.is64Bit() 1551 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1552 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1553 switch (SourceVT) { 1554 case MVT::i8: 1555 Opcode = NVPTX::ST_i8_asi; 1556 break; 1557 case MVT::i16: 1558 Opcode = NVPTX::ST_i16_asi; 1559 break; 1560 case MVT::i32: 1561 Opcode = NVPTX::ST_i32_asi; 1562 break; 1563 case MVT::i64: 1564 Opcode = NVPTX::ST_i64_asi; 1565 break; 1566 case MVT::f32: 1567 Opcode = NVPTX::ST_f32_asi; 1568 break; 1569 case MVT::f64: 1570 Opcode = NVPTX::ST_f64_asi; 1571 break; 1572 default: 1573 return nullptr; 1574 } 1575 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1576 getI32Imm(vecType), getI32Imm(toType), 1577 getI32Imm(toTypeWidth), Base, Offset, Chain }; 1578 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1579 } else if (Subtarget.is64Bit() 1580 ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1581 : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1582 if (Subtarget.is64Bit()) { 1583 switch (SourceVT) { 1584 case MVT::i8: 1585 Opcode = NVPTX::ST_i8_ari_64; 1586 break; 1587 case MVT::i16: 1588 Opcode = NVPTX::ST_i16_ari_64; 1589 break; 1590 case MVT::i32: 1591 Opcode = NVPTX::ST_i32_ari_64; 1592 break; 1593 case MVT::i64: 1594 Opcode = NVPTX::ST_i64_ari_64; 1595 break; 1596 case MVT::f32: 1597 Opcode = NVPTX::ST_f32_ari_64; 1598 break; 1599 case MVT::f64: 1600 Opcode = NVPTX::ST_f64_ari_64; 1601 break; 1602 default: 1603 return nullptr; 1604 } 1605 } else { 1606 switch (SourceVT) { 1607 case MVT::i8: 1608 Opcode = NVPTX::ST_i8_ari; 1609 break; 1610 case MVT::i16: 1611 Opcode = NVPTX::ST_i16_ari; 1612 break; 1613 case MVT::i32: 1614 Opcode = NVPTX::ST_i32_ari; 1615 break; 1616 case MVT::i64: 1617 Opcode = NVPTX::ST_i64_ari; 1618 break; 1619 case MVT::f32: 1620 Opcode = NVPTX::ST_f32_ari; 1621 break; 1622 case MVT::f64: 1623 Opcode = NVPTX::ST_f64_ari; 1624 break; 1625 default: 1626 return nullptr; 1627 } 1628 } 1629 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1630 getI32Imm(vecType), getI32Imm(toType), 1631 getI32Imm(toTypeWidth), Base, Offset, Chain }; 1632 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1633 } else { 1634 if (Subtarget.is64Bit()) { 1635 switch (SourceVT) { 1636 case MVT::i8: 1637 Opcode = NVPTX::ST_i8_areg_64; 1638 break; 1639 case MVT::i16: 1640 Opcode = NVPTX::ST_i16_areg_64; 1641 break; 1642 case MVT::i32: 1643 Opcode = NVPTX::ST_i32_areg_64; 1644 break; 1645 case MVT::i64: 1646 Opcode = NVPTX::ST_i64_areg_64; 1647 break; 1648 case MVT::f32: 1649 Opcode = NVPTX::ST_f32_areg_64; 1650 break; 1651 case MVT::f64: 1652 Opcode = NVPTX::ST_f64_areg_64; 1653 break; 1654 default: 1655 return nullptr; 1656 } 1657 } else { 1658 switch (SourceVT) { 1659 case MVT::i8: 1660 Opcode = NVPTX::ST_i8_areg; 1661 break; 1662 case MVT::i16: 1663 Opcode = NVPTX::ST_i16_areg; 1664 break; 1665 case MVT::i32: 1666 Opcode = NVPTX::ST_i32_areg; 1667 break; 1668 case MVT::i64: 1669 Opcode = NVPTX::ST_i64_areg; 1670 break; 1671 case MVT::f32: 1672 Opcode = NVPTX::ST_f32_areg; 1673 break; 1674 case MVT::f64: 1675 Opcode = NVPTX::ST_f64_areg; 1676 break; 1677 default: 1678 return nullptr; 1679 } 1680 } 1681 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1682 getI32Imm(vecType), getI32Imm(toType), 1683 getI32Imm(toTypeWidth), N2, Chain }; 1684 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1685 } 1686 1687 if (NVPTXST) { 1688 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1689 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1690 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1691 } 1692 1693 return NVPTXST; 1694} 1695 1696SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { 1697 SDValue Chain = N->getOperand(0); 1698 SDValue Op1 = N->getOperand(1); 1699 SDValue Addr, Offset, Base; 1700 unsigned Opcode; 1701 SDLoc DL(N); 1702 SDNode *ST; 1703 EVT EltVT = Op1.getValueType(); 1704 MemSDNode *MemSD = cast<MemSDNode>(N); 1705 EVT StoreVT = MemSD->getMemoryVT(); 1706 1707 // Address Space Setting 1708 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 1709 1710 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { 1711 report_fatal_error("Cannot store to pointer that points to constant " 1712 "memory space"); 1713 } 1714 1715 // Volatile Setting 1716 // - .volatile is only availalble for .global and .shared 1717 bool IsVolatile = MemSD->isVolatile(); 1718 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1719 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1720 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1721 IsVolatile = false; 1722 1723 // Type Setting: toType + toTypeWidth 1724 // - for integer type, always use 'u' 1725 assert(StoreVT.isSimple() && "Store value is not simple"); 1726 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); 1727 unsigned ToTypeWidth = ScalarVT.getSizeInBits(); 1728 unsigned ToType; 1729 if (ScalarVT.isFloatingPoint()) 1730 ToType = NVPTX::PTXLdStInstCode::Float; 1731 else 1732 ToType = NVPTX::PTXLdStInstCode::Unsigned; 1733 1734 SmallVector<SDValue, 12> StOps; 1735 SDValue N2; 1736 unsigned VecType; 1737 1738 switch (N->getOpcode()) { 1739 case NVPTXISD::StoreV2: 1740 VecType = NVPTX::PTXLdStInstCode::V2; 1741 StOps.push_back(N->getOperand(1)); 1742 StOps.push_back(N->getOperand(2)); 1743 N2 = N->getOperand(3); 1744 break; 1745 case NVPTXISD::StoreV4: 1746 VecType = NVPTX::PTXLdStInstCode::V4; 1747 StOps.push_back(N->getOperand(1)); 1748 StOps.push_back(N->getOperand(2)); 1749 StOps.push_back(N->getOperand(3)); 1750 StOps.push_back(N->getOperand(4)); 1751 N2 = N->getOperand(5); 1752 break; 1753 default: 1754 return nullptr; 1755 } 1756 1757 StOps.push_back(getI32Imm(IsVolatile)); 1758 StOps.push_back(getI32Imm(CodeAddrSpace)); 1759 StOps.push_back(getI32Imm(VecType)); 1760 StOps.push_back(getI32Imm(ToType)); 1761 StOps.push_back(getI32Imm(ToTypeWidth)); 1762 1763 if (SelectDirectAddr(N2, Addr)) { 1764 switch (N->getOpcode()) { 1765 default: 1766 return nullptr; 1767 case NVPTXISD::StoreV2: 1768 switch (EltVT.getSimpleVT().SimpleTy) { 1769 default: 1770 return nullptr; 1771 case MVT::i8: 1772 Opcode = NVPTX::STV_i8_v2_avar; 1773 break; 1774 case MVT::i16: 1775 Opcode = NVPTX::STV_i16_v2_avar; 1776 break; 1777 case MVT::i32: 1778 Opcode = NVPTX::STV_i32_v2_avar; 1779 break; 1780 case MVT::i64: 1781 Opcode = NVPTX::STV_i64_v2_avar; 1782 break; 1783 case MVT::f32: 1784 Opcode = NVPTX::STV_f32_v2_avar; 1785 break; 1786 case MVT::f64: 1787 Opcode = NVPTX::STV_f64_v2_avar; 1788 break; 1789 } 1790 break; 1791 case NVPTXISD::StoreV4: 1792 switch (EltVT.getSimpleVT().SimpleTy) { 1793 default: 1794 return nullptr; 1795 case MVT::i8: 1796 Opcode = NVPTX::STV_i8_v4_avar; 1797 break; 1798 case MVT::i16: 1799 Opcode = NVPTX::STV_i16_v4_avar; 1800 break; 1801 case MVT::i32: 1802 Opcode = NVPTX::STV_i32_v4_avar; 1803 break; 1804 case MVT::f32: 1805 Opcode = NVPTX::STV_f32_v4_avar; 1806 break; 1807 } 1808 break; 1809 } 1810 StOps.push_back(Addr); 1811 } else if (Subtarget.is64Bit() 1812 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1813 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1814 switch (N->getOpcode()) { 1815 default: 1816 return nullptr; 1817 case NVPTXISD::StoreV2: 1818 switch (EltVT.getSimpleVT().SimpleTy) { 1819 default: 1820 return nullptr; 1821 case MVT::i8: 1822 Opcode = NVPTX::STV_i8_v2_asi; 1823 break; 1824 case MVT::i16: 1825 Opcode = NVPTX::STV_i16_v2_asi; 1826 break; 1827 case MVT::i32: 1828 Opcode = NVPTX::STV_i32_v2_asi; 1829 break; 1830 case MVT::i64: 1831 Opcode = NVPTX::STV_i64_v2_asi; 1832 break; 1833 case MVT::f32: 1834 Opcode = NVPTX::STV_f32_v2_asi; 1835 break; 1836 case MVT::f64: 1837 Opcode = NVPTX::STV_f64_v2_asi; 1838 break; 1839 } 1840 break; 1841 case NVPTXISD::StoreV4: 1842 switch (EltVT.getSimpleVT().SimpleTy) { 1843 default: 1844 return nullptr; 1845 case MVT::i8: 1846 Opcode = NVPTX::STV_i8_v4_asi; 1847 break; 1848 case MVT::i16: 1849 Opcode = NVPTX::STV_i16_v4_asi; 1850 break; 1851 case MVT::i32: 1852 Opcode = NVPTX::STV_i32_v4_asi; 1853 break; 1854 case MVT::f32: 1855 Opcode = NVPTX::STV_f32_v4_asi; 1856 break; 1857 } 1858 break; 1859 } 1860 StOps.push_back(Base); 1861 StOps.push_back(Offset); 1862 } else if (Subtarget.is64Bit() 1863 ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1864 : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1865 if (Subtarget.is64Bit()) { 1866 switch (N->getOpcode()) { 1867 default: 1868 return nullptr; 1869 case NVPTXISD::StoreV2: 1870 switch (EltVT.getSimpleVT().SimpleTy) { 1871 default: 1872 return nullptr; 1873 case MVT::i8: 1874 Opcode = NVPTX::STV_i8_v2_ari_64; 1875 break; 1876 case MVT::i16: 1877 Opcode = NVPTX::STV_i16_v2_ari_64; 1878 break; 1879 case MVT::i32: 1880 Opcode = NVPTX::STV_i32_v2_ari_64; 1881 break; 1882 case MVT::i64: 1883 Opcode = NVPTX::STV_i64_v2_ari_64; 1884 break; 1885 case MVT::f32: 1886 Opcode = NVPTX::STV_f32_v2_ari_64; 1887 break; 1888 case MVT::f64: 1889 Opcode = NVPTX::STV_f64_v2_ari_64; 1890 break; 1891 } 1892 break; 1893 case NVPTXISD::StoreV4: 1894 switch (EltVT.getSimpleVT().SimpleTy) { 1895 default: 1896 return nullptr; 1897 case MVT::i8: 1898 Opcode = NVPTX::STV_i8_v4_ari_64; 1899 break; 1900 case MVT::i16: 1901 Opcode = NVPTX::STV_i16_v4_ari_64; 1902 break; 1903 case MVT::i32: 1904 Opcode = NVPTX::STV_i32_v4_ari_64; 1905 break; 1906 case MVT::f32: 1907 Opcode = NVPTX::STV_f32_v4_ari_64; 1908 break; 1909 } 1910 break; 1911 } 1912 } else { 1913 switch (N->getOpcode()) { 1914 default: 1915 return nullptr; 1916 case NVPTXISD::StoreV2: 1917 switch (EltVT.getSimpleVT().SimpleTy) { 1918 default: 1919 return nullptr; 1920 case MVT::i8: 1921 Opcode = NVPTX::STV_i8_v2_ari; 1922 break; 1923 case MVT::i16: 1924 Opcode = NVPTX::STV_i16_v2_ari; 1925 break; 1926 case MVT::i32: 1927 Opcode = NVPTX::STV_i32_v2_ari; 1928 break; 1929 case MVT::i64: 1930 Opcode = NVPTX::STV_i64_v2_ari; 1931 break; 1932 case MVT::f32: 1933 Opcode = NVPTX::STV_f32_v2_ari; 1934 break; 1935 case MVT::f64: 1936 Opcode = NVPTX::STV_f64_v2_ari; 1937 break; 1938 } 1939 break; 1940 case NVPTXISD::StoreV4: 1941 switch (EltVT.getSimpleVT().SimpleTy) { 1942 default: 1943 return nullptr; 1944 case MVT::i8: 1945 Opcode = NVPTX::STV_i8_v4_ari; 1946 break; 1947 case MVT::i16: 1948 Opcode = NVPTX::STV_i16_v4_ari; 1949 break; 1950 case MVT::i32: 1951 Opcode = NVPTX::STV_i32_v4_ari; 1952 break; 1953 case MVT::f32: 1954 Opcode = NVPTX::STV_f32_v4_ari; 1955 break; 1956 } 1957 break; 1958 } 1959 } 1960 StOps.push_back(Base); 1961 StOps.push_back(Offset); 1962 } else { 1963 if (Subtarget.is64Bit()) { 1964 switch (N->getOpcode()) { 1965 default: 1966 return nullptr; 1967 case NVPTXISD::StoreV2: 1968 switch (EltVT.getSimpleVT().SimpleTy) { 1969 default: 1970 return nullptr; 1971 case MVT::i8: 1972 Opcode = NVPTX::STV_i8_v2_areg_64; 1973 break; 1974 case MVT::i16: 1975 Opcode = NVPTX::STV_i16_v2_areg_64; 1976 break; 1977 case MVT::i32: 1978 Opcode = NVPTX::STV_i32_v2_areg_64; 1979 break; 1980 case MVT::i64: 1981 Opcode = NVPTX::STV_i64_v2_areg_64; 1982 break; 1983 case MVT::f32: 1984 Opcode = NVPTX::STV_f32_v2_areg_64; 1985 break; 1986 case MVT::f64: 1987 Opcode = NVPTX::STV_f64_v2_areg_64; 1988 break; 1989 } 1990 break; 1991 case NVPTXISD::StoreV4: 1992 switch (EltVT.getSimpleVT().SimpleTy) { 1993 default: 1994 return nullptr; 1995 case MVT::i8: 1996 Opcode = NVPTX::STV_i8_v4_areg_64; 1997 break; 1998 case MVT::i16: 1999 Opcode = NVPTX::STV_i16_v4_areg_64; 2000 break; 2001 case MVT::i32: 2002 Opcode = NVPTX::STV_i32_v4_areg_64; 2003 break; 2004 case MVT::f32: 2005 Opcode = NVPTX::STV_f32_v4_areg_64; 2006 break; 2007 } 2008 break; 2009 } 2010 } else { 2011 switch (N->getOpcode()) { 2012 default: 2013 return nullptr; 2014 case NVPTXISD::StoreV2: 2015 switch (EltVT.getSimpleVT().SimpleTy) { 2016 default: 2017 return nullptr; 2018 case MVT::i8: 2019 Opcode = NVPTX::STV_i8_v2_areg; 2020 break; 2021 case MVT::i16: 2022 Opcode = NVPTX::STV_i16_v2_areg; 2023 break; 2024 case MVT::i32: 2025 Opcode = NVPTX::STV_i32_v2_areg; 2026 break; 2027 case MVT::i64: 2028 Opcode = NVPTX::STV_i64_v2_areg; 2029 break; 2030 case MVT::f32: 2031 Opcode = NVPTX::STV_f32_v2_areg; 2032 break; 2033 case MVT::f64: 2034 Opcode = NVPTX::STV_f64_v2_areg; 2035 break; 2036 } 2037 break; 2038 case NVPTXISD::StoreV4: 2039 switch (EltVT.getSimpleVT().SimpleTy) { 2040 default: 2041 return nullptr; 2042 case MVT::i8: 2043 Opcode = NVPTX::STV_i8_v4_areg; 2044 break; 2045 case MVT::i16: 2046 Opcode = NVPTX::STV_i16_v4_areg; 2047 break; 2048 case MVT::i32: 2049 Opcode = NVPTX::STV_i32_v4_areg; 2050 break; 2051 case MVT::f32: 2052 Opcode = NVPTX::STV_f32_v4_areg; 2053 break; 2054 } 2055 break; 2056 } 2057 } 2058 StOps.push_back(N2); 2059 } 2060 2061 StOps.push_back(Chain); 2062 2063 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); 2064 2065 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2066 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2067 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); 2068 2069 return ST; 2070} 2071 2072SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) { 2073 SDValue Chain = Node->getOperand(0); 2074 SDValue Offset = Node->getOperand(2); 2075 SDValue Flag = Node->getOperand(3); 2076 SDLoc DL(Node); 2077 MemSDNode *Mem = cast<MemSDNode>(Node); 2078 2079 unsigned VecSize; 2080 switch (Node->getOpcode()) { 2081 default: 2082 return nullptr; 2083 case NVPTXISD::LoadParam: 2084 VecSize = 1; 2085 break; 2086 case NVPTXISD::LoadParamV2: 2087 VecSize = 2; 2088 break; 2089 case NVPTXISD::LoadParamV4: 2090 VecSize = 4; 2091 break; 2092 } 2093 2094 EVT EltVT = Node->getValueType(0); 2095 EVT MemVT = Mem->getMemoryVT(); 2096 2097 unsigned Opc = 0; 2098 2099 switch (VecSize) { 2100 default: 2101 return nullptr; 2102 case 1: 2103 switch (MemVT.getSimpleVT().SimpleTy) { 2104 default: 2105 return nullptr; 2106 case MVT::i1: 2107 Opc = NVPTX::LoadParamMemI8; 2108 break; 2109 case MVT::i8: 2110 Opc = NVPTX::LoadParamMemI8; 2111 break; 2112 case MVT::i16: 2113 Opc = NVPTX::LoadParamMemI16; 2114 break; 2115 case MVT::i32: 2116 Opc = NVPTX::LoadParamMemI32; 2117 break; 2118 case MVT::i64: 2119 Opc = NVPTX::LoadParamMemI64; 2120 break; 2121 case MVT::f32: 2122 Opc = NVPTX::LoadParamMemF32; 2123 break; 2124 case MVT::f64: 2125 Opc = NVPTX::LoadParamMemF64; 2126 break; 2127 } 2128 break; 2129 case 2: 2130 switch (MemVT.getSimpleVT().SimpleTy) { 2131 default: 2132 return nullptr; 2133 case MVT::i1: 2134 Opc = NVPTX::LoadParamMemV2I8; 2135 break; 2136 case MVT::i8: 2137 Opc = NVPTX::LoadParamMemV2I8; 2138 break; 2139 case MVT::i16: 2140 Opc = NVPTX::LoadParamMemV2I16; 2141 break; 2142 case MVT::i32: 2143 Opc = NVPTX::LoadParamMemV2I32; 2144 break; 2145 case MVT::i64: 2146 Opc = NVPTX::LoadParamMemV2I64; 2147 break; 2148 case MVT::f32: 2149 Opc = NVPTX::LoadParamMemV2F32; 2150 break; 2151 case MVT::f64: 2152 Opc = NVPTX::LoadParamMemV2F64; 2153 break; 2154 } 2155 break; 2156 case 4: 2157 switch (MemVT.getSimpleVT().SimpleTy) { 2158 default: 2159 return nullptr; 2160 case MVT::i1: 2161 Opc = NVPTX::LoadParamMemV4I8; 2162 break; 2163 case MVT::i8: 2164 Opc = NVPTX::LoadParamMemV4I8; 2165 break; 2166 case MVT::i16: 2167 Opc = NVPTX::LoadParamMemV4I16; 2168 break; 2169 case MVT::i32: 2170 Opc = NVPTX::LoadParamMemV4I32; 2171 break; 2172 case MVT::f32: 2173 Opc = NVPTX::LoadParamMemV4F32; 2174 break; 2175 } 2176 break; 2177 } 2178 2179 SDVTList VTs; 2180 if (VecSize == 1) { 2181 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); 2182 } else if (VecSize == 2) { 2183 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); 2184 } else { 2185 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; 2186 VTs = CurDAG->getVTList(EVTs); 2187 } 2188 2189 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2190 2191 SmallVector<SDValue, 2> Ops; 2192 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2193 Ops.push_back(Chain); 2194 Ops.push_back(Flag); 2195 2196 SDNode *Ret = 2197 CurDAG->getMachineNode(Opc, DL, VTs, Ops); 2198 return Ret; 2199} 2200 2201SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) { 2202 SDLoc DL(N); 2203 SDValue Chain = N->getOperand(0); 2204 SDValue Offset = N->getOperand(1); 2205 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2206 MemSDNode *Mem = cast<MemSDNode>(N); 2207 2208 // How many elements do we have? 2209 unsigned NumElts = 1; 2210 switch (N->getOpcode()) { 2211 default: 2212 return nullptr; 2213 case NVPTXISD::StoreRetval: 2214 NumElts = 1; 2215 break; 2216 case NVPTXISD::StoreRetvalV2: 2217 NumElts = 2; 2218 break; 2219 case NVPTXISD::StoreRetvalV4: 2220 NumElts = 4; 2221 break; 2222 } 2223 2224 // Build vector of operands 2225 SmallVector<SDValue, 6> Ops; 2226 for (unsigned i = 0; i < NumElts; ++i) 2227 Ops.push_back(N->getOperand(i + 2)); 2228 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2229 Ops.push_back(Chain); 2230 2231 // Determine target opcode 2232 // If we have an i1, use an 8-bit store. The lowering code in 2233 // NVPTXISelLowering will have already emitted an upcast. 2234 unsigned Opcode = 0; 2235 switch (NumElts) { 2236 default: 2237 return nullptr; 2238 case 1: 2239 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2240 default: 2241 return nullptr; 2242 case MVT::i1: 2243 Opcode = NVPTX::StoreRetvalI8; 2244 break; 2245 case MVT::i8: 2246 Opcode = NVPTX::StoreRetvalI8; 2247 break; 2248 case MVT::i16: 2249 Opcode = NVPTX::StoreRetvalI16; 2250 break; 2251 case MVT::i32: 2252 Opcode = NVPTX::StoreRetvalI32; 2253 break; 2254 case MVT::i64: 2255 Opcode = NVPTX::StoreRetvalI64; 2256 break; 2257 case MVT::f32: 2258 Opcode = NVPTX::StoreRetvalF32; 2259 break; 2260 case MVT::f64: 2261 Opcode = NVPTX::StoreRetvalF64; 2262 break; 2263 } 2264 break; 2265 case 2: 2266 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2267 default: 2268 return nullptr; 2269 case MVT::i1: 2270 Opcode = NVPTX::StoreRetvalV2I8; 2271 break; 2272 case MVT::i8: 2273 Opcode = NVPTX::StoreRetvalV2I8; 2274 break; 2275 case MVT::i16: 2276 Opcode = NVPTX::StoreRetvalV2I16; 2277 break; 2278 case MVT::i32: 2279 Opcode = NVPTX::StoreRetvalV2I32; 2280 break; 2281 case MVT::i64: 2282 Opcode = NVPTX::StoreRetvalV2I64; 2283 break; 2284 case MVT::f32: 2285 Opcode = NVPTX::StoreRetvalV2F32; 2286 break; 2287 case MVT::f64: 2288 Opcode = NVPTX::StoreRetvalV2F64; 2289 break; 2290 } 2291 break; 2292 case 4: 2293 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2294 default: 2295 return nullptr; 2296 case MVT::i1: 2297 Opcode = NVPTX::StoreRetvalV4I8; 2298 break; 2299 case MVT::i8: 2300 Opcode = NVPTX::StoreRetvalV4I8; 2301 break; 2302 case MVT::i16: 2303 Opcode = NVPTX::StoreRetvalV4I16; 2304 break; 2305 case MVT::i32: 2306 Opcode = NVPTX::StoreRetvalV4I32; 2307 break; 2308 case MVT::f32: 2309 Opcode = NVPTX::StoreRetvalV4F32; 2310 break; 2311 } 2312 break; 2313 } 2314 2315 SDNode *Ret = 2316 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 2317 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2318 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2319 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2320 2321 return Ret; 2322} 2323 2324SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) { 2325 SDLoc DL(N); 2326 SDValue Chain = N->getOperand(0); 2327 SDValue Param = N->getOperand(1); 2328 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue(); 2329 SDValue Offset = N->getOperand(2); 2330 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2331 MemSDNode *Mem = cast<MemSDNode>(N); 2332 SDValue Flag = N->getOperand(N->getNumOperands() - 1); 2333 2334 // How many elements do we have? 2335 unsigned NumElts = 1; 2336 switch (N->getOpcode()) { 2337 default: 2338 return nullptr; 2339 case NVPTXISD::StoreParamU32: 2340 case NVPTXISD::StoreParamS32: 2341 case NVPTXISD::StoreParam: 2342 NumElts = 1; 2343 break; 2344 case NVPTXISD::StoreParamV2: 2345 NumElts = 2; 2346 break; 2347 case NVPTXISD::StoreParamV4: 2348 NumElts = 4; 2349 break; 2350 } 2351 2352 // Build vector of operands 2353 SmallVector<SDValue, 8> Ops; 2354 for (unsigned i = 0; i < NumElts; ++i) 2355 Ops.push_back(N->getOperand(i + 3)); 2356 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32)); 2357 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2358 Ops.push_back(Chain); 2359 Ops.push_back(Flag); 2360 2361 // Determine target opcode 2362 // If we have an i1, use an 8-bit store. The lowering code in 2363 // NVPTXISelLowering will have already emitted an upcast. 2364 unsigned Opcode = 0; 2365 switch (N->getOpcode()) { 2366 default: 2367 switch (NumElts) { 2368 default: 2369 return nullptr; 2370 case 1: 2371 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2372 default: 2373 return nullptr; 2374 case MVT::i1: 2375 Opcode = NVPTX::StoreParamI8; 2376 break; 2377 case MVT::i8: 2378 Opcode = NVPTX::StoreParamI8; 2379 break; 2380 case MVT::i16: 2381 Opcode = NVPTX::StoreParamI16; 2382 break; 2383 case MVT::i32: 2384 Opcode = NVPTX::StoreParamI32; 2385 break; 2386 case MVT::i64: 2387 Opcode = NVPTX::StoreParamI64; 2388 break; 2389 case MVT::f32: 2390 Opcode = NVPTX::StoreParamF32; 2391 break; 2392 case MVT::f64: 2393 Opcode = NVPTX::StoreParamF64; 2394 break; 2395 } 2396 break; 2397 case 2: 2398 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2399 default: 2400 return nullptr; 2401 case MVT::i1: 2402 Opcode = NVPTX::StoreParamV2I8; 2403 break; 2404 case MVT::i8: 2405 Opcode = NVPTX::StoreParamV2I8; 2406 break; 2407 case MVT::i16: 2408 Opcode = NVPTX::StoreParamV2I16; 2409 break; 2410 case MVT::i32: 2411 Opcode = NVPTX::StoreParamV2I32; 2412 break; 2413 case MVT::i64: 2414 Opcode = NVPTX::StoreParamV2I64; 2415 break; 2416 case MVT::f32: 2417 Opcode = NVPTX::StoreParamV2F32; 2418 break; 2419 case MVT::f64: 2420 Opcode = NVPTX::StoreParamV2F64; 2421 break; 2422 } 2423 break; 2424 case 4: 2425 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2426 default: 2427 return nullptr; 2428 case MVT::i1: 2429 Opcode = NVPTX::StoreParamV4I8; 2430 break; 2431 case MVT::i8: 2432 Opcode = NVPTX::StoreParamV4I8; 2433 break; 2434 case MVT::i16: 2435 Opcode = NVPTX::StoreParamV4I16; 2436 break; 2437 case MVT::i32: 2438 Opcode = NVPTX::StoreParamV4I32; 2439 break; 2440 case MVT::f32: 2441 Opcode = NVPTX::StoreParamV4F32; 2442 break; 2443 } 2444 break; 2445 } 2446 break; 2447 // Special case: if we have a sign-extend/zero-extend node, insert the 2448 // conversion instruction first, and use that as the value operand to 2449 // the selected StoreParam node. 2450 case NVPTXISD::StoreParamU32: { 2451 Opcode = NVPTX::StoreParamI32; 2452 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2453 MVT::i32); 2454 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL, 2455 MVT::i32, Ops[0], CvtNone); 2456 Ops[0] = SDValue(Cvt, 0); 2457 break; 2458 } 2459 case NVPTXISD::StoreParamS32: { 2460 Opcode = NVPTX::StoreParamI32; 2461 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2462 MVT::i32); 2463 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL, 2464 MVT::i32, Ops[0], CvtNone); 2465 Ops[0] = SDValue(Cvt, 0); 2466 break; 2467 } 2468 } 2469 2470 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 2471 SDNode *Ret = 2472 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); 2473 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2474 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2475 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2476 2477 return Ret; 2478} 2479 2480SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) { 2481 SDValue Chain = N->getOperand(0); 2482 SDValue TexRef = N->getOperand(1); 2483 SDValue SampRef = N->getOperand(2); 2484 SDNode *Ret = nullptr; 2485 unsigned Opc = 0; 2486 SmallVector<SDValue, 8> Ops; 2487 2488 switch (N->getOpcode()) { 2489 default: return nullptr; 2490 case NVPTXISD::Tex1DFloatI32: 2491 Opc = NVPTX::TEX_1D_F32_I32; 2492 break; 2493 case NVPTXISD::Tex1DFloatFloat: 2494 Opc = NVPTX::TEX_1D_F32_F32; 2495 break; 2496 case NVPTXISD::Tex1DFloatFloatLevel: 2497 Opc = NVPTX::TEX_1D_F32_F32_LEVEL; 2498 break; 2499 case NVPTXISD::Tex1DFloatFloatGrad: 2500 Opc = NVPTX::TEX_1D_F32_F32_GRAD; 2501 break; 2502 case NVPTXISD::Tex1DI32I32: 2503 Opc = NVPTX::TEX_1D_I32_I32; 2504 break; 2505 case NVPTXISD::Tex1DI32Float: 2506 Opc = NVPTX::TEX_1D_I32_F32; 2507 break; 2508 case NVPTXISD::Tex1DI32FloatLevel: 2509 Opc = NVPTX::TEX_1D_I32_F32_LEVEL; 2510 break; 2511 case NVPTXISD::Tex1DI32FloatGrad: 2512 Opc = NVPTX::TEX_1D_I32_F32_GRAD; 2513 break; 2514 case NVPTXISD::Tex1DArrayFloatI32: 2515 Opc = NVPTX::TEX_1D_ARRAY_F32_I32; 2516 break; 2517 case NVPTXISD::Tex1DArrayFloatFloat: 2518 Opc = NVPTX::TEX_1D_ARRAY_F32_F32; 2519 break; 2520 case NVPTXISD::Tex1DArrayFloatFloatLevel: 2521 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL; 2522 break; 2523 case NVPTXISD::Tex1DArrayFloatFloatGrad: 2524 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD; 2525 break; 2526 case NVPTXISD::Tex1DArrayI32I32: 2527 Opc = NVPTX::TEX_1D_ARRAY_I32_I32; 2528 break; 2529 case NVPTXISD::Tex1DArrayI32Float: 2530 Opc = NVPTX::TEX_1D_ARRAY_I32_F32; 2531 break; 2532 case NVPTXISD::Tex1DArrayI32FloatLevel: 2533 Opc = NVPTX::TEX_1D_ARRAY_I32_F32_LEVEL; 2534 break; 2535 case NVPTXISD::Tex1DArrayI32FloatGrad: 2536 Opc = NVPTX::TEX_1D_ARRAY_I32_F32_GRAD; 2537 break; 2538 case NVPTXISD::Tex2DFloatI32: 2539 Opc = NVPTX::TEX_2D_F32_I32; 2540 break; 2541 case NVPTXISD::Tex2DFloatFloat: 2542 Opc = NVPTX::TEX_2D_F32_F32; 2543 break; 2544 case NVPTXISD::Tex2DFloatFloatLevel: 2545 Opc = NVPTX::TEX_2D_F32_F32_LEVEL; 2546 break; 2547 case NVPTXISD::Tex2DFloatFloatGrad: 2548 Opc = NVPTX::TEX_2D_F32_F32_GRAD; 2549 break; 2550 case NVPTXISD::Tex2DI32I32: 2551 Opc = NVPTX::TEX_2D_I32_I32; 2552 break; 2553 case NVPTXISD::Tex2DI32Float: 2554 Opc = NVPTX::TEX_2D_I32_F32; 2555 break; 2556 case NVPTXISD::Tex2DI32FloatLevel: 2557 Opc = NVPTX::TEX_2D_I32_F32_LEVEL; 2558 break; 2559 case NVPTXISD::Tex2DI32FloatGrad: 2560 Opc = NVPTX::TEX_2D_I32_F32_GRAD; 2561 break; 2562 case NVPTXISD::Tex2DArrayFloatI32: 2563 Opc = NVPTX::TEX_2D_ARRAY_F32_I32; 2564 break; 2565 case NVPTXISD::Tex2DArrayFloatFloat: 2566 Opc = NVPTX::TEX_2D_ARRAY_F32_F32; 2567 break; 2568 case NVPTXISD::Tex2DArrayFloatFloatLevel: 2569 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL; 2570 break; 2571 case NVPTXISD::Tex2DArrayFloatFloatGrad: 2572 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD; 2573 break; 2574 case NVPTXISD::Tex2DArrayI32I32: 2575 Opc = NVPTX::TEX_2D_ARRAY_I32_I32; 2576 break; 2577 case NVPTXISD::Tex2DArrayI32Float: 2578 Opc = NVPTX::TEX_2D_ARRAY_I32_F32; 2579 break; 2580 case NVPTXISD::Tex2DArrayI32FloatLevel: 2581 Opc = NVPTX::TEX_2D_ARRAY_I32_F32_LEVEL; 2582 break; 2583 case NVPTXISD::Tex2DArrayI32FloatGrad: 2584 Opc = NVPTX::TEX_2D_ARRAY_I32_F32_GRAD; 2585 break; 2586 case NVPTXISD::Tex3DFloatI32: 2587 Opc = NVPTX::TEX_3D_F32_I32; 2588 break; 2589 case NVPTXISD::Tex3DFloatFloat: 2590 Opc = NVPTX::TEX_3D_F32_F32; 2591 break; 2592 case NVPTXISD::Tex3DFloatFloatLevel: 2593 Opc = NVPTX::TEX_3D_F32_F32_LEVEL; 2594 break; 2595 case NVPTXISD::Tex3DFloatFloatGrad: 2596 Opc = NVPTX::TEX_3D_F32_F32_GRAD; 2597 break; 2598 case NVPTXISD::Tex3DI32I32: 2599 Opc = NVPTX::TEX_3D_I32_I32; 2600 break; 2601 case NVPTXISD::Tex3DI32Float: 2602 Opc = NVPTX::TEX_3D_I32_F32; 2603 break; 2604 case NVPTXISD::Tex3DI32FloatLevel: 2605 Opc = NVPTX::TEX_3D_I32_F32_LEVEL; 2606 break; 2607 case NVPTXISD::Tex3DI32FloatGrad: 2608 Opc = NVPTX::TEX_3D_I32_F32_GRAD; 2609 break; 2610 } 2611 2612 Ops.push_back(TexRef); 2613 Ops.push_back(SampRef); 2614 2615 // Copy over indices 2616 for (unsigned i = 3; i < N->getNumOperands(); ++i) { 2617 Ops.push_back(N->getOperand(i)); 2618 } 2619 2620 Ops.push_back(Chain); 2621 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 2622 return Ret; 2623} 2624 2625SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) { 2626 SDValue Chain = N->getOperand(0); 2627 SDValue TexHandle = N->getOperand(1); 2628 SDNode *Ret = nullptr; 2629 unsigned Opc = 0; 2630 SmallVector<SDValue, 8> Ops; 2631 switch (N->getOpcode()) { 2632 default: return nullptr; 2633 case NVPTXISD::Suld1DI8Trap: 2634 Opc = NVPTX::SULD_1D_I8_TRAP; 2635 Ops.push_back(TexHandle); 2636 Ops.push_back(N->getOperand(2)); 2637 Ops.push_back(Chain); 2638 break; 2639 case NVPTXISD::Suld1DI16Trap: 2640 Opc = NVPTX::SULD_1D_I16_TRAP; 2641 Ops.push_back(TexHandle); 2642 Ops.push_back(N->getOperand(2)); 2643 Ops.push_back(Chain); 2644 break; 2645 case NVPTXISD::Suld1DI32Trap: 2646 Opc = NVPTX::SULD_1D_I32_TRAP; 2647 Ops.push_back(TexHandle); 2648 Ops.push_back(N->getOperand(2)); 2649 Ops.push_back(Chain); 2650 break; 2651 case NVPTXISD::Suld1DV2I8Trap: 2652 Opc = NVPTX::SULD_1D_V2I8_TRAP; 2653 Ops.push_back(TexHandle); 2654 Ops.push_back(N->getOperand(2)); 2655 Ops.push_back(Chain); 2656 break; 2657 case NVPTXISD::Suld1DV2I16Trap: 2658 Opc = NVPTX::SULD_1D_V2I16_TRAP; 2659 Ops.push_back(TexHandle); 2660 Ops.push_back(N->getOperand(2)); 2661 Ops.push_back(Chain); 2662 break; 2663 case NVPTXISD::Suld1DV2I32Trap: 2664 Opc = NVPTX::SULD_1D_V2I32_TRAP; 2665 Ops.push_back(TexHandle); 2666 Ops.push_back(N->getOperand(2)); 2667 Ops.push_back(Chain); 2668 break; 2669 case NVPTXISD::Suld1DV4I8Trap: 2670 Opc = NVPTX::SULD_1D_V4I8_TRAP; 2671 Ops.push_back(TexHandle); 2672 Ops.push_back(N->getOperand(2)); 2673 Ops.push_back(Chain); 2674 break; 2675 case NVPTXISD::Suld1DV4I16Trap: 2676 Opc = NVPTX::SULD_1D_V4I16_TRAP; 2677 Ops.push_back(TexHandle); 2678 Ops.push_back(N->getOperand(2)); 2679 Ops.push_back(Chain); 2680 break; 2681 case NVPTXISD::Suld1DV4I32Trap: 2682 Opc = NVPTX::SULD_1D_V4I32_TRAP; 2683 Ops.push_back(TexHandle); 2684 Ops.push_back(N->getOperand(2)); 2685 Ops.push_back(Chain); 2686 break; 2687 case NVPTXISD::Suld1DArrayI8Trap: 2688 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP; 2689 Ops.push_back(TexHandle); 2690 Ops.push_back(N->getOperand(2)); 2691 Ops.push_back(N->getOperand(3)); 2692 Ops.push_back(Chain); 2693 break; 2694 case NVPTXISD::Suld1DArrayI16Trap: 2695 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP; 2696 Ops.push_back(TexHandle); 2697 Ops.push_back(N->getOperand(2)); 2698 Ops.push_back(N->getOperand(3)); 2699 Ops.push_back(Chain); 2700 break; 2701 case NVPTXISD::Suld1DArrayI32Trap: 2702 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP; 2703 Ops.push_back(TexHandle); 2704 Ops.push_back(N->getOperand(2)); 2705 Ops.push_back(N->getOperand(3)); 2706 Ops.push_back(Chain); 2707 break; 2708 case NVPTXISD::Suld1DArrayV2I8Trap: 2709 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP; 2710 Ops.push_back(TexHandle); 2711 Ops.push_back(N->getOperand(2)); 2712 Ops.push_back(N->getOperand(3)); 2713 Ops.push_back(Chain); 2714 break; 2715 case NVPTXISD::Suld1DArrayV2I16Trap: 2716 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP; 2717 Ops.push_back(TexHandle); 2718 Ops.push_back(N->getOperand(2)); 2719 Ops.push_back(N->getOperand(3)); 2720 Ops.push_back(Chain); 2721 break; 2722 case NVPTXISD::Suld1DArrayV2I32Trap: 2723 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP; 2724 Ops.push_back(TexHandle); 2725 Ops.push_back(N->getOperand(2)); 2726 Ops.push_back(N->getOperand(3)); 2727 Ops.push_back(Chain); 2728 break; 2729 case NVPTXISD::Suld1DArrayV4I8Trap: 2730 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP; 2731 Ops.push_back(TexHandle); 2732 Ops.push_back(N->getOperand(2)); 2733 Ops.push_back(N->getOperand(3)); 2734 Ops.push_back(Chain); 2735 break; 2736 case NVPTXISD::Suld1DArrayV4I16Trap: 2737 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP; 2738 Ops.push_back(TexHandle); 2739 Ops.push_back(N->getOperand(2)); 2740 Ops.push_back(N->getOperand(3)); 2741 Ops.push_back(Chain); 2742 break; 2743 case NVPTXISD::Suld1DArrayV4I32Trap: 2744 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP; 2745 Ops.push_back(TexHandle); 2746 Ops.push_back(N->getOperand(2)); 2747 Ops.push_back(N->getOperand(3)); 2748 Ops.push_back(Chain); 2749 break; 2750 case NVPTXISD::Suld2DI8Trap: 2751 Opc = NVPTX::SULD_2D_I8_TRAP; 2752 Ops.push_back(TexHandle); 2753 Ops.push_back(N->getOperand(2)); 2754 Ops.push_back(N->getOperand(3)); 2755 Ops.push_back(Chain); 2756 break; 2757 case NVPTXISD::Suld2DI16Trap: 2758 Opc = NVPTX::SULD_2D_I16_TRAP; 2759 Ops.push_back(TexHandle); 2760 Ops.push_back(N->getOperand(2)); 2761 Ops.push_back(N->getOperand(3)); 2762 Ops.push_back(Chain); 2763 break; 2764 case NVPTXISD::Suld2DI32Trap: 2765 Opc = NVPTX::SULD_2D_I32_TRAP; 2766 Ops.push_back(TexHandle); 2767 Ops.push_back(N->getOperand(2)); 2768 Ops.push_back(N->getOperand(3)); 2769 Ops.push_back(Chain); 2770 break; 2771 case NVPTXISD::Suld2DV2I8Trap: 2772 Opc = NVPTX::SULD_2D_V2I8_TRAP; 2773 Ops.push_back(TexHandle); 2774 Ops.push_back(N->getOperand(2)); 2775 Ops.push_back(N->getOperand(3)); 2776 Ops.push_back(Chain); 2777 break; 2778 case NVPTXISD::Suld2DV2I16Trap: 2779 Opc = NVPTX::SULD_2D_V2I16_TRAP; 2780 Ops.push_back(TexHandle); 2781 Ops.push_back(N->getOperand(2)); 2782 Ops.push_back(N->getOperand(3)); 2783 Ops.push_back(Chain); 2784 break; 2785 case NVPTXISD::Suld2DV2I32Trap: 2786 Opc = NVPTX::SULD_2D_V2I32_TRAP; 2787 Ops.push_back(TexHandle); 2788 Ops.push_back(N->getOperand(2)); 2789 Ops.push_back(N->getOperand(3)); 2790 Ops.push_back(Chain); 2791 break; 2792 case NVPTXISD::Suld2DV4I8Trap: 2793 Opc = NVPTX::SULD_2D_V4I8_TRAP; 2794 Ops.push_back(TexHandle); 2795 Ops.push_back(N->getOperand(2)); 2796 Ops.push_back(N->getOperand(3)); 2797 Ops.push_back(Chain); 2798 break; 2799 case NVPTXISD::Suld2DV4I16Trap: 2800 Opc = NVPTX::SULD_2D_V4I16_TRAP; 2801 Ops.push_back(TexHandle); 2802 Ops.push_back(N->getOperand(2)); 2803 Ops.push_back(N->getOperand(3)); 2804 Ops.push_back(Chain); 2805 break; 2806 case NVPTXISD::Suld2DV4I32Trap: 2807 Opc = NVPTX::SULD_2D_V4I32_TRAP; 2808 Ops.push_back(TexHandle); 2809 Ops.push_back(N->getOperand(2)); 2810 Ops.push_back(N->getOperand(3)); 2811 Ops.push_back(Chain); 2812 break; 2813 case NVPTXISD::Suld2DArrayI8Trap: 2814 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP; 2815 Ops.push_back(TexHandle); 2816 Ops.push_back(N->getOperand(2)); 2817 Ops.push_back(N->getOperand(3)); 2818 Ops.push_back(N->getOperand(4)); 2819 Ops.push_back(Chain); 2820 break; 2821 case NVPTXISD::Suld2DArrayI16Trap: 2822 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP; 2823 Ops.push_back(TexHandle); 2824 Ops.push_back(N->getOperand(2)); 2825 Ops.push_back(N->getOperand(3)); 2826 Ops.push_back(N->getOperand(4)); 2827 Ops.push_back(Chain); 2828 break; 2829 case NVPTXISD::Suld2DArrayI32Trap: 2830 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP; 2831 Ops.push_back(TexHandle); 2832 Ops.push_back(N->getOperand(2)); 2833 Ops.push_back(N->getOperand(3)); 2834 Ops.push_back(N->getOperand(4)); 2835 Ops.push_back(Chain); 2836 break; 2837 case NVPTXISD::Suld2DArrayV2I8Trap: 2838 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP; 2839 Ops.push_back(TexHandle); 2840 Ops.push_back(N->getOperand(2)); 2841 Ops.push_back(N->getOperand(3)); 2842 Ops.push_back(N->getOperand(4)); 2843 Ops.push_back(Chain); 2844 break; 2845 case NVPTXISD::Suld2DArrayV2I16Trap: 2846 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP; 2847 Ops.push_back(TexHandle); 2848 Ops.push_back(N->getOperand(2)); 2849 Ops.push_back(N->getOperand(3)); 2850 Ops.push_back(N->getOperand(4)); 2851 Ops.push_back(Chain); 2852 break; 2853 case NVPTXISD::Suld2DArrayV2I32Trap: 2854 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP; 2855 Ops.push_back(TexHandle); 2856 Ops.push_back(N->getOperand(2)); 2857 Ops.push_back(N->getOperand(3)); 2858 Ops.push_back(N->getOperand(4)); 2859 Ops.push_back(Chain); 2860 break; 2861 case NVPTXISD::Suld2DArrayV4I8Trap: 2862 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP; 2863 Ops.push_back(TexHandle); 2864 Ops.push_back(N->getOperand(2)); 2865 Ops.push_back(N->getOperand(3)); 2866 Ops.push_back(N->getOperand(4)); 2867 Ops.push_back(Chain); 2868 break; 2869 case NVPTXISD::Suld2DArrayV4I16Trap: 2870 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP; 2871 Ops.push_back(TexHandle); 2872 Ops.push_back(N->getOperand(2)); 2873 Ops.push_back(N->getOperand(3)); 2874 Ops.push_back(N->getOperand(4)); 2875 Ops.push_back(Chain); 2876 break; 2877 case NVPTXISD::Suld2DArrayV4I32Trap: 2878 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP; 2879 Ops.push_back(TexHandle); 2880 Ops.push_back(N->getOperand(2)); 2881 Ops.push_back(N->getOperand(3)); 2882 Ops.push_back(N->getOperand(4)); 2883 Ops.push_back(Chain); 2884 break; 2885 case NVPTXISD::Suld3DI8Trap: 2886 Opc = NVPTX::SULD_3D_I8_TRAP; 2887 Ops.push_back(TexHandle); 2888 Ops.push_back(N->getOperand(2)); 2889 Ops.push_back(N->getOperand(3)); 2890 Ops.push_back(N->getOperand(4)); 2891 Ops.push_back(Chain); 2892 break; 2893 case NVPTXISD::Suld3DI16Trap: 2894 Opc = NVPTX::SULD_3D_I16_TRAP; 2895 Ops.push_back(TexHandle); 2896 Ops.push_back(N->getOperand(2)); 2897 Ops.push_back(N->getOperand(3)); 2898 Ops.push_back(N->getOperand(4)); 2899 Ops.push_back(Chain); 2900 break; 2901 case NVPTXISD::Suld3DI32Trap: 2902 Opc = NVPTX::SULD_3D_I32_TRAP; 2903 Ops.push_back(TexHandle); 2904 Ops.push_back(N->getOperand(2)); 2905 Ops.push_back(N->getOperand(3)); 2906 Ops.push_back(N->getOperand(4)); 2907 Ops.push_back(Chain); 2908 break; 2909 case NVPTXISD::Suld3DV2I8Trap: 2910 Opc = NVPTX::SULD_3D_V2I8_TRAP; 2911 Ops.push_back(TexHandle); 2912 Ops.push_back(N->getOperand(2)); 2913 Ops.push_back(N->getOperand(3)); 2914 Ops.push_back(N->getOperand(4)); 2915 Ops.push_back(Chain); 2916 break; 2917 case NVPTXISD::Suld3DV2I16Trap: 2918 Opc = NVPTX::SULD_3D_V2I16_TRAP; 2919 Ops.push_back(TexHandle); 2920 Ops.push_back(N->getOperand(2)); 2921 Ops.push_back(N->getOperand(3)); 2922 Ops.push_back(N->getOperand(4)); 2923 Ops.push_back(Chain); 2924 break; 2925 case NVPTXISD::Suld3DV2I32Trap: 2926 Opc = NVPTX::SULD_3D_V2I32_TRAP; 2927 Ops.push_back(TexHandle); 2928 Ops.push_back(N->getOperand(2)); 2929 Ops.push_back(N->getOperand(3)); 2930 Ops.push_back(N->getOperand(4)); 2931 Ops.push_back(Chain); 2932 break; 2933 case NVPTXISD::Suld3DV4I8Trap: 2934 Opc = NVPTX::SULD_3D_V4I8_TRAP; 2935 Ops.push_back(TexHandle); 2936 Ops.push_back(N->getOperand(2)); 2937 Ops.push_back(N->getOperand(3)); 2938 Ops.push_back(N->getOperand(4)); 2939 Ops.push_back(Chain); 2940 break; 2941 case NVPTXISD::Suld3DV4I16Trap: 2942 Opc = NVPTX::SULD_3D_V4I16_TRAP; 2943 Ops.push_back(TexHandle); 2944 Ops.push_back(N->getOperand(2)); 2945 Ops.push_back(N->getOperand(3)); 2946 Ops.push_back(N->getOperand(4)); 2947 Ops.push_back(Chain); 2948 break; 2949 case NVPTXISD::Suld3DV4I32Trap: 2950 Opc = NVPTX::SULD_3D_V4I32_TRAP; 2951 Ops.push_back(TexHandle); 2952 Ops.push_back(N->getOperand(2)); 2953 Ops.push_back(N->getOperand(3)); 2954 Ops.push_back(N->getOperand(4)); 2955 Ops.push_back(Chain); 2956 break; 2957 } 2958 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 2959 return Ret; 2960} 2961 2962// SelectDirectAddr - Match a direct address for DAG. 2963// A direct address could be a globaladdress or externalsymbol. 2964bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { 2965 // Return true if TGA or ES. 2966 if (N.getOpcode() == ISD::TargetGlobalAddress || 2967 N.getOpcode() == ISD::TargetExternalSymbol) { 2968 Address = N; 2969 return true; 2970 } 2971 if (N.getOpcode() == NVPTXISD::Wrapper) { 2972 Address = N.getOperand(0); 2973 return true; 2974 } 2975 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 2976 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); 2977 if (IID == Intrinsic::nvvm_ptr_gen_to_param) 2978 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) 2979 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); 2980 } 2981 return false; 2982} 2983 2984// symbol+offset 2985bool NVPTXDAGToDAGISel::SelectADDRsi_imp( 2986 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 2987 if (Addr.getOpcode() == ISD::ADD) { 2988 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 2989 SDValue base = Addr.getOperand(0); 2990 if (SelectDirectAddr(base, Base)) { 2991 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 2992 return true; 2993 } 2994 } 2995 } 2996 return false; 2997} 2998 2999// symbol+offset 3000bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, 3001 SDValue &Base, SDValue &Offset) { 3002 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); 3003} 3004 3005// symbol+offset 3006bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, 3007 SDValue &Base, SDValue &Offset) { 3008 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); 3009} 3010 3011// register+offset 3012bool NVPTXDAGToDAGISel::SelectADDRri_imp( 3013 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 3014 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 3015 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 3016 Offset = CurDAG->getTargetConstant(0, mvt); 3017 return true; 3018 } 3019 if (Addr.getOpcode() == ISD::TargetExternalSymbol || 3020 Addr.getOpcode() == ISD::TargetGlobalAddress) 3021 return false; // direct calls. 3022 3023 if (Addr.getOpcode() == ISD::ADD) { 3024 if (SelectDirectAddr(Addr.getOperand(0), Addr)) { 3025 return false; 3026 } 3027 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 3028 if (FrameIndexSDNode *FIN = 3029 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) 3030 // Constant offset from frame ref. 3031 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 3032 else 3033 Base = Addr.getOperand(0); 3034 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 3035 return true; 3036 } 3037 } 3038 return false; 3039} 3040 3041// register+offset 3042bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, 3043 SDValue &Base, SDValue &Offset) { 3044 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); 3045} 3046 3047// register+offset 3048bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, 3049 SDValue &Base, SDValue &Offset) { 3050 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 3051} 3052 3053bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 3054 unsigned int spN) const { 3055 const Value *Src = nullptr; 3056 // Even though MemIntrinsicSDNode is a subclas of MemSDNode, 3057 // the classof() for MemSDNode does not include MemIntrinsicSDNode 3058 // (See SelectionDAGNodes.h). So we need to check for both. 3059 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 3060 if (spN == 0 && mN->getMemOperand()->getPseudoValue()) 3061 return true; 3062 Src = mN->getMemOperand()->getValue(); 3063 } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) { 3064 if (spN == 0 && mN->getMemOperand()->getPseudoValue()) 3065 return true; 3066 Src = mN->getMemOperand()->getValue(); 3067 } 3068 if (!Src) 3069 return false; 3070 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) 3071 return (PT->getAddressSpace() == spN); 3072 return false; 3073} 3074 3075/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 3076/// inline asm expressions. 3077bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( 3078 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) { 3079 SDValue Op0, Op1; 3080 switch (ConstraintCode) { 3081 default: 3082 return true; 3083 case 'm': // memory 3084 if (SelectDirectAddr(Op, Op0)) { 3085 OutOps.push_back(Op0); 3086 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32)); 3087 return false; 3088 } 3089 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { 3090 OutOps.push_back(Op0); 3091 OutOps.push_back(Op1); 3092 return false; 3093 } 3094 break; 3095 } 3096 return true; 3097} 3098