LegalizeVectorOps.cpp revision 36b56886974eae4f9c5ebc96befd3e7bfe5de338
1//===-- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ---===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SelectionDAG::LegalizeVectors method. 11// 12// The vector legalizer looks for vector operations which might need to be 13// scalarized and legalizes them. This is a separate step from Legalize because 14// scalarizing can introduce illegal types. For example, suppose we have an 15// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition 16// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the 17// operation, which introduces nodes with the illegal type i64 which must be 18// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC; 19// the operation must be unrolled, which introduces nodes with the illegal 20// type i8 which must be promoted. 21// 22// This does not legalize vector manipulations like ISD::BUILD_VECTOR, 23// or operations that happen to take a vector which are custom-lowered; 24// the legalization for such operations never produces nodes 25// with illegal types, so it's okay to put off legalizing them until 26// SelectionDAG::Legalize runs. 27// 28//===----------------------------------------------------------------------===// 29 30#include "llvm/CodeGen/SelectionDAG.h" 31#include "llvm/Target/TargetLowering.h" 32using namespace llvm; 33 34namespace { 35class VectorLegalizer { 36 SelectionDAG& DAG; 37 const TargetLowering &TLI; 38 bool Changed; // Keep track of whether anything changed 39 40 /// LegalizedNodes - For nodes that are of legal width, and that have more 41 /// than one use, this map indicates what regularized operand to use. This 42 /// allows us to avoid legalizing the same thing more than once. 43 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes; 44 45 // Adds a node to the translation cache 46 void AddLegalizedOperand(SDValue From, SDValue To) { 47 LegalizedNodes.insert(std::make_pair(From, To)); 48 // If someone requests legalization of the new node, return itself. 49 if (From != To) 50 LegalizedNodes.insert(std::make_pair(To, To)); 51 } 52 53 // Legalizes the given node 54 SDValue LegalizeOp(SDValue Op); 55 // Assuming the node is legal, "legalize" the results 56 SDValue TranslateLegalizeResults(SDValue Op, SDValue Result); 57 // Implements unrolling a VSETCC. 58 SDValue UnrollVSETCC(SDValue Op); 59 // Implements expansion for FNEG; falls back to UnrollVectorOp if FSUB 60 // isn't legal. 61 // Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if 62 // SINT_TO_FLOAT and SHR on vectors isn't legal. 63 SDValue ExpandUINT_TO_FLOAT(SDValue Op); 64 // Implement expansion for SIGN_EXTEND_INREG using SRL and SRA. 65 SDValue ExpandSEXTINREG(SDValue Op); 66 // Implement vselect in terms of XOR, AND, OR when blend is not supported 67 // by the target. 68 SDValue ExpandVSELECT(SDValue Op); 69 SDValue ExpandSELECT(SDValue Op); 70 SDValue ExpandLoad(SDValue Op); 71 SDValue ExpandStore(SDValue Op); 72 SDValue ExpandFNEG(SDValue Op); 73 // Implements vector promotion; this is essentially just bitcasting the 74 // operands to a different type and bitcasting the result back to the 75 // original type. 76 SDValue PromoteVectorOp(SDValue Op); 77 // Implements [SU]INT_TO_FP vector promotion; this is a [zs]ext of the input 78 // operand to the next size up. 79 SDValue PromoteVectorOpINT_TO_FP(SDValue Op); 80 // Implements FP_TO_[SU]INT vector promotion of the result type; it is 81 // promoted to the next size up integer type. The result is then truncated 82 // back to the original type. 83 SDValue PromoteVectorOpFP_TO_INT(SDValue Op, bool isSigned); 84 85 public: 86 bool Run(); 87 VectorLegalizer(SelectionDAG& dag) : 88 DAG(dag), TLI(dag.getTargetLoweringInfo()), Changed(false) {} 89}; 90 91bool VectorLegalizer::Run() { 92 // Before we start legalizing vector nodes, check if there are any vectors. 93 bool HasVectors = false; 94 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 95 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) { 96 // Check if the values of the nodes contain vectors. We don't need to check 97 // the operands because we are going to check their values at some point. 98 for (SDNode::value_iterator J = I->value_begin(), E = I->value_end(); 99 J != E; ++J) 100 HasVectors |= J->isVector(); 101 102 // If we found a vector node we can start the legalization. 103 if (HasVectors) 104 break; 105 } 106 107 // If this basic block has no vectors then no need to legalize vectors. 108 if (!HasVectors) 109 return false; 110 111 // The legalize process is inherently a bottom-up recursive process (users 112 // legalize their uses before themselves). Given infinite stack space, we 113 // could just start legalizing on the root and traverse the whole graph. In 114 // practice however, this causes us to run out of stack space on large basic 115 // blocks. To avoid this problem, compute an ordering of the nodes where each 116 // node is only legalized after all of its operands are legalized. 117 DAG.AssignTopologicalOrder(); 118 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 119 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) 120 LegalizeOp(SDValue(I, 0)); 121 122 // Finally, it's possible the root changed. Get the new root. 123 SDValue OldRoot = DAG.getRoot(); 124 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); 125 DAG.setRoot(LegalizedNodes[OldRoot]); 126 127 LegalizedNodes.clear(); 128 129 // Remove dead nodes now. 130 DAG.RemoveDeadNodes(); 131 132 return Changed; 133} 134 135SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) { 136 // Generic legalization: just pass the operand through. 137 for (unsigned i = 0, e = Op.getNode()->getNumValues(); i != e; ++i) 138 AddLegalizedOperand(Op.getValue(i), Result.getValue(i)); 139 return Result.getValue(Op.getResNo()); 140} 141 142SDValue VectorLegalizer::LegalizeOp(SDValue Op) { 143 // Note that LegalizeOp may be reentered even from single-use nodes, which 144 // means that we always must cache transformed nodes. 145 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); 146 if (I != LegalizedNodes.end()) return I->second; 147 148 SDNode* Node = Op.getNode(); 149 150 // Legalize the operands 151 SmallVector<SDValue, 8> Ops; 152 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 153 Ops.push_back(LegalizeOp(Node->getOperand(i))); 154 155 SDValue Result = 156 SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops.data(), Ops.size()), 0); 157 158 if (Op.getOpcode() == ISD::LOAD) { 159 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 160 ISD::LoadExtType ExtType = LD->getExtensionType(); 161 if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) { 162 if (TLI.isLoadExtLegal(LD->getExtensionType(), LD->getMemoryVT())) 163 return TranslateLegalizeResults(Op, Result); 164 Changed = true; 165 return LegalizeOp(ExpandLoad(Op)); 166 } 167 } else if (Op.getOpcode() == ISD::STORE) { 168 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); 169 EVT StVT = ST->getMemoryVT(); 170 MVT ValVT = ST->getValue().getSimpleValueType(); 171 if (StVT.isVector() && ST->isTruncatingStore()) 172 switch (TLI.getTruncStoreAction(ValVT, StVT.getSimpleVT())) { 173 default: llvm_unreachable("This action is not supported yet!"); 174 case TargetLowering::Legal: 175 return TranslateLegalizeResults(Op, Result); 176 case TargetLowering::Custom: 177 Changed = true; 178 return TranslateLegalizeResults(Op, TLI.LowerOperation(Result, DAG)); 179 case TargetLowering::Expand: 180 Changed = true; 181 return LegalizeOp(ExpandStore(Op)); 182 } 183 } 184 185 bool HasVectorValue = false; 186 for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end(); 187 J != E; 188 ++J) 189 HasVectorValue |= J->isVector(); 190 if (!HasVectorValue) 191 return TranslateLegalizeResults(Op, Result); 192 193 EVT QueryType; 194 switch (Op.getOpcode()) { 195 default: 196 return TranslateLegalizeResults(Op, Result); 197 case ISD::ADD: 198 case ISD::SUB: 199 case ISD::MUL: 200 case ISD::SDIV: 201 case ISD::UDIV: 202 case ISD::SREM: 203 case ISD::UREM: 204 case ISD::FADD: 205 case ISD::FSUB: 206 case ISD::FMUL: 207 case ISD::FDIV: 208 case ISD::FREM: 209 case ISD::AND: 210 case ISD::OR: 211 case ISD::XOR: 212 case ISD::SHL: 213 case ISD::SRA: 214 case ISD::SRL: 215 case ISD::ROTL: 216 case ISD::ROTR: 217 case ISD::BSWAP: 218 case ISD::CTLZ: 219 case ISD::CTTZ: 220 case ISD::CTLZ_ZERO_UNDEF: 221 case ISD::CTTZ_ZERO_UNDEF: 222 case ISD::CTPOP: 223 case ISD::SELECT: 224 case ISD::VSELECT: 225 case ISD::SELECT_CC: 226 case ISD::SETCC: 227 case ISD::ZERO_EXTEND: 228 case ISD::ANY_EXTEND: 229 case ISD::TRUNCATE: 230 case ISD::SIGN_EXTEND: 231 case ISD::FP_TO_SINT: 232 case ISD::FP_TO_UINT: 233 case ISD::FNEG: 234 case ISD::FABS: 235 case ISD::FCOPYSIGN: 236 case ISD::FSQRT: 237 case ISD::FSIN: 238 case ISD::FCOS: 239 case ISD::FPOWI: 240 case ISD::FPOW: 241 case ISD::FLOG: 242 case ISD::FLOG2: 243 case ISD::FLOG10: 244 case ISD::FEXP: 245 case ISD::FEXP2: 246 case ISD::FCEIL: 247 case ISD::FTRUNC: 248 case ISD::FRINT: 249 case ISD::FNEARBYINT: 250 case ISD::FROUND: 251 case ISD::FFLOOR: 252 case ISD::FP_ROUND: 253 case ISD::FP_EXTEND: 254 case ISD::FMA: 255 case ISD::SIGN_EXTEND_INREG: 256 QueryType = Node->getValueType(0); 257 break; 258 case ISD::FP_ROUND_INREG: 259 QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 260 break; 261 case ISD::SINT_TO_FP: 262 case ISD::UINT_TO_FP: 263 QueryType = Node->getOperand(0).getValueType(); 264 break; 265 } 266 267 switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) { 268 case TargetLowering::Promote: 269 switch (Op.getOpcode()) { 270 default: 271 // "Promote" the operation by bitcasting 272 Result = PromoteVectorOp(Op); 273 Changed = true; 274 break; 275 case ISD::SINT_TO_FP: 276 case ISD::UINT_TO_FP: 277 // "Promote" the operation by extending the operand. 278 Result = PromoteVectorOpINT_TO_FP(Op); 279 Changed = true; 280 break; 281 case ISD::FP_TO_UINT: 282 case ISD::FP_TO_SINT: 283 // Promote the operation by extending the operand. 284 Result = PromoteVectorOpFP_TO_INT(Op, Op->getOpcode() == ISD::FP_TO_SINT); 285 Changed = true; 286 break; 287 } 288 break; 289 case TargetLowering::Legal: break; 290 case TargetLowering::Custom: { 291 SDValue Tmp1 = TLI.LowerOperation(Op, DAG); 292 if (Tmp1.getNode()) { 293 Result = Tmp1; 294 break; 295 } 296 // FALL THROUGH 297 } 298 case TargetLowering::Expand: 299 if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) 300 Result = ExpandSEXTINREG(Op); 301 else if (Node->getOpcode() == ISD::VSELECT) 302 Result = ExpandVSELECT(Op); 303 else if (Node->getOpcode() == ISD::SELECT) 304 Result = ExpandSELECT(Op); 305 else if (Node->getOpcode() == ISD::UINT_TO_FP) 306 Result = ExpandUINT_TO_FLOAT(Op); 307 else if (Node->getOpcode() == ISD::FNEG) 308 Result = ExpandFNEG(Op); 309 else if (Node->getOpcode() == ISD::SETCC) 310 Result = UnrollVSETCC(Op); 311 else 312 Result = DAG.UnrollVectorOp(Op.getNode()); 313 break; 314 } 315 316 // Make sure that the generated code is itself legal. 317 if (Result != Op) { 318 Result = LegalizeOp(Result); 319 Changed = true; 320 } 321 322 // Note that LegalizeOp may be reentered even from single-use nodes, which 323 // means that we always must cache transformed nodes. 324 AddLegalizedOperand(Op, Result); 325 return Result; 326} 327 328SDValue VectorLegalizer::PromoteVectorOp(SDValue Op) { 329 // Vector "promotion" is basically just bitcasting and doing the operation 330 // in a different type. For example, x86 promotes ISD::AND on v2i32 to 331 // v1i64. 332 MVT VT = Op.getSimpleValueType(); 333 assert(Op.getNode()->getNumValues() == 1 && 334 "Can't promote a vector with multiple results!"); 335 MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT); 336 SDLoc dl(Op); 337 SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 338 339 for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 340 if (Op.getOperand(j).getValueType().isVector()) 341 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Op.getOperand(j)); 342 else 343 Operands[j] = Op.getOperand(j); 344 } 345 346 Op = DAG.getNode(Op.getOpcode(), dl, NVT, &Operands[0], Operands.size()); 347 348 return DAG.getNode(ISD::BITCAST, dl, VT, Op); 349} 350 351SDValue VectorLegalizer::PromoteVectorOpINT_TO_FP(SDValue Op) { 352 // INT_TO_FP operations may require the input operand be promoted even 353 // when the type is otherwise legal. 354 EVT VT = Op.getOperand(0).getValueType(); 355 assert(Op.getNode()->getNumValues() == 1 && 356 "Can't promote a vector with multiple results!"); 357 358 // Normal getTypeToPromoteTo() doesn't work here, as that will promote 359 // by widening the vector w/ the same element width and twice the number 360 // of elements. We want the other way around, the same number of elements, 361 // each twice the width. 362 // 363 // Increase the bitwidth of the element to the next pow-of-two 364 // (which is greater than 8 bits). 365 366 EVT NVT = VT.widenIntegerVectorElementType(*DAG.getContext()); 367 assert(NVT.isSimple() && "Promoting to a non-simple vector type!"); 368 SDLoc dl(Op); 369 SmallVector<SDValue, 4> Operands(Op.getNumOperands()); 370 371 unsigned Opc = Op.getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : 372 ISD::SIGN_EXTEND; 373 for (unsigned j = 0; j != Op.getNumOperands(); ++j) { 374 if (Op.getOperand(j).getValueType().isVector()) 375 Operands[j] = DAG.getNode(Opc, dl, NVT, Op.getOperand(j)); 376 else 377 Operands[j] = Op.getOperand(j); 378 } 379 380 return DAG.getNode(Op.getOpcode(), dl, Op.getValueType(), &Operands[0], 381 Operands.size()); 382} 383 384// For FP_TO_INT we promote the result type to a vector type with wider 385// elements and then truncate the result. This is different from the default 386// PromoteVector which uses bitcast to promote thus assumning that the 387// promoted vector type has the same overall size. 388SDValue VectorLegalizer::PromoteVectorOpFP_TO_INT(SDValue Op, bool isSigned) { 389 assert(Op.getNode()->getNumValues() == 1 && 390 "Can't promote a vector with multiple results!"); 391 EVT VT = Op.getValueType(); 392 393 EVT NewVT; 394 unsigned NewOpc; 395 while (1) { 396 NewVT = VT.widenIntegerVectorElementType(*DAG.getContext()); 397 assert(NewVT.isSimple() && "Promoting to a non-simple vector type!"); 398 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewVT)) { 399 NewOpc = ISD::FP_TO_SINT; 400 break; 401 } 402 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewVT)) { 403 NewOpc = ISD::FP_TO_UINT; 404 break; 405 } 406 } 407 408 SDLoc loc(Op); 409 SDValue promoted = DAG.getNode(NewOpc, SDLoc(Op), NewVT, Op.getOperand(0)); 410 return DAG.getNode(ISD::TRUNCATE, SDLoc(Op), VT, promoted); 411} 412 413 414SDValue VectorLegalizer::ExpandLoad(SDValue Op) { 415 SDLoc dl(Op); 416 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); 417 SDValue Chain = LD->getChain(); 418 SDValue BasePTR = LD->getBasePtr(); 419 EVT SrcVT = LD->getMemoryVT(); 420 ISD::LoadExtType ExtType = LD->getExtensionType(); 421 422 SmallVector<SDValue, 8> Vals; 423 SmallVector<SDValue, 8> LoadChains; 424 unsigned NumElem = SrcVT.getVectorNumElements(); 425 426 EVT SrcEltVT = SrcVT.getScalarType(); 427 EVT DstEltVT = Op.getNode()->getValueType(0).getScalarType(); 428 429 if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) { 430 // When elements in a vector is not byte-addressable, we cannot directly 431 // load each element by advancing pointer, which could only address bytes. 432 // Instead, we load all significant words, mask bits off, and concatenate 433 // them to form each element. Finally, they are extended to destination 434 // scalar type to build the destination vector. 435 EVT WideVT = TLI.getPointerTy(); 436 437 assert(WideVT.isRound() && 438 "Could not handle the sophisticated case when the widest integer is" 439 " not power of 2."); 440 assert(WideVT.bitsGE(SrcEltVT) && 441 "Type is not legalized?"); 442 443 unsigned WideBytes = WideVT.getStoreSize(); 444 unsigned Offset = 0; 445 unsigned RemainingBytes = SrcVT.getStoreSize(); 446 SmallVector<SDValue, 8> LoadVals; 447 448 while (RemainingBytes > 0) { 449 SDValue ScalarLoad; 450 unsigned LoadBytes = WideBytes; 451 452 if (RemainingBytes >= LoadBytes) { 453 ScalarLoad = DAG.getLoad(WideVT, dl, Chain, BasePTR, 454 LD->getPointerInfo().getWithOffset(Offset), 455 LD->isVolatile(), LD->isNonTemporal(), 456 LD->isInvariant(), LD->getAlignment(), 457 LD->getTBAAInfo()); 458 } else { 459 EVT LoadVT = WideVT; 460 while (RemainingBytes < LoadBytes) { 461 LoadBytes >>= 1; // Reduce the load size by half. 462 LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3); 463 } 464 ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR, 465 LD->getPointerInfo().getWithOffset(Offset), 466 LoadVT, LD->isVolatile(), 467 LD->isNonTemporal(), LD->getAlignment(), 468 LD->getTBAAInfo()); 469 } 470 471 RemainingBytes -= LoadBytes; 472 Offset += LoadBytes; 473 BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 474 DAG.getConstant(LoadBytes, BasePTR.getValueType())); 475 476 LoadVals.push_back(ScalarLoad.getValue(0)); 477 LoadChains.push_back(ScalarLoad.getValue(1)); 478 } 479 480 // Extract bits, pack and extend/trunc them into destination type. 481 unsigned SrcEltBits = SrcEltVT.getSizeInBits(); 482 SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, WideVT); 483 484 unsigned BitOffset = 0; 485 unsigned WideIdx = 0; 486 unsigned WideBits = WideVT.getSizeInBits(); 487 488 for (unsigned Idx = 0; Idx != NumElem; ++Idx) { 489 SDValue Lo, Hi, ShAmt; 490 491 if (BitOffset < WideBits) { 492 ShAmt = DAG.getConstant(BitOffset, TLI.getShiftAmountTy(WideVT)); 493 Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt); 494 Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask); 495 } 496 497 BitOffset += SrcEltBits; 498 if (BitOffset >= WideBits) { 499 WideIdx++; 500 Offset -= WideBits; 501 if (Offset > 0) { 502 ShAmt = DAG.getConstant(SrcEltBits - Offset, 503 TLI.getShiftAmountTy(WideVT)); 504 Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt); 505 Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask); 506 } 507 } 508 509 if (Hi.getNode()) 510 Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi); 511 512 switch (ExtType) { 513 default: llvm_unreachable("Unknown extended-load op!"); 514 case ISD::EXTLOAD: 515 Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT); 516 break; 517 case ISD::ZEXTLOAD: 518 Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT); 519 break; 520 case ISD::SEXTLOAD: 521 ShAmt = DAG.getConstant(WideBits - SrcEltBits, 522 TLI.getShiftAmountTy(WideVT)); 523 Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt); 524 Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt); 525 Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT); 526 break; 527 } 528 Vals.push_back(Lo); 529 } 530 } else { 531 unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; 532 533 for (unsigned Idx=0; Idx<NumElem; Idx++) { 534 SDValue ScalarLoad = DAG.getExtLoad(ExtType, dl, 535 Op.getNode()->getValueType(0).getScalarType(), 536 Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride), 537 SrcVT.getScalarType(), 538 LD->isVolatile(), LD->isNonTemporal(), 539 LD->getAlignment(), LD->getTBAAInfo()); 540 541 BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 542 DAG.getConstant(Stride, BasePTR.getValueType())); 543 544 Vals.push_back(ScalarLoad.getValue(0)); 545 LoadChains.push_back(ScalarLoad.getValue(1)); 546 } 547 } 548 549 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 550 &LoadChains[0], LoadChains.size()); 551 SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, 552 Op.getNode()->getValueType(0), &Vals[0], Vals.size()); 553 554 AddLegalizedOperand(Op.getValue(0), Value); 555 AddLegalizedOperand(Op.getValue(1), NewChain); 556 557 return (Op.getResNo() ? NewChain : Value); 558} 559 560SDValue VectorLegalizer::ExpandStore(SDValue Op) { 561 SDLoc dl(Op); 562 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); 563 SDValue Chain = ST->getChain(); 564 SDValue BasePTR = ST->getBasePtr(); 565 SDValue Value = ST->getValue(); 566 EVT StVT = ST->getMemoryVT(); 567 568 unsigned Alignment = ST->getAlignment(); 569 bool isVolatile = ST->isVolatile(); 570 bool isNonTemporal = ST->isNonTemporal(); 571 const MDNode *TBAAInfo = ST->getTBAAInfo(); 572 573 unsigned NumElem = StVT.getVectorNumElements(); 574 // The type of the data we want to save 575 EVT RegVT = Value.getValueType(); 576 EVT RegSclVT = RegVT.getScalarType(); 577 // The type of data as saved in memory. 578 EVT MemSclVT = StVT.getScalarType(); 579 580 // Cast floats into integers 581 unsigned ScalarSize = MemSclVT.getSizeInBits(); 582 583 // Round odd types to the next pow of two. 584 if (!isPowerOf2_32(ScalarSize)) 585 ScalarSize = NextPowerOf2(ScalarSize); 586 587 // Store Stride in bytes 588 unsigned Stride = ScalarSize/8; 589 // Extract each of the elements from the original vector 590 // and save them into memory individually. 591 SmallVector<SDValue, 8> Stores; 592 for (unsigned Idx = 0; Idx < NumElem; Idx++) { 593 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 594 RegSclVT, Value, DAG.getConstant(Idx, TLI.getVectorIdxTy())); 595 596 // This scalar TruncStore may be illegal, but we legalize it later. 597 SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR, 598 ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT, 599 isVolatile, isNonTemporal, Alignment, TBAAInfo); 600 601 BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, 602 DAG.getConstant(Stride, BasePTR.getValueType())); 603 604 Stores.push_back(Store); 605 } 606 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 607 &Stores[0], Stores.size()); 608 AddLegalizedOperand(Op, TF); 609 return TF; 610} 611 612SDValue VectorLegalizer::ExpandSELECT(SDValue Op) { 613 // Lower a select instruction where the condition is a scalar and the 614 // operands are vectors. Lower this select to VSELECT and implement it 615 // using XOR AND OR. The selector bit is broadcasted. 616 EVT VT = Op.getValueType(); 617 SDLoc DL(Op); 618 619 SDValue Mask = Op.getOperand(0); 620 SDValue Op1 = Op.getOperand(1); 621 SDValue Op2 = Op.getOperand(2); 622 623 assert(VT.isVector() && !Mask.getValueType().isVector() 624 && Op1.getValueType() == Op2.getValueType() && "Invalid type"); 625 626 unsigned NumElem = VT.getVectorNumElements(); 627 628 // If we can't even use the basic vector operations of 629 // AND,OR,XOR, we will have to scalarize the op. 630 // Notice that the operation may be 'promoted' which means that it is 631 // 'bitcasted' to another type which is handled. 632 // Also, we need to be able to construct a splat vector using BUILD_VECTOR. 633 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || 634 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || 635 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || 636 TLI.getOperationAction(ISD::BUILD_VECTOR, VT) == TargetLowering::Expand) 637 return DAG.UnrollVectorOp(Op.getNode()); 638 639 // Generate a mask operand. 640 EVT MaskTy = VT.changeVectorElementTypeToInteger(); 641 642 // What is the size of each element in the vector mask. 643 EVT BitTy = MaskTy.getScalarType(); 644 645 Mask = DAG.getSelect(DL, BitTy, Mask, 646 DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), BitTy), 647 DAG.getConstant(0, BitTy)); 648 649 // Broadcast the mask so that the entire vector is all-one or all zero. 650 SmallVector<SDValue, 8> Ops(NumElem, Mask); 651 Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskTy, &Ops[0], Ops.size()); 652 653 // Bitcast the operands to be the same type as the mask. 654 // This is needed when we select between FP types because 655 // the mask is a vector of integers. 656 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1); 657 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2); 658 659 SDValue AllOnes = DAG.getConstant( 660 APInt::getAllOnesValue(BitTy.getSizeInBits()), MaskTy); 661 SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes); 662 663 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask); 664 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask); 665 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2); 666 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val); 667} 668 669SDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) { 670 EVT VT = Op.getValueType(); 671 672 // Make sure that the SRA and SHL instructions are available. 673 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand || 674 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand) 675 return DAG.UnrollVectorOp(Op.getNode()); 676 677 SDLoc DL(Op); 678 EVT OrigTy = cast<VTSDNode>(Op->getOperand(1))->getVT(); 679 680 unsigned BW = VT.getScalarType().getSizeInBits(); 681 unsigned OrigBW = OrigTy.getScalarType().getSizeInBits(); 682 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, VT); 683 684 Op = Op.getOperand(0); 685 Op = DAG.getNode(ISD::SHL, DL, VT, Op, ShiftSz); 686 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz); 687} 688 689SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) { 690 // Implement VSELECT in terms of XOR, AND, OR 691 // on platforms which do not support blend natively. 692 SDLoc DL(Op); 693 694 SDValue Mask = Op.getOperand(0); 695 SDValue Op1 = Op.getOperand(1); 696 SDValue Op2 = Op.getOperand(2); 697 698 EVT VT = Mask.getValueType(); 699 700 // If we can't even use the basic vector operations of 701 // AND,OR,XOR, we will have to scalarize the op. 702 // Notice that the operation may be 'promoted' which means that it is 703 // 'bitcasted' to another type which is handled. 704 // This operation also isn't safe with AND, OR, XOR when the boolean 705 // type is 0/1 as we need an all ones vector constant to mask with. 706 // FIXME: Sign extend 1 to all ones if thats legal on the target. 707 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || 708 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || 709 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || 710 TLI.getBooleanContents(true) != 711 TargetLowering::ZeroOrNegativeOneBooleanContent) 712 return DAG.UnrollVectorOp(Op.getNode()); 713 714 // If the mask and the type are different sizes, unroll the vector op. This 715 // can occur when getSetCCResultType returns something that is different in 716 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8. 717 if (VT.getSizeInBits() != Op1.getValueType().getSizeInBits()) 718 return DAG.UnrollVectorOp(Op.getNode()); 719 720 // Bitcast the operands to be the same type as the mask. 721 // This is needed when we select between FP types because 722 // the mask is a vector of integers. 723 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1); 724 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2); 725 726 SDValue AllOnes = DAG.getConstant( 727 APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), VT); 728 SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes); 729 730 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask); 731 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask); 732 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2); 733 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val); 734} 735 736SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) { 737 EVT VT = Op.getOperand(0).getValueType(); 738 SDLoc DL(Op); 739 740 // Make sure that the SINT_TO_FP and SRL instructions are available. 741 if (TLI.getOperationAction(ISD::SINT_TO_FP, VT) == TargetLowering::Expand || 742 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) 743 return DAG.UnrollVectorOp(Op.getNode()); 744 745 EVT SVT = VT.getScalarType(); 746 assert((SVT.getSizeInBits() == 64 || SVT.getSizeInBits() == 32) && 747 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide"); 748 749 unsigned BW = SVT.getSizeInBits(); 750 SDValue HalfWord = DAG.getConstant(BW/2, VT); 751 752 // Constants to clear the upper part of the word. 753 // Notice that we can also use SHL+SHR, but using a constant is slightly 754 // faster on x86. 755 uint64_t HWMask = (SVT.getSizeInBits()==64)?0x00000000FFFFFFFF:0x0000FFFF; 756 SDValue HalfWordMask = DAG.getConstant(HWMask, VT); 757 758 // Two to the power of half-word-size. 759 SDValue TWOHW = DAG.getConstantFP((1<<(BW/2)), Op.getValueType()); 760 761 // Clear upper part of LO, lower HI 762 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord); 763 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), HalfWordMask); 764 765 // Convert hi and lo to floats 766 // Convert the hi part back to the upper values 767 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), HI); 768 fHI = DAG.getNode(ISD::FMUL, DL, Op.getValueType(), fHI, TWOHW); 769 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), LO); 770 771 // Add the two halves 772 return DAG.getNode(ISD::FADD, DL, Op.getValueType(), fHI, fLO); 773} 774 775 776SDValue VectorLegalizer::ExpandFNEG(SDValue Op) { 777 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) { 778 SDValue Zero = DAG.getConstantFP(-0.0, Op.getValueType()); 779 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 780 Zero, Op.getOperand(0)); 781 } 782 return DAG.UnrollVectorOp(Op.getNode()); 783} 784 785SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) { 786 EVT VT = Op.getValueType(); 787 unsigned NumElems = VT.getVectorNumElements(); 788 EVT EltVT = VT.getVectorElementType(); 789 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1), CC = Op.getOperand(2); 790 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 791 SDLoc dl(Op); 792 SmallVector<SDValue, 8> Ops(NumElems); 793 for (unsigned i = 0; i < NumElems; ++i) { 794 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 795 DAG.getConstant(i, TLI.getVectorIdxTy())); 796 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 797 DAG.getConstant(i, TLI.getVectorIdxTy())); 798 Ops[i] = DAG.getNode(ISD::SETCC, dl, 799 TLI.getSetCCResultType(*DAG.getContext(), TmpEltVT), 800 LHSElem, RHSElem, CC); 801 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], 802 DAG.getConstant(APInt::getAllOnesValue 803 (EltVT.getSizeInBits()), EltVT), 804 DAG.getConstant(0, EltVT)); 805 } 806 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElems); 807} 808 809} 810 811bool SelectionDAG::LegalizeVectors() { 812 return VectorLegalizer(*this).Run(); 813} 814