LegalizeVectorTypes.cpp revision c2c27b3627cf7a8724f2e1ec6a93b1dceea09c25
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in multiple vectors of a smaller type. For example, 19// implementing <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26using namespace llvm; 27 28//===----------------------------------------------------------------------===// 29// Result Vector Scalarization: <1 x ty> -> ty. 30//===----------------------------------------------------------------------===// 31 32void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 33 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 34 cerr << "\n"); 35 SDValue R = SDValue(); 36 37 switch (N->getOpcode()) { 38 default: 39#ifndef NDEBUG 40 cerr << "ScalarizeVectorResult #" << ResNo << ": "; 41 N->dump(&DAG); cerr << "\n"; 42#endif 43 assert(0 && "Do not know how to scalarize the result of this operator!"); 44 abort(); 45 46 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 47 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 48 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 49 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 50 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 51 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 52 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 53 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 54 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 55 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 56 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 57 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 58 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 59 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 60 61 case ISD::CTLZ: 62 case ISD::CTPOP: 63 case ISD::CTTZ: 64 case ISD::FABS: 65 case ISD::FCOS: 66 case ISD::FNEG: 67 case ISD::FP_TO_SINT: 68 case ISD::FP_TO_UINT: 69 case ISD::FSIN: 70 case ISD::FSQRT: 71 case ISD::FTRUNC: 72 case ISD::FFLOOR: 73 case ISD::FCEIL: 74 case ISD::FRINT: 75 case ISD::FNEARBYINT: 76 case ISD::UINT_TO_FP: 77 case ISD::SINT_TO_FP: 78 case ISD::TRUNCATE: 79 case ISD::SIGN_EXTEND: 80 case ISD::ZERO_EXTEND: 81 case ISD::ANY_EXTEND: 82 R = ScalarizeVecRes_UnaryOp(N); 83 break; 84 85 case ISD::ADD: 86 case ISD::AND: 87 case ISD::FADD: 88 case ISD::FDIV: 89 case ISD::FMUL: 90 case ISD::FPOW: 91 case ISD::FREM: 92 case ISD::FSUB: 93 case ISD::MUL: 94 case ISD::OR: 95 case ISD::SDIV: 96 case ISD::SREM: 97 case ISD::SUB: 98 case ISD::UDIV: 99 case ISD::UREM: 100 case ISD::XOR: 101 case ISD::SHL: 102 case ISD::SRA: 103 case ISD::SRL: 104 R = ScalarizeVecRes_BinOp(N); 105 break; 106 } 107 108 // If R is null, the sub-method took care of registering the result. 109 if (R.getNode()) 110 SetScalarizedVector(SDValue(N, ResNo), R); 111} 112 113SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 114 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 115 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 116 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 117 LHS.getValueType(), LHS, RHS); 118} 119 120SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 121 MVT NewVT = N->getValueType(0).getVectorElementType(); 122 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 123 NewVT, N->getOperand(0)); 124} 125 126SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 127 MVT NewVT = N->getValueType(0).getVectorElementType(); 128 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 129 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 130 Op0, DAG.getValueType(NewVT), 131 DAG.getValueType(Op0.getValueType()), 132 N->getOperand(3), 133 N->getOperand(4), 134 cast<CvtRndSatSDNode>(N)->getCvtCode()); 135} 136 137SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 138 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 139 N->getValueType(0).getVectorElementType(), 140 N->getOperand(0), N->getOperand(1)); 141} 142 143SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 144 SDValue Op = GetScalarizedVector(N->getOperand(0)); 145 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 146 Op.getValueType(), Op, N->getOperand(1)); 147} 148 149SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 150 // The value to insert may have a wider type than the vector element type, 151 // so be sure to truncate it to the element type if necessary. 152 SDValue Op = N->getOperand(1); 153 MVT EltVT = N->getValueType(0).getVectorElementType(); 154 if (Op.getValueType() != EltVT) 155 // FIXME: Can this happen for floating point types? 156 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 157 return Op; 158} 159 160SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 161 assert(N->isUnindexed() && "Indexed vector load?"); 162 163 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 164 N->getExtensionType(), 165 N->getValueType(0).getVectorElementType(), 166 N->getChain(), N->getBasePtr(), 167 DAG.getUNDEF(N->getBasePtr().getValueType()), 168 N->getSrcValue(), N->getSrcValueOffset(), 169 N->getMemoryVT().getVectorElementType(), 170 N->isVolatile(), N->getAlignment()); 171 172 // Legalized the chain result - switch anything that used the old chain to 173 // use the new one. 174 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 175 return Result; 176} 177 178SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 179 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 180 MVT DestVT = N->getValueType(0).getVectorElementType(); 181 SDValue Op = GetScalarizedVector(N->getOperand(0)); 182 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 183} 184 185SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 186 // If the operand is wider than the vector element type then it is implicitly 187 // truncated. Make that explicit here. 188 MVT EltVT = N->getValueType(0).getVectorElementType(); 189 SDValue InOp = N->getOperand(0); 190 if (InOp.getValueType() != EltVT) 191 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 192 return InOp; 193} 194 195SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 196 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 197 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 198 LHS.getValueType(), N->getOperand(0), LHS, 199 GetScalarizedVector(N->getOperand(2))); 200} 201 202SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 203 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 204 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 205 N->getOperand(0), N->getOperand(1), 206 LHS, GetScalarizedVector(N->getOperand(3)), 207 N->getOperand(4)); 208} 209 210SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 211 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 212} 213 214SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 215 // Figure out if the scalar is the LHS or RHS and return it. 216 SDValue Arg = N->getOperand(2).getOperand(0); 217 if (Arg.getOpcode() == ISD::UNDEF) 218 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 219 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 220 return GetScalarizedVector(N->getOperand(Op)); 221} 222 223SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 224 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 225 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 226 MVT NVT = N->getValueType(0).getVectorElementType(); 227 MVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 228 DebugLoc DL = N->getDebugLoc(); 229 230 // Turn it into a scalar SETCC. 231 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 232 233 // VSETCC always returns a sign-extended value, while SETCC may not. The 234 // SETCC result type may not match the vector element type. Correct these. 235 if (NVT.bitsLE(SVT)) { 236 // The SETCC result type is bigger than the vector element type. 237 // Ensure the SETCC result is sign-extended. 238 if (TLI.getBooleanContents() != 239 TargetLowering::ZeroOrNegativeOneBooleanContent) 240 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 241 DAG.getValueType(MVT::i1)); 242 // Truncate to the final type. 243 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 244 } 245 246 // The SETCC result type is smaller than the vector element type. 247 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 248 if (TLI.getBooleanContents() != 249 TargetLowering::ZeroOrNegativeOneBooleanContent) 250 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 251 // Sign extend to the final type. 252 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 253} 254SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 255 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 256 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 257 DebugLoc DL = N->getDebugLoc(); 258 259 // Turn it into a scalar SETCC. 260 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 261} 262 263 264//===----------------------------------------------------------------------===// 265// Operand Vector Scalarization <1 x ty> -> ty. 266//===----------------------------------------------------------------------===// 267 268bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 269 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 270 cerr << "\n"); 271 SDValue Res = SDValue(); 272 273 if (Res.getNode() == 0) { 274 switch (N->getOpcode()) { 275 default: 276#ifndef NDEBUG 277 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": "; 278 N->dump(&DAG); cerr << "\n"; 279#endif 280 assert(0 && "Do not know how to scalarize this operator's operand!"); 281 case ISD::BIT_CONVERT: 282 Res = ScalarizeVecOp_BIT_CONVERT(N); 283 break; 284 case ISD::CONCAT_VECTORS: 285 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 286 break; 287 case ISD::EXTRACT_VECTOR_ELT: 288 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 289 break; 290 case ISD::STORE: 291 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 292 break; 293 } 294 } 295 296 // If the result is null, the sub-method took care of registering results etc. 297 if (!Res.getNode()) return false; 298 299 // If the result is N, the sub-method updated N in place. Tell the legalizer 300 // core about this. 301 if (Res.getNode() == N) 302 return true; 303 304 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 305 "Invalid operand expansion"); 306 307 ReplaceValueWith(SDValue(N, 0), Res); 308 return false; 309} 310 311/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 312/// to be scalarized, it must be <1 x ty>. Convert the element instead. 313SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 314 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 315 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 316 N->getValueType(0), Elt); 317} 318 319/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 320/// use a BUILD_VECTOR instead. 321SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 322 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 323 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 324 Ops[i] = GetScalarizedVector(N->getOperand(i)); 325 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 326 &Ops[0], Ops.size()); 327} 328 329/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 330/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 331/// index. 332SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 333 return GetScalarizedVector(N->getOperand(0)); 334} 335 336/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 337/// scalarized, it must be <1 x ty>. Just store the element. 338SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 339 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 340 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 341 DebugLoc dl = N->getDebugLoc(); 342 343 if (N->isTruncatingStore()) 344 return DAG.getTruncStore(N->getChain(), dl, 345 GetScalarizedVector(N->getOperand(1)), 346 N->getBasePtr(), 347 N->getSrcValue(), N->getSrcValueOffset(), 348 N->getMemoryVT().getVectorElementType(), 349 N->isVolatile(), N->getAlignment()); 350 351 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 352 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 353 N->isVolatile(), N->getAlignment()); 354} 355 356 357//===----------------------------------------------------------------------===// 358// Result Vector Splitting 359//===----------------------------------------------------------------------===// 360 361/// SplitVectorResult - This method is called when the specified result of the 362/// specified node is found to need vector splitting. At this point, the node 363/// may also have invalid operands or may have other results that need 364/// legalization, we just know that (at least) one result needs vector 365/// splitting. 366void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 367 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n"); 368 SDValue Lo, Hi; 369 370 switch (N->getOpcode()) { 371 default: 372#ifndef NDEBUG 373 cerr << "SplitVectorResult #" << ResNo << ": "; 374 N->dump(&DAG); cerr << "\n"; 375#endif 376 assert(0 && "Do not know how to split the result of this operator!"); 377 abort(); 378 379 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 380 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 381 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 382 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 383 384 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 385 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 386 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 387 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 388 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 389 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 390 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 391 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 392 case ISD::LOAD: 393 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 394 break; 395 case ISD::VECTOR_SHUFFLE: 396 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 397 break; 398 399 case ISD::VSETCC: 400 case ISD::SETCC: 401 SplitVecRes_SETCC(N, Lo, Hi); 402 break; 403 404 case ISD::CTTZ: 405 case ISD::CTLZ: 406 case ISD::CTPOP: 407 case ISD::FNEG: 408 case ISD::FABS: 409 case ISD::FSQRT: 410 case ISD::FSIN: 411 case ISD::FCOS: 412 case ISD::FTRUNC: 413 case ISD::FFLOOR: 414 case ISD::FCEIL: 415 case ISD::FRINT: 416 case ISD::FNEARBYINT: 417 case ISD::FP_TO_SINT: 418 case ISD::FP_TO_UINT: 419 case ISD::SINT_TO_FP: 420 case ISD::UINT_TO_FP: 421 case ISD::TRUNCATE: 422 case ISD::SIGN_EXTEND: 423 case ISD::ZERO_EXTEND: 424 case ISD::ANY_EXTEND: 425 SplitVecRes_UnaryOp(N, Lo, Hi); 426 break; 427 428 case ISD::ADD: 429 case ISD::SUB: 430 case ISD::MUL: 431 case ISD::FADD: 432 case ISD::FSUB: 433 case ISD::FMUL: 434 case ISD::SDIV: 435 case ISD::UDIV: 436 case ISD::FDIV: 437 case ISD::FPOW: 438 case ISD::AND: 439 case ISD::OR: 440 case ISD::XOR: 441 case ISD::SHL: 442 case ISD::SRA: 443 case ISD::SRL: 444 case ISD::UREM: 445 case ISD::SREM: 446 case ISD::FREM: 447 SplitVecRes_BinOp(N, Lo, Hi); 448 break; 449 } 450 451 // If Lo/Hi is null, the sub-method took care of registering results etc. 452 if (Lo.getNode()) 453 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 454} 455 456void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 457 SDValue &Hi) { 458 SDValue LHSLo, LHSHi; 459 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 460 SDValue RHSLo, RHSHi; 461 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 462 DebugLoc dl = N->getDebugLoc(); 463 464 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 465 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 466} 467 468void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 469 SDValue &Hi) { 470 // We know the result is a vector. The input may be either a vector or a 471 // scalar value. 472 MVT LoVT, HiVT; 473 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 474 DebugLoc dl = N->getDebugLoc(); 475 476 SDValue InOp = N->getOperand(0); 477 MVT InVT = InOp.getValueType(); 478 479 // Handle some special cases efficiently. 480 switch (getTypeAction(InVT)) { 481 default: 482 assert(false && "Unknown type action!"); 483 case Legal: 484 case PromoteInteger: 485 case SoftenFloat: 486 case ScalarizeVector: 487 break; 488 case ExpandInteger: 489 case ExpandFloat: 490 // A scalar to vector conversion, where the scalar needs expansion. 491 // If the vector is being split in two then we can just convert the 492 // expanded pieces. 493 if (LoVT == HiVT) { 494 GetExpandedOp(InOp, Lo, Hi); 495 if (TLI.isBigEndian()) 496 std::swap(Lo, Hi); 497 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 498 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 499 return; 500 } 501 break; 502 case SplitVector: 503 // If the input is a vector that needs to be split, convert each split 504 // piece of the input now. 505 GetSplitVector(InOp, Lo, Hi); 506 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 507 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 508 return; 509 } 510 511 // In the general case, convert the input to an integer and split it by hand. 512 MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits()); 513 MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits()); 514 if (TLI.isBigEndian()) 515 std::swap(LoIntVT, HiIntVT); 516 517 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 518 519 if (TLI.isBigEndian()) 520 std::swap(Lo, Hi); 521 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 522 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 523} 524 525void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 526 SDValue &Hi) { 527 MVT LoVT, HiVT; 528 DebugLoc dl = N->getDebugLoc(); 529 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 530 unsigned LoNumElts = LoVT.getVectorNumElements(); 531 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 532 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 533 534 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 535 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 536} 537 538void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 539 SDValue &Hi) { 540 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 541 DebugLoc dl = N->getDebugLoc(); 542 unsigned NumSubvectors = N->getNumOperands() / 2; 543 if (NumSubvectors == 1) { 544 Lo = N->getOperand(0); 545 Hi = N->getOperand(1); 546 return; 547 } 548 549 MVT LoVT, HiVT; 550 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 551 552 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 553 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 554 555 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 556 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 557} 558 559void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 560 SDValue &Hi) { 561 MVT LoVT, HiVT; 562 DebugLoc dl = N->getDebugLoc(); 563 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 564 565 SDValue DTyOpLo = DAG.getValueType(LoVT); 566 SDValue DTyOpHi = DAG.getValueType(HiVT); 567 568 SDValue RndOp = N->getOperand(3); 569 SDValue SatOp = N->getOperand(4); 570 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 571 572 // Split the input. 573 SDValue VLo, VHi; 574 MVT InVT = N->getOperand(0).getValueType(); 575 switch (getTypeAction(InVT)) { 576 default: assert(0 && "Unexpected type action!"); 577 case Legal: { 578 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 579 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 580 LoVT.getVectorNumElements()); 581 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 582 DAG.getIntPtrConstant(0)); 583 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 584 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 585 break; 586 } 587 case SplitVector: 588 GetSplitVector(N->getOperand(0), VLo, VHi); 589 break; 590 case WidenVector: { 591 // If the result needs to be split and the input needs to be widened, 592 // the two types must have different lengths. Use the widened result 593 // and extract from it to do the split. 594 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 595 SDValue InOp = GetWidenedVector(N->getOperand(0)); 596 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 597 LoVT.getVectorNumElements()); 598 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 599 DAG.getIntPtrConstant(0)); 600 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 601 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 602 break; 603 } 604 } 605 606 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 607 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 608 609 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 610 CvtCode); 611 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 612 CvtCode); 613} 614 615void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 616 SDValue &Hi) { 617 SDValue Vec = N->getOperand(0); 618 SDValue Idx = N->getOperand(1); 619 MVT IdxVT = Idx.getValueType(); 620 DebugLoc dl = N->getDebugLoc(); 621 622 MVT LoVT, HiVT; 623 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 624 // The indices are not guaranteed to be a multiple of the new vector 625 // size unless the original vector type was split in two. 626 assert(LoVT == HiVT && "Non power-of-two vectors not supported!"); 627 628 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 629 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 630 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 631 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 632} 633 634void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 635 SDValue &Hi) { 636 DebugLoc dl = N->getDebugLoc(); 637 GetSplitVector(N->getOperand(0), Lo, Hi); 638 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 639 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 640} 641 642void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 643 SDValue &Hi) { 644 SDValue Vec = N->getOperand(0); 645 SDValue Elt = N->getOperand(1); 646 SDValue Idx = N->getOperand(2); 647 DebugLoc dl = N->getDebugLoc(); 648 GetSplitVector(Vec, Lo, Hi); 649 650 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 651 unsigned IdxVal = CIdx->getZExtValue(); 652 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 653 if (IdxVal < LoNumElts) 654 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 655 Lo.getValueType(), Lo, Elt, Idx); 656 else 657 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 658 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 659 return; 660 } 661 662 // Spill the vector to the stack. 663 MVT VecVT = Vec.getValueType(); 664 MVT EltVT = VecVT.getVectorElementType(); 665 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 666 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 667 668 // Store the new element. This may be larger than the vector element type, 669 // so use a truncating store. 670 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 671 unsigned Alignment = 672 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForMVT()); 673 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 674 675 // Load the Lo part from the stack slot. 676 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 677 678 // Increment the pointer to the other part. 679 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 680 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 681 DAG.getIntPtrConstant(IncrementSize)); 682 683 // Load the Hi part from the stack slot. 684 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 685 MinAlign(Alignment, IncrementSize)); 686} 687 688void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 689 SDValue &Hi) { 690 MVT LoVT, HiVT; 691 DebugLoc dl = N->getDebugLoc(); 692 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 693 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 694 Hi = DAG.getUNDEF(HiVT); 695} 696 697void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 698 SDValue &Hi) { 699 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 700 MVT LoVT, HiVT; 701 DebugLoc dl = LD->getDebugLoc(); 702 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 703 704 ISD::LoadExtType ExtType = LD->getExtensionType(); 705 SDValue Ch = LD->getChain(); 706 SDValue Ptr = LD->getBasePtr(); 707 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 708 const Value *SV = LD->getSrcValue(); 709 int SVOffset = LD->getSrcValueOffset(); 710 MVT MemoryVT = LD->getMemoryVT(); 711 unsigned Alignment = LD->getAlignment(); 712 bool isVolatile = LD->isVolatile(); 713 714 MVT LoMemVT, HiMemVT; 715 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 716 717 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 718 SV, SVOffset, LoMemVT, isVolatile, Alignment); 719 720 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 721 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 722 DAG.getIntPtrConstant(IncrementSize)); 723 SVOffset += IncrementSize; 724 Alignment = MinAlign(Alignment, IncrementSize); 725 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 726 SV, SVOffset, HiMemVT, isVolatile, Alignment); 727 728 // Build a factor node to remember that this load is independent of the 729 // other one. 730 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 731 Hi.getValue(1)); 732 733 // Legalized the chain result - switch anything that used the old chain to 734 // use the new one. 735 ReplaceValueWith(SDValue(LD, 1), Ch); 736} 737 738void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 739 SDValue &Hi) { 740 // Get the dest types - they may not match the input types, e.g. int_to_fp. 741 MVT LoVT, HiVT; 742 DebugLoc dl = N->getDebugLoc(); 743 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 744 745 // Split the input. 746 MVT InVT = N->getOperand(0).getValueType(); 747 switch (getTypeAction(InVT)) { 748 default: assert(0 && "Unexpected type action!"); 749 case Legal: { 750 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 751 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 752 LoVT.getVectorNumElements()); 753 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 754 DAG.getIntPtrConstant(0)); 755 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 756 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 757 break; 758 } 759 case SplitVector: 760 GetSplitVector(N->getOperand(0), Lo, Hi); 761 break; 762 case WidenVector: { 763 // If the result needs to be split and the input needs to be widened, 764 // the two types must have different lengths. Use the widened result 765 // and extract from it to do the split. 766 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 767 SDValue InOp = GetWidenedVector(N->getOperand(0)); 768 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 769 LoVT.getVectorNumElements()); 770 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 771 DAG.getIntPtrConstant(0)); 772 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 773 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 774 break; 775 } 776 } 777 778 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 779 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 780} 781 782void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 783 SDValue &Lo, SDValue &Hi) { 784 // The low and high parts of the original input give four input vectors. 785 SDValue Inputs[4]; 786 DebugLoc dl = N->getDebugLoc(); 787 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 788 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 789 MVT NewVT = Inputs[0].getValueType(); 790 unsigned NewElts = NewVT.getVectorNumElements(); 791 assert(NewVT == Inputs[1].getValueType() && 792 "Non power-of-two vectors not supported!"); 793 794 // If Lo or Hi uses elements from at most two of the four input vectors, then 795 // express it as a vector shuffle of those two inputs. Otherwise extract the 796 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 797 SmallVector<int, 16> Ops; 798 for (unsigned High = 0; High < 2; ++High) { 799 SDValue &Output = High ? Hi : Lo; 800 801 // Build a shuffle mask for the output, discovering on the fly which 802 // input vectors to use as shuffle operands (recorded in InputUsed). 803 // If building a suitable shuffle vector proves too hard, then bail 804 // out with useBuildVector set. 805 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 806 unsigned FirstMaskIdx = High * NewElts; 807 bool useBuildVector = false; 808 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 809 // The mask element. This indexes into the input. 810 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 811 812 // The input vector this mask element indexes into. 813 unsigned Input = (unsigned)Idx / NewElts; 814 815 if (Input >= array_lengthof(Inputs)) { 816 // The mask element does not index into any input vector. 817 Ops.push_back(-1); 818 continue; 819 } 820 821 // Turn the index into an offset from the start of the input vector. 822 Idx -= Input * NewElts; 823 824 // Find or create a shuffle vector operand to hold this input. 825 unsigned OpNo; 826 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 827 if (InputUsed[OpNo] == Input) { 828 // This input vector is already an operand. 829 break; 830 } else if (InputUsed[OpNo] == -1U) { 831 // Create a new operand for this input vector. 832 InputUsed[OpNo] = Input; 833 break; 834 } 835 } 836 837 if (OpNo >= array_lengthof(InputUsed)) { 838 // More than two input vectors used! Give up on trying to create a 839 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 840 useBuildVector = true; 841 break; 842 } 843 844 // Add the mask index for the new shuffle vector. 845 Ops.push_back(Idx + OpNo * NewElts); 846 } 847 848 if (useBuildVector) { 849 MVT EltVT = NewVT.getVectorElementType(); 850 SmallVector<SDValue, 16> SVOps; 851 852 // Extract the input elements by hand. 853 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 854 // The mask element. This indexes into the input. 855 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 856 857 // The input vector this mask element indexes into. 858 unsigned Input = (unsigned)Idx / NewElts; 859 860 if (Input >= array_lengthof(Inputs)) { 861 // The mask element is "undef" or indexes off the end of the input. 862 SVOps.push_back(DAG.getUNDEF(EltVT)); 863 continue; 864 } 865 866 // Turn the index into an offset from the start of the input vector. 867 Idx -= Input * NewElts; 868 869 // Extract the vector element by hand. 870 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 871 Inputs[Input], DAG.getIntPtrConstant(Idx))); 872 } 873 874 // Construct the Lo/Hi output using a BUILD_VECTOR. 875 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 876 } else if (InputUsed[0] == -1U) { 877 // No input vectors were used! The result is undefined. 878 Output = DAG.getUNDEF(NewVT); 879 } else { 880 SDValue Op0 = Inputs[InputUsed[0]]; 881 // If only one input was used, use an undefined vector for the other. 882 SDValue Op1 = InputUsed[1] == -1U ? 883 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 884 // At least one input vector was used. Create a new shuffle vector. 885 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 886 } 887 888 Ops.clear(); 889 } 890} 891 892void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 893 MVT LoVT, HiVT; 894 DebugLoc DL = N->getDebugLoc(); 895 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 896 897 // Split the input. 898 MVT InVT = N->getOperand(0).getValueType(); 899 SDValue LL, LH, RL, RH; 900 switch (getTypeAction(InVT)) { 901 default: assert(0 && "Unexpected type action!"); 902 case WidenVector: assert(0 && "Unimp"); 903 case Legal: { 904 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 905 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 906 LoVT.getVectorNumElements()); 907 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 908 DAG.getIntPtrConstant(0)); 909 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 910 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 911 912 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 913 DAG.getIntPtrConstant(0)); 914 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 915 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 916 break; 917 } 918 case SplitVector: 919 GetSplitVector(N->getOperand(0), LL, LH); 920 GetSplitVector(N->getOperand(1), RL, RH); 921 break; 922 } 923 924 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 925 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 926} 927 928//===----------------------------------------------------------------------===// 929// Operand Vector Splitting 930//===----------------------------------------------------------------------===// 931 932/// SplitVectorOperand - This method is called when the specified operand of the 933/// specified node is found to need vector splitting. At this point, all of the 934/// result types of the node are known to be legal, but other operands of the 935/// node may need legalization as well as the specified one. 936bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 937 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n"); 938 SDValue Res = SDValue(); 939 940 if (Res.getNode() == 0) { 941 switch (N->getOpcode()) { 942 default: 943#ifndef NDEBUG 944 cerr << "SplitVectorOperand Op #" << OpNo << ": "; 945 N->dump(&DAG); cerr << "\n"; 946#endif 947 assert(0 && "Do not know how to split this operator's operand!"); 948 abort(); 949 950 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 951 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 952 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 953 case ISD::STORE: 954 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 955 break; 956 957 case ISD::CTTZ: 958 case ISD::CTLZ: 959 case ISD::CTPOP: 960 case ISD::FP_TO_SINT: 961 case ISD::FP_TO_UINT: 962 case ISD::SINT_TO_FP: 963 case ISD::UINT_TO_FP: 964 case ISD::TRUNCATE: 965 case ISD::SIGN_EXTEND: 966 case ISD::ZERO_EXTEND: 967 case ISD::ANY_EXTEND: 968 Res = SplitVecOp_UnaryOp(N); 969 break; 970 } 971 } 972 973 // If the result is null, the sub-method took care of registering results etc. 974 if (!Res.getNode()) return false; 975 976 // If the result is N, the sub-method updated N in place. Tell the legalizer 977 // core about this. 978 if (Res.getNode() == N) 979 return true; 980 981 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 982 "Invalid operand expansion"); 983 984 ReplaceValueWith(SDValue(N, 0), Res); 985 return false; 986} 987 988SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 989 // The result has a legal vector type, but the input needs splitting. 990 MVT ResVT = N->getValueType(0); 991 SDValue Lo, Hi; 992 DebugLoc dl = N->getDebugLoc(); 993 GetSplitVector(N->getOperand(0), Lo, Hi); 994 assert(Lo.getValueType() == Hi.getValueType() && 995 "Returns legal non-power-of-two vector type?"); 996 MVT InVT = Lo.getValueType(); 997 998 MVT OutVT = MVT::getVectorVT(ResVT.getVectorElementType(), 999 InVT.getVectorNumElements()); 1000 1001 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1002 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1003 1004 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1005} 1006 1007SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 1008 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 1009 // end up being split all the way down to individual components. Convert the 1010 // split pieces into integers and reassemble. 1011 SDValue Lo, Hi; 1012 GetSplitVector(N->getOperand(0), Lo, Hi); 1013 Lo = BitConvertToInteger(Lo); 1014 Hi = BitConvertToInteger(Hi); 1015 1016 if (TLI.isBigEndian()) 1017 std::swap(Lo, Hi); 1018 1019 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 1020 JoinIntegers(Lo, Hi)); 1021} 1022 1023SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1024 // We know that the extracted result type is legal. For now, assume the index 1025 // is a constant. 1026 MVT SubVT = N->getValueType(0); 1027 SDValue Idx = N->getOperand(1); 1028 DebugLoc dl = N->getDebugLoc(); 1029 SDValue Lo, Hi; 1030 GetSplitVector(N->getOperand(0), Lo, Hi); 1031 1032 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1033 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1034 1035 if (IdxVal < LoElts) { 1036 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1037 "Extracted subvector crosses vector split!"); 1038 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1039 } else { 1040 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1041 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1042 } 1043} 1044 1045SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1046 SDValue Vec = N->getOperand(0); 1047 SDValue Idx = N->getOperand(1); 1048 MVT VecVT = Vec.getValueType(); 1049 1050 if (isa<ConstantSDNode>(Idx)) { 1051 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1052 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1053 1054 SDValue Lo, Hi; 1055 GetSplitVector(Vec, Lo, Hi); 1056 1057 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1058 1059 if (IdxVal < LoElts) 1060 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1061 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1062 DAG.getConstant(IdxVal - LoElts, 1063 Idx.getValueType())); 1064 } 1065 1066 // Store the vector to the stack. 1067 MVT EltVT = VecVT.getVectorElementType(); 1068 DebugLoc dl = N->getDebugLoc(); 1069 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1070 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1071 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1072 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1073 1074 // Load back the required element. 1075 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1076 return DAG.getLoad(EltVT, dl, Store, StackPtr, SV, 0); 1077} 1078 1079SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1080 assert(N->isUnindexed() && "Indexed store of vector?"); 1081 assert(OpNo == 1 && "Can only split the stored value"); 1082 DebugLoc dl = N->getDebugLoc(); 1083 1084 bool isTruncating = N->isTruncatingStore(); 1085 SDValue Ch = N->getChain(); 1086 SDValue Ptr = N->getBasePtr(); 1087 int SVOffset = N->getSrcValueOffset(); 1088 MVT MemoryVT = N->getMemoryVT(); 1089 unsigned Alignment = N->getAlignment(); 1090 bool isVol = N->isVolatile(); 1091 SDValue Lo, Hi; 1092 GetSplitVector(N->getOperand(1), Lo, Hi); 1093 1094 MVT LoMemVT, HiMemVT; 1095 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1096 1097 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1098 1099 if (isTruncating) 1100 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1101 LoMemVT, isVol, Alignment); 1102 else 1103 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1104 isVol, Alignment); 1105 1106 // Increment the pointer to the other half. 1107 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1108 DAG.getIntPtrConstant(IncrementSize)); 1109 1110 if (isTruncating) 1111 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 1112 N->getSrcValue(), SVOffset+IncrementSize, 1113 HiMemVT, 1114 isVol, MinAlign(Alignment, IncrementSize)); 1115 else 1116 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 1117 isVol, MinAlign(Alignment, IncrementSize)); 1118 1119 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1120} 1121 1122 1123//===----------------------------------------------------------------------===// 1124// Result Vector Widening 1125//===----------------------------------------------------------------------===// 1126 1127void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1128 DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG); 1129 cerr << "\n"); 1130 SDValue Res = SDValue(); 1131 1132 switch (N->getOpcode()) { 1133 default: 1134#ifndef NDEBUG 1135 cerr << "WidenVectorResult #" << ResNo << ": "; 1136 N->dump(&DAG); cerr << "\n"; 1137#endif 1138 assert(0 && "Do not know how to widen the result of this operator!"); 1139 abort(); 1140 1141 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1142 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1143 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1144 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1145 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1146 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1147 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1148 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1149 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1150 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1151 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1152 case ISD::VECTOR_SHUFFLE: 1153 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1154 break; 1155 case ISD::VSETCC: 1156 Res = WidenVecRes_VSETCC(N); 1157 break; 1158 1159 case ISD::ADD: 1160 case ISD::AND: 1161 case ISD::BSWAP: 1162 case ISD::FADD: 1163 case ISD::FCOPYSIGN: 1164 case ISD::FDIV: 1165 case ISD::FMUL: 1166 case ISD::FPOW: 1167 case ISD::FPOWI: 1168 case ISD::FREM: 1169 case ISD::FSUB: 1170 case ISD::MUL: 1171 case ISD::MULHS: 1172 case ISD::MULHU: 1173 case ISD::OR: 1174 case ISD::SDIV: 1175 case ISD::SREM: 1176 case ISD::UDIV: 1177 case ISD::UREM: 1178 case ISD::SUB: 1179 case ISD::XOR: 1180 Res = WidenVecRes_Binary(N); 1181 break; 1182 1183 case ISD::SHL: 1184 case ISD::SRA: 1185 case ISD::SRL: 1186 Res = WidenVecRes_Shift(N); 1187 break; 1188 1189 case ISD::FP_ROUND: 1190 case ISD::FP_TO_SINT: 1191 case ISD::FP_TO_UINT: 1192 case ISD::SINT_TO_FP: 1193 case ISD::UINT_TO_FP: 1194 case ISD::TRUNCATE: 1195 case ISD::SIGN_EXTEND: 1196 case ISD::ZERO_EXTEND: 1197 case ISD::ANY_EXTEND: 1198 Res = WidenVecRes_Convert(N); 1199 break; 1200 1201 case ISD::CTLZ: 1202 case ISD::CTPOP: 1203 case ISD::CTTZ: 1204 case ISD::FABS: 1205 case ISD::FCOS: 1206 case ISD::FNEG: 1207 case ISD::FSIN: 1208 case ISD::FSQRT: 1209 Res = WidenVecRes_Unary(N); 1210 break; 1211 } 1212 1213 // If Res is null, the sub-method took care of registering the result. 1214 if (Res.getNode()) 1215 SetWidenedVector(SDValue(N, ResNo), Res); 1216} 1217 1218SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1219 // Binary op widening. 1220 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1221 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1222 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1223 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1224} 1225 1226SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1227 SDValue InOp = N->getOperand(0); 1228 DebugLoc dl = N->getDebugLoc(); 1229 1230 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1231 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1232 1233 MVT InVT = InOp.getValueType(); 1234 MVT InEltVT = InVT.getVectorElementType(); 1235 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts); 1236 1237 unsigned Opcode = N->getOpcode(); 1238 unsigned InVTNumElts = InVT.getVectorNumElements(); 1239 1240 if (getTypeAction(InVT) == WidenVector) { 1241 InOp = GetWidenedVector(N->getOperand(0)); 1242 InVT = InOp.getValueType(); 1243 InVTNumElts = InVT.getVectorNumElements(); 1244 if (InVTNumElts == WidenNumElts) 1245 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1246 } 1247 1248 if (TLI.isTypeLegal(InWidenVT)) { 1249 // Because the result and the input are different vector types, widening 1250 // the result could create a legal type but widening the input might make 1251 // it an illegal type that might lead to repeatedly splitting the input 1252 // and then widening it. To avoid this, we widen the input only if 1253 // it results in a legal type. 1254 if (WidenNumElts % InVTNumElts == 0) { 1255 // Widen the input and call convert on the widened input vector. 1256 unsigned NumConcat = WidenNumElts/InVTNumElts; 1257 SmallVector<SDValue, 16> Ops(NumConcat); 1258 Ops[0] = InOp; 1259 SDValue UndefVal = DAG.getUNDEF(InVT); 1260 for (unsigned i = 1; i != NumConcat; ++i) 1261 Ops[i] = UndefVal; 1262 return DAG.getNode(Opcode, dl, WidenVT, 1263 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1264 &Ops[0], NumConcat)); 1265 } 1266 1267 if (InVTNumElts % WidenNumElts == 0) { 1268 // Extract the input and convert the shorten input vector. 1269 return DAG.getNode(Opcode, dl, WidenVT, 1270 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1271 InOp, DAG.getIntPtrConstant(0))); 1272 } 1273 } 1274 1275 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1276 SmallVector<SDValue, 16> Ops(WidenNumElts); 1277 MVT EltVT = WidenVT.getVectorElementType(); 1278 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1279 unsigned i; 1280 for (i=0; i < MinElts; ++i) 1281 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1282 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1283 DAG.getIntPtrConstant(i))); 1284 1285 SDValue UndefVal = DAG.getUNDEF(EltVT); 1286 for (; i < WidenNumElts; ++i) 1287 Ops[i] = UndefVal; 1288 1289 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1290} 1291 1292SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1293 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1294 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1295 SDValue ShOp = N->getOperand(1); 1296 1297 MVT ShVT = ShOp.getValueType(); 1298 if (getTypeAction(ShVT) == WidenVector) { 1299 ShOp = GetWidenedVector(ShOp); 1300 ShVT = ShOp.getValueType(); 1301 } 1302 MVT ShWidenVT = MVT::getVectorVT(ShVT.getVectorElementType(), 1303 WidenVT.getVectorNumElements()); 1304 if (ShVT != ShWidenVT) 1305 ShOp = ModifyToType(ShOp, ShWidenVT); 1306 1307 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1308} 1309 1310SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1311 // Unary op widening. 1312 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1313 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1314 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1315} 1316 1317SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1318 SDValue InOp = N->getOperand(0); 1319 MVT InVT = InOp.getValueType(); 1320 MVT VT = N->getValueType(0); 1321 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1322 DebugLoc dl = N->getDebugLoc(); 1323 1324 switch (getTypeAction(InVT)) { 1325 default: 1326 assert(false && "Unknown type action!"); 1327 break; 1328 case Legal: 1329 break; 1330 case PromoteInteger: 1331 // If the InOp is promoted to the same size, convert it. Otherwise, 1332 // fall out of the switch and widen the promoted input. 1333 InOp = GetPromotedInteger(InOp); 1334 InVT = InOp.getValueType(); 1335 if (WidenVT.bitsEq(InVT)) 1336 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1337 break; 1338 case SoftenFloat: 1339 case ExpandInteger: 1340 case ExpandFloat: 1341 case ScalarizeVector: 1342 case SplitVector: 1343 break; 1344 case WidenVector: 1345 // If the InOp is widened to the same size, convert it. Otherwise, fall 1346 // out of the switch and widen the widened input. 1347 InOp = GetWidenedVector(InOp); 1348 InVT = InOp.getValueType(); 1349 if (WidenVT.bitsEq(InVT)) 1350 // The input widens to the same size. Convert to the widen value. 1351 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1352 break; 1353 } 1354 1355 unsigned WidenSize = WidenVT.getSizeInBits(); 1356 unsigned InSize = InVT.getSizeInBits(); 1357 if (WidenSize % InSize == 0) { 1358 // Determine new input vector type. The new input vector type will use 1359 // the same element type (if its a vector) or use the input type as a 1360 // vector. It is the same size as the type to widen to. 1361 MVT NewInVT; 1362 unsigned NewNumElts = WidenSize / InSize; 1363 if (InVT.isVector()) { 1364 MVT InEltVT = InVT.getVectorElementType(); 1365 NewInVT= MVT::getVectorVT(InEltVT, WidenSize / InEltVT.getSizeInBits()); 1366 } else { 1367 NewInVT = MVT::getVectorVT(InVT, NewNumElts); 1368 } 1369 1370 if (TLI.isTypeLegal(NewInVT)) { 1371 // Because the result and the input are different vector types, widening 1372 // the result could create a legal type but widening the input might make 1373 // it an illegal type that might lead to repeatedly splitting the input 1374 // and then widening it. To avoid this, we widen the input only if 1375 // it results in a legal type. 1376 SmallVector<SDValue, 16> Ops(NewNumElts); 1377 SDValue UndefVal = DAG.getUNDEF(InVT); 1378 Ops[0] = InOp; 1379 for (unsigned i = 1; i < NewNumElts; ++i) 1380 Ops[i] = UndefVal; 1381 1382 SDValue NewVec; 1383 if (InVT.isVector()) 1384 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1385 NewInVT, &Ops[0], NewNumElts); 1386 else 1387 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1388 NewInVT, &Ops[0], NewNumElts); 1389 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1390 } 1391 } 1392 1393 // This should occur rarely. Lower the bit-convert to a store/load 1394 // from the stack. Create the stack frame object. Make sure it is aligned 1395 // for both the source and destination types. 1396 SDValue FIPtr = DAG.CreateStackTemporary(InVT, WidenVT); 1397 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1398 const Value *SV = PseudoSourceValue::getFixedStack(FI); 1399 1400 // Emit a store to the stack slot. 1401 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0); 1402 1403 // Result is a load from the stack slot. 1404 return DAG.getLoad(WidenVT, dl, Store, FIPtr, SV, 0); 1405} 1406 1407SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1408 DebugLoc dl = N->getDebugLoc(); 1409 // Build a vector with undefined for the new nodes. 1410 MVT VT = N->getValueType(0); 1411 MVT EltVT = VT.getVectorElementType(); 1412 unsigned NumElts = VT.getVectorNumElements(); 1413 1414 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1415 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1416 1417 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1418 NewOps.reserve(WidenNumElts); 1419 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1420 NewOps.push_back(DAG.getUNDEF(EltVT)); 1421 1422 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1423} 1424 1425SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1426 MVT InVT = N->getOperand(0).getValueType(); 1427 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1428 DebugLoc dl = N->getDebugLoc(); 1429 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1430 unsigned NumOperands = N->getNumOperands(); 1431 1432 bool InputWidened = false; // Indicates we need to widen the input. 1433 if (getTypeAction(InVT) != WidenVector) { 1434 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1435 // Add undef vectors to widen to correct length. 1436 unsigned NumConcat = WidenVT.getVectorNumElements() / 1437 InVT.getVectorNumElements(); 1438 SDValue UndefVal = DAG.getUNDEF(InVT); 1439 SmallVector<SDValue, 16> Ops(NumConcat); 1440 for (unsigned i=0; i < NumOperands; ++i) 1441 Ops[i] = N->getOperand(i); 1442 for (unsigned i = NumOperands; i != NumConcat; ++i) 1443 Ops[i] = UndefVal; 1444 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1445 } 1446 } else { 1447 InputWidened = true; 1448 if (WidenVT == TLI.getTypeToTransformTo(InVT)) { 1449 // The inputs and the result are widen to the same value. 1450 unsigned i; 1451 for (i=1; i < NumOperands; ++i) 1452 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1453 break; 1454 1455 if (i > NumOperands) 1456 // Everything but the first operand is an UNDEF so just return the 1457 // widened first operand. 1458 return GetWidenedVector(N->getOperand(0)); 1459 1460 if (NumOperands == 2) { 1461 // Replace concat of two operands with a shuffle. 1462 SmallVector<int, 16> MaskOps(WidenNumElts); 1463 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1464 MaskOps[i] = i; 1465 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1466 } 1467 return DAG.getVectorShuffle(WidenVT, dl, 1468 GetWidenedVector(N->getOperand(0)), 1469 GetWidenedVector(N->getOperand(1)), 1470 &MaskOps[0]); 1471 } 1472 } 1473 } 1474 1475 // Fall back to use extracts and build vector. 1476 MVT EltVT = WidenVT.getVectorElementType(); 1477 unsigned NumInElts = InVT.getVectorNumElements(); 1478 SmallVector<SDValue, 16> Ops(WidenNumElts); 1479 unsigned Idx = 0; 1480 for (unsigned i=0; i < NumOperands; ++i) { 1481 SDValue InOp = N->getOperand(i); 1482 if (InputWidened) 1483 InOp = GetWidenedVector(InOp); 1484 for (unsigned j=0; j < NumInElts; ++j) 1485 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1486 DAG.getIntPtrConstant(j)); 1487 } 1488 SDValue UndefVal = DAG.getUNDEF(EltVT); 1489 for (; Idx < WidenNumElts; ++Idx) 1490 Ops[Idx] = UndefVal; 1491 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1492} 1493 1494SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1495 DebugLoc dl = N->getDebugLoc(); 1496 SDValue InOp = N->getOperand(0); 1497 SDValue RndOp = N->getOperand(3); 1498 SDValue SatOp = N->getOperand(4); 1499 1500 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1501 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1502 1503 MVT InVT = InOp.getValueType(); 1504 MVT InEltVT = InVT.getVectorElementType(); 1505 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts); 1506 1507 SDValue DTyOp = DAG.getValueType(WidenVT); 1508 SDValue STyOp = DAG.getValueType(InWidenVT); 1509 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1510 1511 unsigned InVTNumElts = InVT.getVectorNumElements(); 1512 if (getTypeAction(InVT) == WidenVector) { 1513 InOp = GetWidenedVector(InOp); 1514 InVT = InOp.getValueType(); 1515 InVTNumElts = InVT.getVectorNumElements(); 1516 if (InVTNumElts == WidenNumElts) 1517 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1518 SatOp, CvtCode); 1519 } 1520 1521 if (TLI.isTypeLegal(InWidenVT)) { 1522 // Because the result and the input are different vector types, widening 1523 // the result could create a legal type but widening the input might make 1524 // it an illegal type that might lead to repeatedly splitting the input 1525 // and then widening it. To avoid this, we widen the input only if 1526 // it results in a legal type. 1527 if (WidenNumElts % InVTNumElts == 0) { 1528 // Widen the input and call convert on the widened input vector. 1529 unsigned NumConcat = WidenNumElts/InVTNumElts; 1530 SmallVector<SDValue, 16> Ops(NumConcat); 1531 Ops[0] = InOp; 1532 SDValue UndefVal = DAG.getUNDEF(InVT); 1533 for (unsigned i = 1; i != NumConcat; ++i) { 1534 Ops[i] = UndefVal; 1535 } 1536 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1537 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1538 SatOp, CvtCode); 1539 } 1540 1541 if (InVTNumElts % WidenNumElts == 0) { 1542 // Extract the input and convert the shorten input vector. 1543 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1544 DAG.getIntPtrConstant(0)); 1545 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1546 SatOp, CvtCode); 1547 } 1548 } 1549 1550 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1551 SmallVector<SDValue, 16> Ops(WidenNumElts); 1552 MVT EltVT = WidenVT.getVectorElementType(); 1553 DTyOp = DAG.getValueType(EltVT); 1554 STyOp = DAG.getValueType(InEltVT); 1555 1556 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1557 unsigned i; 1558 for (i=0; i < MinElts; ++i) { 1559 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1560 DAG.getIntPtrConstant(i)); 1561 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1562 SatOp, CvtCode); 1563 } 1564 1565 SDValue UndefVal = DAG.getUNDEF(EltVT); 1566 for (; i < WidenNumElts; ++i) 1567 Ops[i] = UndefVal; 1568 1569 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1570} 1571 1572SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1573 MVT VT = N->getValueType(0); 1574 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1575 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1576 SDValue InOp = N->getOperand(0); 1577 SDValue Idx = N->getOperand(1); 1578 DebugLoc dl = N->getDebugLoc(); 1579 1580 if (getTypeAction(InOp.getValueType()) == WidenVector) 1581 InOp = GetWidenedVector(InOp); 1582 1583 MVT InVT = InOp.getValueType(); 1584 1585 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1586 if (CIdx) { 1587 unsigned IdxVal = CIdx->getZExtValue(); 1588 // Check if we can just return the input vector after widening. 1589 if (IdxVal == 0 && InVT == WidenVT) 1590 return InOp; 1591 1592 // Check if we can extract from the vector. 1593 unsigned InNumElts = InVT.getVectorNumElements(); 1594 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1595 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1596 } 1597 1598 // We could try widening the input to the right length but for now, extract 1599 // the original elements, fill the rest with undefs and build a vector. 1600 SmallVector<SDValue, 16> Ops(WidenNumElts); 1601 MVT EltVT = VT.getVectorElementType(); 1602 MVT IdxVT = Idx.getValueType(); 1603 unsigned NumElts = VT.getVectorNumElements(); 1604 unsigned i; 1605 if (CIdx) { 1606 unsigned IdxVal = CIdx->getZExtValue(); 1607 for (i=0; i < NumElts; ++i) 1608 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1609 DAG.getConstant(IdxVal+i, IdxVT)); 1610 } else { 1611 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1612 for (i=1; i < NumElts; ++i) { 1613 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1614 DAG.getConstant(i, IdxVT)); 1615 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1616 } 1617 } 1618 1619 SDValue UndefVal = DAG.getUNDEF(EltVT); 1620 for (; i < WidenNumElts; ++i) 1621 Ops[i] = UndefVal; 1622 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1623} 1624 1625SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1626 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1627 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1628 InOp.getValueType(), InOp, 1629 N->getOperand(1), N->getOperand(2)); 1630} 1631 1632SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1633 LoadSDNode *LD = cast<LoadSDNode>(N); 1634 MVT WidenVT = TLI.getTypeToTransformTo(LD->getValueType(0)); 1635 MVT LdVT = LD->getMemoryVT(); 1636 DebugLoc dl = N->getDebugLoc(); 1637 assert(LdVT.isVector() && WidenVT.isVector()); 1638 1639 // Load information 1640 SDValue Chain = LD->getChain(); 1641 SDValue BasePtr = LD->getBasePtr(); 1642 int SVOffset = LD->getSrcValueOffset(); 1643 unsigned Align = LD->getAlignment(); 1644 bool isVolatile = LD->isVolatile(); 1645 const Value *SV = LD->getSrcValue(); 1646 ISD::LoadExtType ExtType = LD->getExtensionType(); 1647 1648 SDValue Result; 1649 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1650 if (ExtType != ISD::NON_EXTLOAD) { 1651 // For extension loads, we can not play the tricks of chopping legal 1652 // vector types and bit cast it to the right type. Instead, we unroll 1653 // the load and build a vector. 1654 MVT EltVT = WidenVT.getVectorElementType(); 1655 MVT LdEltVT = LdVT.getVectorElementType(); 1656 unsigned NumElts = LdVT.getVectorNumElements(); 1657 1658 // Load each element and widen 1659 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1660 SmallVector<SDValue, 16> Ops(WidenNumElts); 1661 unsigned Increment = LdEltVT.getSizeInBits() / 8; 1662 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 1663 LdEltVT, isVolatile, Align); 1664 LdChain.push_back(Ops[0].getValue(1)); 1665 unsigned i = 0, Offset = Increment; 1666 for (i=1; i < NumElts; ++i, Offset += Increment) { 1667 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1668 BasePtr, DAG.getIntPtrConstant(Offset)); 1669 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 1670 SVOffset + Offset, LdEltVT, isVolatile, Align); 1671 LdChain.push_back(Ops[i].getValue(1)); 1672 } 1673 1674 // Fill the rest with undefs 1675 SDValue UndefVal = DAG.getUNDEF(EltVT); 1676 for (; i != WidenNumElts; ++i) 1677 Ops[i] = UndefVal; 1678 1679 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 1680 } else { 1681 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 1682 unsigned int LdWidth = LdVT.getSizeInBits(); 1683 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 1684 Align, isVolatile, LdWidth, WidenVT, dl); 1685 } 1686 1687 // If we generate a single load, we can use that for the chain. Otherwise, 1688 // build a factor node to remember the multiple loads are independent and 1689 // chain to that. 1690 SDValue NewChain; 1691 if (LdChain.size() == 1) 1692 NewChain = LdChain[0]; 1693 else 1694 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 1695 LdChain.size()); 1696 1697 // Modified the chain - switch anything that used the old chain to use 1698 // the new one. 1699 ReplaceValueWith(SDValue(N, 1), Chain); 1700 1701 return Result; 1702} 1703 1704SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1705 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1706 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1707 WidenVT, N->getOperand(0)); 1708} 1709 1710SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1711 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1712 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1713 1714 SDValue Cond1 = N->getOperand(0); 1715 MVT CondVT = Cond1.getValueType(); 1716 if (CondVT.isVector()) { 1717 MVT CondEltVT = CondVT.getVectorElementType(); 1718 MVT CondWidenVT = MVT::getVectorVT(CondEltVT, WidenNumElts); 1719 if (getTypeAction(CondVT) == WidenVector) 1720 Cond1 = GetWidenedVector(Cond1); 1721 1722 if (Cond1.getValueType() != CondWidenVT) 1723 Cond1 = ModifyToType(Cond1, CondWidenVT); 1724 } 1725 1726 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1727 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1728 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1729 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1730 WidenVT, Cond1, InOp1, InOp2); 1731} 1732 1733SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1734 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1735 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1736 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1737 InOp1.getValueType(), N->getOperand(0), 1738 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1739} 1740 1741SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1742 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1743 return DAG.getUNDEF(WidenVT); 1744} 1745 1746SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1747 MVT VT = N->getValueType(0); 1748 DebugLoc dl = N->getDebugLoc(); 1749 1750 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1751 unsigned NumElts = VT.getVectorNumElements(); 1752 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1753 1754 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1755 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1756 1757 // Adjust mask based on new input vector length. 1758 SmallVector<int, 16> NewMask; 1759 for (unsigned i = 0; i != NumElts; ++i) { 1760 int Idx = N->getMaskElt(i); 1761 if (Idx < (int)NumElts) 1762 NewMask.push_back(Idx); 1763 else 1764 NewMask.push_back(Idx - NumElts + WidenNumElts); 1765 } 1766 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1767 NewMask.push_back(-1); 1768 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1769} 1770 1771SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1772 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1773 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1774 1775 SDValue InOp1 = N->getOperand(0); 1776 MVT InVT = InOp1.getValueType(); 1777 assert(InVT.isVector() && "can not widen non vector type"); 1778 MVT WidenInVT = MVT::getVectorVT(InVT.getVectorElementType(), WidenNumElts); 1779 InOp1 = GetWidenedVector(InOp1); 1780 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1781 1782 // Assume that the input and output will be widen appropriately. If not, 1783 // we will have to unroll it at some point. 1784 assert(InOp1.getValueType() == WidenInVT && 1785 InOp2.getValueType() == WidenInVT && 1786 "Input not widened to expected type!"); 1787 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1788 WidenVT, InOp1, InOp2, N->getOperand(2)); 1789} 1790 1791 1792//===----------------------------------------------------------------------===// 1793// Widen Vector Operand 1794//===----------------------------------------------------------------------===// 1795bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1796 DEBUG(cerr << "Widen node operand " << ResNo << ": "; N->dump(&DAG); 1797 cerr << "\n"); 1798 SDValue Res = SDValue(); 1799 1800 switch (N->getOpcode()) { 1801 default: 1802#ifndef NDEBUG 1803 cerr << "WidenVectorOperand op #" << ResNo << ": "; 1804 N->dump(&DAG); cerr << "\n"; 1805#endif 1806 assert(0 && "Do not know how to widen this operator's operand!"); 1807 abort(); 1808 1809 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1810 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1811 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1812 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1813 1814 case ISD::FP_ROUND: 1815 case ISD::FP_TO_SINT: 1816 case ISD::FP_TO_UINT: 1817 case ISD::SINT_TO_FP: 1818 case ISD::UINT_TO_FP: 1819 case ISD::TRUNCATE: 1820 case ISD::SIGN_EXTEND: 1821 case ISD::ZERO_EXTEND: 1822 case ISD::ANY_EXTEND: 1823 Res = WidenVecOp_Convert(N); 1824 break; 1825 } 1826 1827 // If Res is null, the sub-method took care of registering the result. 1828 if (!Res.getNode()) return false; 1829 1830 // If the result is N, the sub-method updated N in place. Tell the legalizer 1831 // core about this. 1832 if (Res.getNode() == N) 1833 return true; 1834 1835 1836 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1837 "Invalid operand expansion"); 1838 1839 ReplaceValueWith(SDValue(N, 0), Res); 1840 return false; 1841} 1842 1843SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1844 // Since the result is legal and the input is illegal, it is unlikely 1845 // that we can fix the input to a legal type so unroll the convert 1846 // into some scalar code and create a nasty build vector. 1847 MVT VT = N->getValueType(0); 1848 MVT EltVT = VT.getVectorElementType(); 1849 DebugLoc dl = N->getDebugLoc(); 1850 unsigned NumElts = VT.getVectorNumElements(); 1851 SDValue InOp = N->getOperand(0); 1852 if (getTypeAction(InOp.getValueType()) == WidenVector) 1853 InOp = GetWidenedVector(InOp); 1854 MVT InVT = InOp.getValueType(); 1855 MVT InEltVT = InVT.getVectorElementType(); 1856 1857 unsigned Opcode = N->getOpcode(); 1858 SmallVector<SDValue, 16> Ops(NumElts); 1859 for (unsigned i=0; i < NumElts; ++i) 1860 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1861 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1862 DAG.getIntPtrConstant(i))); 1863 1864 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1865} 1866 1867SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1868 MVT VT = N->getValueType(0); 1869 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1870 MVT InWidenVT = InOp.getValueType(); 1871 DebugLoc dl = N->getDebugLoc(); 1872 1873 // Check if we can convert between two legal vector types and extract. 1874 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1875 unsigned Size = VT.getSizeInBits(); 1876 if (InWidenSize % Size == 0 && !VT.isVector()) { 1877 unsigned NewNumElts = InWidenSize / Size; 1878 MVT NewVT = MVT::getVectorVT(VT, NewNumElts); 1879 if (TLI.isTypeLegal(NewVT)) { 1880 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1881 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1882 DAG.getIntPtrConstant(0)); 1883 } 1884 } 1885 1886 // Lower the bit-convert to a store/load from the stack. Create the stack 1887 // frame object. Make sure it is aligned for both the source and destination 1888 // types. 1889 SDValue FIPtr = DAG.CreateStackTemporary(InWidenVT, VT); 1890 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1891 const Value *SV = PseudoSourceValue::getFixedStack(FI); 1892 1893 // Emit a store to the stack slot. 1894 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0); 1895 1896 // Result is a load from the stack slot. 1897 return DAG.getLoad(VT, dl, Store, FIPtr, SV, 0); 1898} 1899 1900SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1901 // If the input vector is not legal, it is likely that we will not find a 1902 // legal vector of the same size. Replace the concatenate vector with a 1903 // nasty build vector. 1904 MVT VT = N->getValueType(0); 1905 MVT EltVT = VT.getVectorElementType(); 1906 DebugLoc dl = N->getDebugLoc(); 1907 unsigned NumElts = VT.getVectorNumElements(); 1908 SmallVector<SDValue, 16> Ops(NumElts); 1909 1910 MVT InVT = N->getOperand(0).getValueType(); 1911 unsigned NumInElts = InVT.getVectorNumElements(); 1912 1913 unsigned Idx = 0; 1914 unsigned NumOperands = N->getNumOperands(); 1915 for (unsigned i=0; i < NumOperands; ++i) { 1916 SDValue InOp = N->getOperand(i); 1917 if (getTypeAction(InOp.getValueType()) == WidenVector) 1918 InOp = GetWidenedVector(InOp); 1919 for (unsigned j=0; j < NumInElts; ++j) 1920 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1921 DAG.getIntPtrConstant(j)); 1922 } 1923 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1924} 1925 1926SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1927 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1928 MVT EltVT = InOp.getValueType().getVectorElementType(); 1929 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1930 EltVT, InOp, N->getOperand(1)); 1931} 1932 1933SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1934 // We have to widen the value but we want only to store the original 1935 // vector type. 1936 StoreSDNode *ST = cast<StoreSDNode>(N); 1937 SDValue Chain = ST->getChain(); 1938 SDValue BasePtr = ST->getBasePtr(); 1939 const Value *SV = ST->getSrcValue(); 1940 int SVOffset = ST->getSrcValueOffset(); 1941 unsigned Align = ST->getAlignment(); 1942 bool isVolatile = ST->isVolatile(); 1943 SDValue ValOp = GetWidenedVector(ST->getValue()); 1944 DebugLoc dl = N->getDebugLoc(); 1945 1946 MVT StVT = ST->getMemoryVT(); 1947 MVT ValVT = ValOp.getValueType(); 1948 // It must be true that we the widen vector type is bigger than where 1949 // we need to store. 1950 assert(StVT.isVector() && ValOp.getValueType().isVector()); 1951 assert(StVT.bitsLT(ValOp.getValueType())); 1952 1953 SmallVector<SDValue, 16> StChain; 1954 if (ST->isTruncatingStore()) { 1955 // For truncating stores, we can not play the tricks of chopping legal 1956 // vector types and bit cast it to the right type. Instead, we unroll 1957 // the store. 1958 MVT StEltVT = StVT.getVectorElementType(); 1959 MVT ValEltVT = ValVT.getVectorElementType(); 1960 unsigned Increment = ValEltVT.getSizeInBits() / 8; 1961 unsigned NumElts = StVT.getVectorNumElements(); 1962 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1963 DAG.getIntPtrConstant(0)); 1964 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 1965 SVOffset, StEltVT, 1966 isVolatile, Align)); 1967 unsigned Offset = Increment; 1968 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 1969 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1970 BasePtr, DAG.getIntPtrConstant(Offset)); 1971 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1972 DAG.getIntPtrConstant(0)); 1973 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 1974 SVOffset + Offset, StEltVT, 1975 isVolatile, MinAlign(Align, Offset))); 1976 } 1977 } 1978 else { 1979 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 1980 // Store value 1981 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 1982 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 1983 } 1984 if (StChain.size() == 1) 1985 return StChain[0]; 1986 else 1987 return DAG.getNode(ISD::TokenFactor, dl, 1988 MVT::Other,&StChain[0],StChain.size()); 1989} 1990 1991//===----------------------------------------------------------------------===// 1992// Vector Widening Utilities 1993//===----------------------------------------------------------------------===// 1994 1995 1996// Utility function to find a vector type and its associated element 1997// type from a preferred width and whose vector type must be the same size 1998// as the VecVT. 1999// TLI: Target lowering used to determine legal types. 2000// Width: Preferred width to store. 2001// VecVT: Vector value type whose size we must match. 2002// Returns NewVecVT and NewEltVT - the vector type and its associated 2003// element type. 2004static void FindAssocWidenVecType(const TargetLowering &TLI, unsigned Width, 2005 MVT VecVT, 2006 MVT& NewEltVT, MVT& NewVecVT) { 2007 unsigned EltWidth = Width + 1; 2008 if (TLI.isTypeLegal(VecVT)) { 2009 // We start with the preferred with, making it a power of 2 and find a 2010 // legal vector type of that width. If not, we reduce it by another of 2. 2011 // For incoming type is legal, this process will end as a vector of the 2012 // smallest loadable type should always be legal. 2013 do { 2014 assert(EltWidth > 0); 2015 EltWidth = 1 << Log2_32(EltWidth - 1); 2016 NewEltVT = MVT::getIntegerVT(EltWidth); 2017 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2018 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts); 2019 } while (!TLI.isTypeLegal(NewVecVT) || 2020 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2021 } else { 2022 // The incoming vector type is illegal and is the result of widening 2023 // a vector to a power of 2. In this case, we will use the preferred 2024 // with as long as it is a multiple of the incoming vector length. 2025 // The legalization process will eventually make this into a legal type 2026 // and remove the illegal bit converts (which would turn to stack converts 2027 // if they are allow to exist). 2028 do { 2029 assert(EltWidth > 0); 2030 EltWidth = 1 << Log2_32(EltWidth - 1); 2031 NewEltVT = MVT::getIntegerVT(EltWidth); 2032 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2033 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts); 2034 } while (!TLI.isTypeLegal(NewEltVT) || 2035 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2036 } 2037} 2038 2039SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2040 SDValue Chain, 2041 SDValue BasePtr, 2042 const Value *SV, 2043 int SVOffset, 2044 unsigned Alignment, 2045 bool isVolatile, 2046 unsigned LdWidth, 2047 MVT ResType, 2048 DebugLoc dl) { 2049 // The strategy assumes that we can efficiently load powers of two widths. 2050 // The routines chops the vector into the largest power of 2 load and 2051 // can be inserted into a legal vector and then cast the result into the 2052 // vector type we want. This avoids unnecessary stack converts. 2053 2054 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 2055 // the load is nonvolatile, we an use a wider load for the value. 2056 2057 // Find the vector type that can load from. 2058 MVT NewEltVT, NewVecVT; 2059 unsigned NewEltVTWidth; 2060 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2061 NewEltVTWidth = NewEltVT.getSizeInBits(); 2062 2063 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 2064 isVolatile, Alignment); 2065 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2066 LdChain.push_back(LdOp.getValue(1)); 2067 2068 // Check if we can load the element with one instruction 2069 if (LdWidth == NewEltVTWidth) { 2070 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2071 } 2072 2073 unsigned Idx = 1; 2074 LdWidth -= NewEltVTWidth; 2075 unsigned Offset = 0; 2076 2077 while (LdWidth > 0) { 2078 unsigned Increment = NewEltVTWidth / 8; 2079 Offset += Increment; 2080 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2081 DAG.getIntPtrConstant(Increment)); 2082 2083 if (LdWidth < NewEltVTWidth) { 2084 // Our current type we are using is too large, use a smaller size by 2085 // using a smaller power of 2 2086 unsigned oNewEltVTWidth = NewEltVTWidth; 2087 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2088 NewEltVTWidth = NewEltVT.getSizeInBits(); 2089 // Readjust position and vector position based on new load type 2090 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2091 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2092 } 2093 2094 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 2095 SVOffset+Offset, isVolatile, 2096 MinAlign(Alignment, Offset)); 2097 LdChain.push_back(LdOp.getValue(1)); 2098 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 2099 DAG.getIntPtrConstant(Idx++)); 2100 2101 LdWidth -= NewEltVTWidth; 2102 } 2103 2104 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2105} 2106 2107void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2108 SDValue Chain, 2109 SDValue BasePtr, 2110 const Value *SV, 2111 int SVOffset, 2112 unsigned Alignment, 2113 bool isVolatile, 2114 SDValue ValOp, 2115 unsigned StWidth, 2116 DebugLoc dl) { 2117 // Breaks the stores into a series of power of 2 width stores. For any 2118 // width, we convert the vector to the vector of element size that we 2119 // want to store. This avoids requiring a stack convert. 2120 2121 // Find a width of the element type we can store with 2122 MVT WidenVT = ValOp.getValueType(); 2123 MVT NewEltVT, NewVecVT; 2124 2125 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2126 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 2127 2128 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2129 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2130 DAG.getIntPtrConstant(0)); 2131 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 2132 isVolatile, Alignment); 2133 StChain.push_back(StOp); 2134 2135 // Check if we are done 2136 if (StWidth == NewEltVTWidth) { 2137 return; 2138 } 2139 2140 unsigned Idx = 1; 2141 StWidth -= NewEltVTWidth; 2142 unsigned Offset = 0; 2143 2144 while (StWidth > 0) { 2145 unsigned Increment = NewEltVTWidth / 8; 2146 Offset += Increment; 2147 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2148 DAG.getIntPtrConstant(Increment)); 2149 2150 if (StWidth < NewEltVTWidth) { 2151 // Our current type we are using is too large, use a smaller size by 2152 // using a smaller power of 2 2153 unsigned oNewEltVTWidth = NewEltVTWidth; 2154 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2155 NewEltVTWidth = NewEltVT.getSizeInBits(); 2156 // Readjust position and vector position based on new load type 2157 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2158 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2159 } 2160 2161 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2162 DAG.getIntPtrConstant(Idx++)); 2163 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2164 SVOffset + Offset, isVolatile, 2165 MinAlign(Alignment, Offset))); 2166 StWidth -= NewEltVTWidth; 2167 } 2168} 2169 2170/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2171/// input vector must have the same element type as NVT. 2172SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, MVT NVT) { 2173 // Note that InOp might have been widened so it might already have 2174 // the right width or it might need be narrowed. 2175 MVT InVT = InOp.getValueType(); 2176 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2177 "input and widen element type must match"); 2178 DebugLoc dl = InOp.getDebugLoc(); 2179 2180 // Check if InOp already has the right width. 2181 if (InVT == NVT) 2182 return InOp; 2183 2184 unsigned InNumElts = InVT.getVectorNumElements(); 2185 unsigned WidenNumElts = NVT.getVectorNumElements(); 2186 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2187 unsigned NumConcat = WidenNumElts / InNumElts; 2188 SmallVector<SDValue, 16> Ops(NumConcat); 2189 SDValue UndefVal = DAG.getUNDEF(InVT); 2190 Ops[0] = InOp; 2191 for (unsigned i = 1; i != NumConcat; ++i) 2192 Ops[i] = UndefVal; 2193 2194 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2195 } 2196 2197 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2198 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2199 DAG.getIntPtrConstant(0)); 2200 2201 // Fall back to extract and build. 2202 SmallVector<SDValue, 16> Ops(WidenNumElts); 2203 MVT EltVT = NVT.getVectorElementType(); 2204 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2205 unsigned Idx; 2206 for (Idx = 0; Idx < MinNumElts; ++Idx) 2207 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2208 DAG.getIntPtrConstant(Idx)); 2209 2210 SDValue UndefVal = DAG.getUNDEF(EltVT); 2211 for ( ; Idx < WidenNumElts; ++Idx) 2212 Ops[Idx] = UndefVal; 2213 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2214} 2215