LegalizeVectorTypes.cpp revision a0167020062bbcfe0558faa29dd705ca6f9a8e2a
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in two vectors of half the size. For example, implementing 19// <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26using namespace llvm; 27 28//===----------------------------------------------------------------------===// 29// Result Vector Scalarization: <1 x ty> -> ty. 30//===----------------------------------------------------------------------===// 31 32void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 33 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 34 cerr << "\n"); 35 SDValue R = SDValue(); 36 37 switch (N->getOpcode()) { 38 default: 39#ifndef NDEBUG 40 cerr << "ScalarizeVectorResult #" << ResNo << ": "; 41 N->dump(&DAG); cerr << "\n"; 42#endif 43 assert(0 && "Do not know how to scalarize the result of this operator!"); 44 abort(); 45 46 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 47 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 48 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 49 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 50 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 51 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 52 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 53 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 54 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 55 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 56 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 57 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 58 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 59 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 60 61 case ISD::CTLZ: 62 case ISD::CTPOP: 63 case ISD::CTTZ: 64 case ISD::FABS: 65 case ISD::FCOS: 66 case ISD::FNEG: 67 case ISD::FP_TO_SINT: 68 case ISD::FP_TO_UINT: 69 case ISD::FSIN: 70 case ISD::FSQRT: 71 case ISD::FTRUNC: 72 case ISD::FFLOOR: 73 case ISD::FCEIL: 74 case ISD::FRINT: 75 case ISD::FNEARBYINT: 76 case ISD::UINT_TO_FP: 77 case ISD::SINT_TO_FP: 78 case ISD::TRUNCATE: 79 case ISD::SIGN_EXTEND: 80 case ISD::ZERO_EXTEND: 81 case ISD::ANY_EXTEND: 82 R = ScalarizeVecRes_UnaryOp(N); 83 break; 84 85 case ISD::ADD: 86 case ISD::AND: 87 case ISD::FADD: 88 case ISD::FDIV: 89 case ISD::FMUL: 90 case ISD::FPOW: 91 case ISD::FREM: 92 case ISD::FSUB: 93 case ISD::MUL: 94 case ISD::OR: 95 case ISD::SDIV: 96 case ISD::SREM: 97 case ISD::SUB: 98 case ISD::UDIV: 99 case ISD::UREM: 100 case ISD::XOR: 101 case ISD::SHL: 102 case ISD::SRA: 103 case ISD::SRL: 104 R = ScalarizeVecRes_BinOp(N); 105 break; 106 } 107 108 // If R is null, the sub-method took care of registering the result. 109 if (R.getNode()) 110 SetScalarizedVector(SDValue(N, ResNo), R); 111} 112 113SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 114 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 115 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 116 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 117 LHS.getValueType(), LHS, RHS); 118} 119 120SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 121 MVT NewVT = N->getValueType(0).getVectorElementType(); 122 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 123 NewVT, N->getOperand(0)); 124} 125 126SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 127 MVT NewVT = N->getValueType(0).getVectorElementType(); 128 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 129 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 130 Op0, DAG.getValueType(NewVT), 131 DAG.getValueType(Op0.getValueType()), 132 N->getOperand(3), 133 N->getOperand(4), 134 cast<CvtRndSatSDNode>(N)->getCvtCode()); 135} 136 137SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 138 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 139 N->getValueType(0).getVectorElementType(), 140 N->getOperand(0), N->getOperand(1)); 141} 142 143SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 144 SDValue Op = GetScalarizedVector(N->getOperand(0)); 145 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 146 Op.getValueType(), Op, N->getOperand(1)); 147} 148 149SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 150 // The value to insert may have a wider type than the vector element type, 151 // so be sure to truncate it to the element type if necessary. 152 SDValue Op = N->getOperand(1); 153 MVT EltVT = N->getValueType(0).getVectorElementType(); 154 if (Op.getValueType() != EltVT) 155 // FIXME: Can this happen for floating point types? 156 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 157 return Op; 158} 159 160SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 161 assert(N->isUnindexed() && "Indexed vector load?"); 162 163 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 164 N->getExtensionType(), 165 N->getValueType(0).getVectorElementType(), 166 N->getChain(), N->getBasePtr(), 167 DAG.getUNDEF(N->getBasePtr().getValueType()), 168 N->getSrcValue(), N->getSrcValueOffset(), 169 N->getMemoryVT().getVectorElementType(), 170 N->isVolatile(), N->getAlignment()); 171 172 // Legalized the chain result - switch anything that used the old chain to 173 // use the new one. 174 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 175 return Result; 176} 177 178SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 179 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 180 MVT DestVT = N->getValueType(0).getVectorElementType(); 181 SDValue Op = GetScalarizedVector(N->getOperand(0)); 182 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 183} 184 185SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 186 // If the operand is wider than the vector element type then it is implicitly 187 // truncated. Make that explicit here. 188 MVT EltVT = N->getValueType(0).getVectorElementType(); 189 SDValue InOp = N->getOperand(0); 190 if (InOp.getValueType() != EltVT) 191 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 192 return InOp; 193} 194 195SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 196 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 197 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 198 LHS.getValueType(), N->getOperand(0), LHS, 199 GetScalarizedVector(N->getOperand(2))); 200} 201 202SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 203 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 204 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 205 N->getOperand(0), N->getOperand(1), 206 LHS, GetScalarizedVector(N->getOperand(3)), 207 N->getOperand(4)); 208} 209 210SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 211 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 212 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 213 DebugLoc DL = N->getDebugLoc(); 214 215 // Turn it into a scalar SETCC. 216 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 217} 218 219SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 220 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 221} 222 223SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 224 // Figure out if the scalar is the LHS or RHS and return it. 225 SDValue Arg = N->getOperand(2).getOperand(0); 226 if (Arg.getOpcode() == ISD::UNDEF) 227 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 228 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 229 return GetScalarizedVector(N->getOperand(Op)); 230} 231 232SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 233 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 234 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 235 MVT NVT = N->getValueType(0).getVectorElementType(); 236 MVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 237 DebugLoc DL = N->getDebugLoc(); 238 239 // Turn it into a scalar SETCC. 240 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 241 242 // VSETCC always returns a sign-extended value, while SETCC may not. The 243 // SETCC result type may not match the vector element type. Correct these. 244 if (NVT.bitsLE(SVT)) { 245 // The SETCC result type is bigger than the vector element type. 246 // Ensure the SETCC result is sign-extended. 247 if (TLI.getBooleanContents() != 248 TargetLowering::ZeroOrNegativeOneBooleanContent) 249 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 250 DAG.getValueType(MVT::i1)); 251 // Truncate to the final type. 252 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 253 } 254 255 // The SETCC result type is smaller than the vector element type. 256 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 257 if (TLI.getBooleanContents() != 258 TargetLowering::ZeroOrNegativeOneBooleanContent) 259 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 260 // Sign extend to the final type. 261 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 262} 263 264 265//===----------------------------------------------------------------------===// 266// Operand Vector Scalarization <1 x ty> -> ty. 267//===----------------------------------------------------------------------===// 268 269bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 270 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 271 cerr << "\n"); 272 SDValue Res = SDValue(); 273 274 if (Res.getNode() == 0) { 275 switch (N->getOpcode()) { 276 default: 277#ifndef NDEBUG 278 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": "; 279 N->dump(&DAG); cerr << "\n"; 280#endif 281 assert(0 && "Do not know how to scalarize this operator's operand!"); 282 case ISD::BIT_CONVERT: 283 Res = ScalarizeVecOp_BIT_CONVERT(N); 284 break; 285 case ISD::CONCAT_VECTORS: 286 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 287 break; 288 case ISD::EXTRACT_VECTOR_ELT: 289 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 290 break; 291 case ISD::STORE: 292 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 293 break; 294 } 295 } 296 297 // If the result is null, the sub-method took care of registering results etc. 298 if (!Res.getNode()) return false; 299 300 // If the result is N, the sub-method updated N in place. Tell the legalizer 301 // core about this. 302 if (Res.getNode() == N) 303 return true; 304 305 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 306 "Invalid operand expansion"); 307 308 ReplaceValueWith(SDValue(N, 0), Res); 309 return false; 310} 311 312/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 313/// to be scalarized, it must be <1 x ty>. Convert the element instead. 314SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 315 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 316 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 317 N->getValueType(0), Elt); 318} 319 320/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 321/// use a BUILD_VECTOR instead. 322SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 323 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 324 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 325 Ops[i] = GetScalarizedVector(N->getOperand(i)); 326 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 327 &Ops[0], Ops.size()); 328} 329 330/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 331/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 332/// index. 333SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 334 return GetScalarizedVector(N->getOperand(0)); 335} 336 337/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 338/// scalarized, it must be <1 x ty>. Just store the element. 339SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 340 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 341 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 342 DebugLoc dl = N->getDebugLoc(); 343 344 if (N->isTruncatingStore()) 345 return DAG.getTruncStore(N->getChain(), dl, 346 GetScalarizedVector(N->getOperand(1)), 347 N->getBasePtr(), 348 N->getSrcValue(), N->getSrcValueOffset(), 349 N->getMemoryVT().getVectorElementType(), 350 N->isVolatile(), N->getAlignment()); 351 352 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 353 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 354 N->isVolatile(), N->getAlignment()); 355} 356 357 358//===----------------------------------------------------------------------===// 359// Result Vector Splitting 360//===----------------------------------------------------------------------===// 361 362/// SplitVectorResult - This method is called when the specified result of the 363/// specified node is found to need vector splitting. At this point, the node 364/// may also have invalid operands or may have other results that need 365/// legalization, we just know that (at least) one result needs vector 366/// splitting. 367void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 368 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n"); 369 SDValue Lo, Hi; 370 371 switch (N->getOpcode()) { 372 default: 373#ifndef NDEBUG 374 cerr << "SplitVectorResult #" << ResNo << ": "; 375 N->dump(&DAG); cerr << "\n"; 376#endif 377 assert(0 && "Do not know how to split the result of this operator!"); 378 abort(); 379 380 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 381 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 382 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 383 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 384 385 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 386 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 387 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 388 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 389 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 390 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 391 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 392 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 393 case ISD::LOAD: 394 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 395 break; 396 case ISD::SETCC: 397 case ISD::VSETCC: 398 SplitVecRes_SETCC(N, Lo, Hi); 399 break; 400 case ISD::VECTOR_SHUFFLE: 401 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 402 break; 403 404 case ISD::CTTZ: 405 case ISD::CTLZ: 406 case ISD::CTPOP: 407 case ISD::FNEG: 408 case ISD::FABS: 409 case ISD::FSQRT: 410 case ISD::FSIN: 411 case ISD::FCOS: 412 case ISD::FTRUNC: 413 case ISD::FFLOOR: 414 case ISD::FCEIL: 415 case ISD::FRINT: 416 case ISD::FNEARBYINT: 417 case ISD::FP_TO_SINT: 418 case ISD::FP_TO_UINT: 419 case ISD::SINT_TO_FP: 420 case ISD::UINT_TO_FP: 421 case ISD::TRUNCATE: 422 case ISD::SIGN_EXTEND: 423 case ISD::ZERO_EXTEND: 424 case ISD::ANY_EXTEND: 425 SplitVecRes_UnaryOp(N, Lo, Hi); 426 break; 427 428 case ISD::ADD: 429 case ISD::SUB: 430 case ISD::MUL: 431 case ISD::FADD: 432 case ISD::FSUB: 433 case ISD::FMUL: 434 case ISD::SDIV: 435 case ISD::UDIV: 436 case ISD::FDIV: 437 case ISD::FPOW: 438 case ISD::AND: 439 case ISD::OR: 440 case ISD::XOR: 441 case ISD::SHL: 442 case ISD::SRA: 443 case ISD::SRL: 444 case ISD::UREM: 445 case ISD::SREM: 446 case ISD::FREM: 447 SplitVecRes_BinOp(N, Lo, Hi); 448 break; 449 } 450 451 // If Lo/Hi is null, the sub-method took care of registering results etc. 452 if (Lo.getNode()) 453 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 454} 455 456void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 457 SDValue &Hi) { 458 SDValue LHSLo, LHSHi; 459 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 460 SDValue RHSLo, RHSHi; 461 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 462 DebugLoc dl = N->getDebugLoc(); 463 464 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 465 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 466} 467 468void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 469 SDValue &Hi) { 470 // We know the result is a vector. The input may be either a vector or a 471 // scalar value. 472 MVT LoVT, HiVT; 473 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 474 DebugLoc dl = N->getDebugLoc(); 475 476 SDValue InOp = N->getOperand(0); 477 MVT InVT = InOp.getValueType(); 478 479 // Handle some special cases efficiently. 480 switch (getTypeAction(InVT)) { 481 default: 482 assert(false && "Unknown type action!"); 483 case Legal: 484 case PromoteInteger: 485 case SoftenFloat: 486 case ScalarizeVector: 487 break; 488 case ExpandInteger: 489 case ExpandFloat: 490 // A scalar to vector conversion, where the scalar needs expansion. 491 // If the vector is being split in two then we can just convert the 492 // expanded pieces. 493 if (LoVT == HiVT) { 494 GetExpandedOp(InOp, Lo, Hi); 495 if (TLI.isBigEndian()) 496 std::swap(Lo, Hi); 497 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 498 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 499 return; 500 } 501 break; 502 case SplitVector: 503 // If the input is a vector that needs to be split, convert each split 504 // piece of the input now. 505 GetSplitVector(InOp, Lo, Hi); 506 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 507 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 508 return; 509 } 510 511 // In the general case, convert the input to an integer and split it by hand. 512 MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits()); 513 MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits()); 514 if (TLI.isBigEndian()) 515 std::swap(LoIntVT, HiIntVT); 516 517 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 518 519 if (TLI.isBigEndian()) 520 std::swap(Lo, Hi); 521 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 522 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 523} 524 525void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 526 SDValue &Hi) { 527 MVT LoVT, HiVT; 528 DebugLoc dl = N->getDebugLoc(); 529 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 530 unsigned LoNumElts = LoVT.getVectorNumElements(); 531 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 532 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 533 534 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 535 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 536} 537 538void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 539 SDValue &Hi) { 540 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 541 DebugLoc dl = N->getDebugLoc(); 542 unsigned NumSubvectors = N->getNumOperands() / 2; 543 if (NumSubvectors == 1) { 544 Lo = N->getOperand(0); 545 Hi = N->getOperand(1); 546 return; 547 } 548 549 MVT LoVT, HiVT; 550 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 551 552 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 553 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 554 555 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 556 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 557} 558 559void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 560 SDValue &Hi) { 561 MVT LoVT, HiVT; 562 DebugLoc dl = N->getDebugLoc(); 563 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 564 565 SDValue DTyOpLo = DAG.getValueType(LoVT); 566 SDValue DTyOpHi = DAG.getValueType(HiVT); 567 568 SDValue RndOp = N->getOperand(3); 569 SDValue SatOp = N->getOperand(4); 570 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 571 572 // Split the input. 573 SDValue VLo, VHi; 574 MVT InVT = N->getOperand(0).getValueType(); 575 switch (getTypeAction(InVT)) { 576 default: assert(0 && "Unexpected type action!"); 577 case Legal: { 578 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 579 LoVT.getVectorNumElements()); 580 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 581 DAG.getIntPtrConstant(0)); 582 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 583 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 584 break; 585 } 586 case SplitVector: 587 GetSplitVector(N->getOperand(0), VLo, VHi); 588 break; 589 case WidenVector: { 590 // If the result needs to be split and the input needs to be widened, 591 // the two types must have different lengths. Use the widened result 592 // and extract from it to do the split. 593 SDValue InOp = GetWidenedVector(N->getOperand(0)); 594 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 595 LoVT.getVectorNumElements()); 596 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 597 DAG.getIntPtrConstant(0)); 598 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 599 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 600 break; 601 } 602 } 603 604 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 605 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 606 607 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 608 CvtCode); 609 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 610 CvtCode); 611} 612 613void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 614 SDValue &Hi) { 615 SDValue Vec = N->getOperand(0); 616 SDValue Idx = N->getOperand(1); 617 MVT IdxVT = Idx.getValueType(); 618 DebugLoc dl = N->getDebugLoc(); 619 620 MVT LoVT, HiVT; 621 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 622 623 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 624 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 625 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 626 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 627} 628 629void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 630 SDValue &Hi) { 631 DebugLoc dl = N->getDebugLoc(); 632 GetSplitVector(N->getOperand(0), Lo, Hi); 633 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 634 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 635} 636 637void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 638 SDValue &Hi) { 639 SDValue Vec = N->getOperand(0); 640 SDValue Elt = N->getOperand(1); 641 SDValue Idx = N->getOperand(2); 642 DebugLoc dl = N->getDebugLoc(); 643 GetSplitVector(Vec, Lo, Hi); 644 645 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 646 unsigned IdxVal = CIdx->getZExtValue(); 647 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 648 if (IdxVal < LoNumElts) 649 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 650 Lo.getValueType(), Lo, Elt, Idx); 651 else 652 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 653 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 654 return; 655 } 656 657 // Spill the vector to the stack. 658 MVT VecVT = Vec.getValueType(); 659 MVT EltVT = VecVT.getVectorElementType(); 660 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 661 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 662 663 // Store the new element. This may be larger than the vector element type, 664 // so use a truncating store. 665 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 666 unsigned Alignment = 667 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForMVT( 668 *DAG.getContext())); 669 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 670 671 // Load the Lo part from the stack slot. 672 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 673 674 // Increment the pointer to the other part. 675 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 676 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 677 DAG.getIntPtrConstant(IncrementSize)); 678 679 // Load the Hi part from the stack slot. 680 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 681 MinAlign(Alignment, IncrementSize)); 682} 683 684void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 685 SDValue &Hi) { 686 MVT LoVT, HiVT; 687 DebugLoc dl = N->getDebugLoc(); 688 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 689 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 690 Hi = DAG.getUNDEF(HiVT); 691} 692 693void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 694 SDValue &Hi) { 695 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 696 MVT LoVT, HiVT; 697 DebugLoc dl = LD->getDebugLoc(); 698 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 699 700 ISD::LoadExtType ExtType = LD->getExtensionType(); 701 SDValue Ch = LD->getChain(); 702 SDValue Ptr = LD->getBasePtr(); 703 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 704 const Value *SV = LD->getSrcValue(); 705 int SVOffset = LD->getSrcValueOffset(); 706 MVT MemoryVT = LD->getMemoryVT(); 707 unsigned Alignment = LD->getAlignment(); 708 bool isVolatile = LD->isVolatile(); 709 710 MVT LoMemVT, HiMemVT; 711 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 712 713 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 714 SV, SVOffset, LoMemVT, isVolatile, Alignment); 715 716 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 717 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 718 DAG.getIntPtrConstant(IncrementSize)); 719 SVOffset += IncrementSize; 720 Alignment = MinAlign(Alignment, IncrementSize); 721 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 722 SV, SVOffset, HiMemVT, isVolatile, Alignment); 723 724 // Build a factor node to remember that this load is independent of the 725 // other one. 726 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 727 Hi.getValue(1)); 728 729 // Legalized the chain result - switch anything that used the old chain to 730 // use the new one. 731 ReplaceValueWith(SDValue(LD, 1), Ch); 732} 733 734void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 735 MVT LoVT, HiVT; 736 DebugLoc DL = N->getDebugLoc(); 737 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 738 739 // Split the input. 740 MVT InVT = N->getOperand(0).getValueType(); 741 SDValue LL, LH, RL, RH; 742 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 743 LoVT.getVectorNumElements()); 744 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 745 DAG.getIntPtrConstant(0)); 746 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 747 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 748 749 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 750 DAG.getIntPtrConstant(0)); 751 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 752 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 753 754 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 755 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 756} 757 758void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 759 SDValue &Hi) { 760 // Get the dest types - they may not match the input types, e.g. int_to_fp. 761 MVT LoVT, HiVT; 762 DebugLoc dl = N->getDebugLoc(); 763 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 764 765 // Split the input. 766 MVT InVT = N->getOperand(0).getValueType(); 767 switch (getTypeAction(InVT)) { 768 default: assert(0 && "Unexpected type action!"); 769 case Legal: { 770 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 771 LoVT.getVectorNumElements()); 772 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 773 DAG.getIntPtrConstant(0)); 774 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 775 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 776 break; 777 } 778 case SplitVector: 779 GetSplitVector(N->getOperand(0), Lo, Hi); 780 break; 781 case WidenVector: { 782 // If the result needs to be split and the input needs to be widened, 783 // the two types must have different lengths. Use the widened result 784 // and extract from it to do the split. 785 SDValue InOp = GetWidenedVector(N->getOperand(0)); 786 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 787 LoVT.getVectorNumElements()); 788 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 789 DAG.getIntPtrConstant(0)); 790 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 791 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 792 break; 793 } 794 } 795 796 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 797 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 798} 799 800void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 801 SDValue &Lo, SDValue &Hi) { 802 // The low and high parts of the original input give four input vectors. 803 SDValue Inputs[4]; 804 DebugLoc dl = N->getDebugLoc(); 805 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 806 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 807 MVT NewVT = Inputs[0].getValueType(); 808 unsigned NewElts = NewVT.getVectorNumElements(); 809 810 // If Lo or Hi uses elements from at most two of the four input vectors, then 811 // express it as a vector shuffle of those two inputs. Otherwise extract the 812 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 813 SmallVector<int, 16> Ops; 814 for (unsigned High = 0; High < 2; ++High) { 815 SDValue &Output = High ? Hi : Lo; 816 817 // Build a shuffle mask for the output, discovering on the fly which 818 // input vectors to use as shuffle operands (recorded in InputUsed). 819 // If building a suitable shuffle vector proves too hard, then bail 820 // out with useBuildVector set. 821 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 822 unsigned FirstMaskIdx = High * NewElts; 823 bool useBuildVector = false; 824 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 825 // The mask element. This indexes into the input. 826 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 827 828 // The input vector this mask element indexes into. 829 unsigned Input = (unsigned)Idx / NewElts; 830 831 if (Input >= array_lengthof(Inputs)) { 832 // The mask element does not index into any input vector. 833 Ops.push_back(-1); 834 continue; 835 } 836 837 // Turn the index into an offset from the start of the input vector. 838 Idx -= Input * NewElts; 839 840 // Find or create a shuffle vector operand to hold this input. 841 unsigned OpNo; 842 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 843 if (InputUsed[OpNo] == Input) { 844 // This input vector is already an operand. 845 break; 846 } else if (InputUsed[OpNo] == -1U) { 847 // Create a new operand for this input vector. 848 InputUsed[OpNo] = Input; 849 break; 850 } 851 } 852 853 if (OpNo >= array_lengthof(InputUsed)) { 854 // More than two input vectors used! Give up on trying to create a 855 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 856 useBuildVector = true; 857 break; 858 } 859 860 // Add the mask index for the new shuffle vector. 861 Ops.push_back(Idx + OpNo * NewElts); 862 } 863 864 if (useBuildVector) { 865 MVT EltVT = NewVT.getVectorElementType(); 866 SmallVector<SDValue, 16> SVOps; 867 868 // Extract the input elements by hand. 869 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 870 // The mask element. This indexes into the input. 871 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 872 873 // The input vector this mask element indexes into. 874 unsigned Input = (unsigned)Idx / NewElts; 875 876 if (Input >= array_lengthof(Inputs)) { 877 // The mask element is "undef" or indexes off the end of the input. 878 SVOps.push_back(DAG.getUNDEF(EltVT)); 879 continue; 880 } 881 882 // Turn the index into an offset from the start of the input vector. 883 Idx -= Input * NewElts; 884 885 // Extract the vector element by hand. 886 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 887 Inputs[Input], DAG.getIntPtrConstant(Idx))); 888 } 889 890 // Construct the Lo/Hi output using a BUILD_VECTOR. 891 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 892 } else if (InputUsed[0] == -1U) { 893 // No input vectors were used! The result is undefined. 894 Output = DAG.getUNDEF(NewVT); 895 } else { 896 SDValue Op0 = Inputs[InputUsed[0]]; 897 // If only one input was used, use an undefined vector for the other. 898 SDValue Op1 = InputUsed[1] == -1U ? 899 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 900 // At least one input vector was used. Create a new shuffle vector. 901 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 902 } 903 904 Ops.clear(); 905 } 906} 907 908 909//===----------------------------------------------------------------------===// 910// Operand Vector Splitting 911//===----------------------------------------------------------------------===// 912 913/// SplitVectorOperand - This method is called when the specified operand of the 914/// specified node is found to need vector splitting. At this point, all of the 915/// result types of the node are known to be legal, but other operands of the 916/// node may need legalization as well as the specified one. 917bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 918 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n"); 919 SDValue Res = SDValue(); 920 921 if (Res.getNode() == 0) { 922 switch (N->getOpcode()) { 923 default: 924#ifndef NDEBUG 925 cerr << "SplitVectorOperand Op #" << OpNo << ": "; 926 N->dump(&DAG); cerr << "\n"; 927#endif 928 assert(0 && "Do not know how to split this operator's operand!"); 929 abort(); 930 931 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 932 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 933 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 934 case ISD::STORE: 935 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 936 break; 937 938 case ISD::CTTZ: 939 case ISD::CTLZ: 940 case ISD::CTPOP: 941 case ISD::FP_TO_SINT: 942 case ISD::FP_TO_UINT: 943 case ISD::SINT_TO_FP: 944 case ISD::UINT_TO_FP: 945 case ISD::TRUNCATE: 946 case ISD::SIGN_EXTEND: 947 case ISD::ZERO_EXTEND: 948 case ISD::ANY_EXTEND: 949 Res = SplitVecOp_UnaryOp(N); 950 break; 951 } 952 } 953 954 // If the result is null, the sub-method took care of registering results etc. 955 if (!Res.getNode()) return false; 956 957 // If the result is N, the sub-method updated N in place. Tell the legalizer 958 // core about this. 959 if (Res.getNode() == N) 960 return true; 961 962 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 963 "Invalid operand expansion"); 964 965 ReplaceValueWith(SDValue(N, 0), Res); 966 return false; 967} 968 969SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 970 // The result has a legal vector type, but the input needs splitting. 971 MVT ResVT = N->getValueType(0); 972 SDValue Lo, Hi; 973 DebugLoc dl = N->getDebugLoc(); 974 GetSplitVector(N->getOperand(0), Lo, Hi); 975 MVT InVT = Lo.getValueType(); 976 977 MVT OutVT = MVT::getVectorVT(ResVT.getVectorElementType(), 978 InVT.getVectorNumElements()); 979 980 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 981 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 982 983 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 984} 985 986SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 987 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 988 // end up being split all the way down to individual components. Convert the 989 // split pieces into integers and reassemble. 990 SDValue Lo, Hi; 991 GetSplitVector(N->getOperand(0), Lo, Hi); 992 Lo = BitConvertToInteger(Lo); 993 Hi = BitConvertToInteger(Hi); 994 995 if (TLI.isBigEndian()) 996 std::swap(Lo, Hi); 997 998 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 999 JoinIntegers(Lo, Hi)); 1000} 1001 1002SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1003 // We know that the extracted result type is legal. For now, assume the index 1004 // is a constant. 1005 MVT SubVT = N->getValueType(0); 1006 SDValue Idx = N->getOperand(1); 1007 DebugLoc dl = N->getDebugLoc(); 1008 SDValue Lo, Hi; 1009 GetSplitVector(N->getOperand(0), Lo, Hi); 1010 1011 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1012 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1013 1014 if (IdxVal < LoElts) { 1015 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1016 "Extracted subvector crosses vector split!"); 1017 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1018 } else { 1019 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1020 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1021 } 1022} 1023 1024SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1025 SDValue Vec = N->getOperand(0); 1026 SDValue Idx = N->getOperand(1); 1027 MVT VecVT = Vec.getValueType(); 1028 1029 if (isa<ConstantSDNode>(Idx)) { 1030 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1031 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1032 1033 SDValue Lo, Hi; 1034 GetSplitVector(Vec, Lo, Hi); 1035 1036 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1037 1038 if (IdxVal < LoElts) 1039 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1040 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1041 DAG.getConstant(IdxVal - LoElts, 1042 Idx.getValueType())); 1043 } 1044 1045 // Store the vector to the stack. 1046 MVT EltVT = VecVT.getVectorElementType(); 1047 DebugLoc dl = N->getDebugLoc(); 1048 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1049 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1050 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1051 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1052 1053 // Load back the required element. 1054 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1055 return DAG.getLoad(EltVT, dl, Store, StackPtr, SV, 0); 1056} 1057 1058SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1059 assert(N->isUnindexed() && "Indexed store of vector?"); 1060 assert(OpNo == 1 && "Can only split the stored value"); 1061 DebugLoc dl = N->getDebugLoc(); 1062 1063 bool isTruncating = N->isTruncatingStore(); 1064 SDValue Ch = N->getChain(); 1065 SDValue Ptr = N->getBasePtr(); 1066 int SVOffset = N->getSrcValueOffset(); 1067 MVT MemoryVT = N->getMemoryVT(); 1068 unsigned Alignment = N->getAlignment(); 1069 bool isVol = N->isVolatile(); 1070 SDValue Lo, Hi; 1071 GetSplitVector(N->getOperand(1), Lo, Hi); 1072 1073 MVT LoMemVT, HiMemVT; 1074 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1075 1076 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1077 1078 if (isTruncating) 1079 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1080 LoMemVT, isVol, Alignment); 1081 else 1082 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1083 isVol, Alignment); 1084 1085 // Increment the pointer to the other half. 1086 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1087 DAG.getIntPtrConstant(IncrementSize)); 1088 1089 if (isTruncating) 1090 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 1091 N->getSrcValue(), SVOffset+IncrementSize, 1092 HiMemVT, 1093 isVol, MinAlign(Alignment, IncrementSize)); 1094 else 1095 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 1096 isVol, MinAlign(Alignment, IncrementSize)); 1097 1098 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1099} 1100 1101 1102//===----------------------------------------------------------------------===// 1103// Result Vector Widening 1104//===----------------------------------------------------------------------===// 1105 1106void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1107 DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG); 1108 cerr << "\n"); 1109 SDValue Res = SDValue(); 1110 1111 switch (N->getOpcode()) { 1112 default: 1113#ifndef NDEBUG 1114 cerr << "WidenVectorResult #" << ResNo << ": "; 1115 N->dump(&DAG); cerr << "\n"; 1116#endif 1117 assert(0 && "Do not know how to widen the result of this operator!"); 1118 abort(); 1119 1120 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1121 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1122 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1123 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1124 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1125 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1126 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1127 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1128 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1129 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1130 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1131 case ISD::VECTOR_SHUFFLE: 1132 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1133 break; 1134 case ISD::VSETCC: 1135 Res = WidenVecRes_VSETCC(N); 1136 break; 1137 1138 case ISD::ADD: 1139 case ISD::AND: 1140 case ISD::BSWAP: 1141 case ISD::FADD: 1142 case ISD::FCOPYSIGN: 1143 case ISD::FDIV: 1144 case ISD::FMUL: 1145 case ISD::FPOW: 1146 case ISD::FPOWI: 1147 case ISD::FREM: 1148 case ISD::FSUB: 1149 case ISD::MUL: 1150 case ISD::MULHS: 1151 case ISD::MULHU: 1152 case ISD::OR: 1153 case ISD::SDIV: 1154 case ISD::SREM: 1155 case ISD::UDIV: 1156 case ISD::UREM: 1157 case ISD::SUB: 1158 case ISD::XOR: 1159 Res = WidenVecRes_Binary(N); 1160 break; 1161 1162 case ISD::SHL: 1163 case ISD::SRA: 1164 case ISD::SRL: 1165 Res = WidenVecRes_Shift(N); 1166 break; 1167 1168 case ISD::FP_ROUND: 1169 case ISD::FP_TO_SINT: 1170 case ISD::FP_TO_UINT: 1171 case ISD::SINT_TO_FP: 1172 case ISD::UINT_TO_FP: 1173 case ISD::TRUNCATE: 1174 case ISD::SIGN_EXTEND: 1175 case ISD::ZERO_EXTEND: 1176 case ISD::ANY_EXTEND: 1177 Res = WidenVecRes_Convert(N); 1178 break; 1179 1180 case ISD::CTLZ: 1181 case ISD::CTPOP: 1182 case ISD::CTTZ: 1183 case ISD::FABS: 1184 case ISD::FCOS: 1185 case ISD::FNEG: 1186 case ISD::FSIN: 1187 case ISD::FSQRT: 1188 Res = WidenVecRes_Unary(N); 1189 break; 1190 } 1191 1192 // If Res is null, the sub-method took care of registering the result. 1193 if (Res.getNode()) 1194 SetWidenedVector(SDValue(N, ResNo), Res); 1195} 1196 1197SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1198 // Binary op widening. 1199 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1200 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1201 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1202 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1203} 1204 1205SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1206 SDValue InOp = N->getOperand(0); 1207 DebugLoc dl = N->getDebugLoc(); 1208 1209 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1210 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1211 1212 MVT InVT = InOp.getValueType(); 1213 MVT InEltVT = InVT.getVectorElementType(); 1214 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts); 1215 1216 unsigned Opcode = N->getOpcode(); 1217 unsigned InVTNumElts = InVT.getVectorNumElements(); 1218 1219 if (getTypeAction(InVT) == WidenVector) { 1220 InOp = GetWidenedVector(N->getOperand(0)); 1221 InVT = InOp.getValueType(); 1222 InVTNumElts = InVT.getVectorNumElements(); 1223 if (InVTNumElts == WidenNumElts) 1224 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1225 } 1226 1227 if (TLI.isTypeLegal(InWidenVT)) { 1228 // Because the result and the input are different vector types, widening 1229 // the result could create a legal type but widening the input might make 1230 // it an illegal type that might lead to repeatedly splitting the input 1231 // and then widening it. To avoid this, we widen the input only if 1232 // it results in a legal type. 1233 if (WidenNumElts % InVTNumElts == 0) { 1234 // Widen the input and call convert on the widened input vector. 1235 unsigned NumConcat = WidenNumElts/InVTNumElts; 1236 SmallVector<SDValue, 16> Ops(NumConcat); 1237 Ops[0] = InOp; 1238 SDValue UndefVal = DAG.getUNDEF(InVT); 1239 for (unsigned i = 1; i != NumConcat; ++i) 1240 Ops[i] = UndefVal; 1241 return DAG.getNode(Opcode, dl, WidenVT, 1242 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1243 &Ops[0], NumConcat)); 1244 } 1245 1246 if (InVTNumElts % WidenNumElts == 0) { 1247 // Extract the input and convert the shorten input vector. 1248 return DAG.getNode(Opcode, dl, WidenVT, 1249 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1250 InOp, DAG.getIntPtrConstant(0))); 1251 } 1252 } 1253 1254 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1255 SmallVector<SDValue, 16> Ops(WidenNumElts); 1256 MVT EltVT = WidenVT.getVectorElementType(); 1257 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1258 unsigned i; 1259 for (i=0; i < MinElts; ++i) 1260 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1261 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1262 DAG.getIntPtrConstant(i))); 1263 1264 SDValue UndefVal = DAG.getUNDEF(EltVT); 1265 for (; i < WidenNumElts; ++i) 1266 Ops[i] = UndefVal; 1267 1268 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1269} 1270 1271SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1272 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1273 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1274 SDValue ShOp = N->getOperand(1); 1275 1276 MVT ShVT = ShOp.getValueType(); 1277 if (getTypeAction(ShVT) == WidenVector) { 1278 ShOp = GetWidenedVector(ShOp); 1279 ShVT = ShOp.getValueType(); 1280 } 1281 MVT ShWidenVT = MVT::getVectorVT(ShVT.getVectorElementType(), 1282 WidenVT.getVectorNumElements()); 1283 if (ShVT != ShWidenVT) 1284 ShOp = ModifyToType(ShOp, ShWidenVT); 1285 1286 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1287} 1288 1289SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1290 // Unary op widening. 1291 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1292 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1293 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1294} 1295 1296SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1297 SDValue InOp = N->getOperand(0); 1298 MVT InVT = InOp.getValueType(); 1299 MVT VT = N->getValueType(0); 1300 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1301 DebugLoc dl = N->getDebugLoc(); 1302 1303 switch (getTypeAction(InVT)) { 1304 default: 1305 assert(false && "Unknown type action!"); 1306 break; 1307 case Legal: 1308 break; 1309 case PromoteInteger: 1310 // If the InOp is promoted to the same size, convert it. Otherwise, 1311 // fall out of the switch and widen the promoted input. 1312 InOp = GetPromotedInteger(InOp); 1313 InVT = InOp.getValueType(); 1314 if (WidenVT.bitsEq(InVT)) 1315 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1316 break; 1317 case SoftenFloat: 1318 case ExpandInteger: 1319 case ExpandFloat: 1320 case ScalarizeVector: 1321 case SplitVector: 1322 break; 1323 case WidenVector: 1324 // If the InOp is widened to the same size, convert it. Otherwise, fall 1325 // out of the switch and widen the widened input. 1326 InOp = GetWidenedVector(InOp); 1327 InVT = InOp.getValueType(); 1328 if (WidenVT.bitsEq(InVT)) 1329 // The input widens to the same size. Convert to the widen value. 1330 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1331 break; 1332 } 1333 1334 unsigned WidenSize = WidenVT.getSizeInBits(); 1335 unsigned InSize = InVT.getSizeInBits(); 1336 if (WidenSize % InSize == 0) { 1337 // Determine new input vector type. The new input vector type will use 1338 // the same element type (if its a vector) or use the input type as a 1339 // vector. It is the same size as the type to widen to. 1340 MVT NewInVT; 1341 unsigned NewNumElts = WidenSize / InSize; 1342 if (InVT.isVector()) { 1343 MVT InEltVT = InVT.getVectorElementType(); 1344 NewInVT= MVT::getVectorVT(InEltVT, WidenSize / InEltVT.getSizeInBits()); 1345 } else { 1346 NewInVT = MVT::getVectorVT(InVT, NewNumElts); 1347 } 1348 1349 if (TLI.isTypeLegal(NewInVT)) { 1350 // Because the result and the input are different vector types, widening 1351 // the result could create a legal type but widening the input might make 1352 // it an illegal type that might lead to repeatedly splitting the input 1353 // and then widening it. To avoid this, we widen the input only if 1354 // it results in a legal type. 1355 SmallVector<SDValue, 16> Ops(NewNumElts); 1356 SDValue UndefVal = DAG.getUNDEF(InVT); 1357 Ops[0] = InOp; 1358 for (unsigned i = 1; i < NewNumElts; ++i) 1359 Ops[i] = UndefVal; 1360 1361 SDValue NewVec; 1362 if (InVT.isVector()) 1363 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1364 NewInVT, &Ops[0], NewNumElts); 1365 else 1366 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1367 NewInVT, &Ops[0], NewNumElts); 1368 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1369 } 1370 } 1371 1372 // This should occur rarely. Lower the bit-convert to a store/load 1373 // from the stack. Create the stack frame object. Make sure it is aligned 1374 // for both the source and destination types. 1375 SDValue FIPtr = DAG.CreateStackTemporary(InVT, WidenVT); 1376 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1377 const Value *SV = PseudoSourceValue::getFixedStack(FI); 1378 1379 // Emit a store to the stack slot. 1380 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0); 1381 1382 // Result is a load from the stack slot. 1383 return DAG.getLoad(WidenVT, dl, Store, FIPtr, SV, 0); 1384} 1385 1386SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1387 DebugLoc dl = N->getDebugLoc(); 1388 // Build a vector with undefined for the new nodes. 1389 MVT VT = N->getValueType(0); 1390 MVT EltVT = VT.getVectorElementType(); 1391 unsigned NumElts = VT.getVectorNumElements(); 1392 1393 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1394 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1395 1396 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1397 NewOps.reserve(WidenNumElts); 1398 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1399 NewOps.push_back(DAG.getUNDEF(EltVT)); 1400 1401 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1402} 1403 1404SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1405 MVT InVT = N->getOperand(0).getValueType(); 1406 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1407 DebugLoc dl = N->getDebugLoc(); 1408 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1409 unsigned NumOperands = N->getNumOperands(); 1410 1411 bool InputWidened = false; // Indicates we need to widen the input. 1412 if (getTypeAction(InVT) != WidenVector) { 1413 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1414 // Add undef vectors to widen to correct length. 1415 unsigned NumConcat = WidenVT.getVectorNumElements() / 1416 InVT.getVectorNumElements(); 1417 SDValue UndefVal = DAG.getUNDEF(InVT); 1418 SmallVector<SDValue, 16> Ops(NumConcat); 1419 for (unsigned i=0; i < NumOperands; ++i) 1420 Ops[i] = N->getOperand(i); 1421 for (unsigned i = NumOperands; i != NumConcat; ++i) 1422 Ops[i] = UndefVal; 1423 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1424 } 1425 } else { 1426 InputWidened = true; 1427 if (WidenVT == TLI.getTypeToTransformTo(InVT)) { 1428 // The inputs and the result are widen to the same value. 1429 unsigned i; 1430 for (i=1; i < NumOperands; ++i) 1431 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1432 break; 1433 1434 if (i > NumOperands) 1435 // Everything but the first operand is an UNDEF so just return the 1436 // widened first operand. 1437 return GetWidenedVector(N->getOperand(0)); 1438 1439 if (NumOperands == 2) { 1440 // Replace concat of two operands with a shuffle. 1441 SmallVector<int, 16> MaskOps(WidenNumElts); 1442 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1443 MaskOps[i] = i; 1444 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1445 } 1446 return DAG.getVectorShuffle(WidenVT, dl, 1447 GetWidenedVector(N->getOperand(0)), 1448 GetWidenedVector(N->getOperand(1)), 1449 &MaskOps[0]); 1450 } 1451 } 1452 } 1453 1454 // Fall back to use extracts and build vector. 1455 MVT EltVT = WidenVT.getVectorElementType(); 1456 unsigned NumInElts = InVT.getVectorNumElements(); 1457 SmallVector<SDValue, 16> Ops(WidenNumElts); 1458 unsigned Idx = 0; 1459 for (unsigned i=0; i < NumOperands; ++i) { 1460 SDValue InOp = N->getOperand(i); 1461 if (InputWidened) 1462 InOp = GetWidenedVector(InOp); 1463 for (unsigned j=0; j < NumInElts; ++j) 1464 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1465 DAG.getIntPtrConstant(j)); 1466 } 1467 SDValue UndefVal = DAG.getUNDEF(EltVT); 1468 for (; Idx < WidenNumElts; ++Idx) 1469 Ops[Idx] = UndefVal; 1470 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1471} 1472 1473SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1474 DebugLoc dl = N->getDebugLoc(); 1475 SDValue InOp = N->getOperand(0); 1476 SDValue RndOp = N->getOperand(3); 1477 SDValue SatOp = N->getOperand(4); 1478 1479 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1480 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1481 1482 MVT InVT = InOp.getValueType(); 1483 MVT InEltVT = InVT.getVectorElementType(); 1484 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts); 1485 1486 SDValue DTyOp = DAG.getValueType(WidenVT); 1487 SDValue STyOp = DAG.getValueType(InWidenVT); 1488 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1489 1490 unsigned InVTNumElts = InVT.getVectorNumElements(); 1491 if (getTypeAction(InVT) == WidenVector) { 1492 InOp = GetWidenedVector(InOp); 1493 InVT = InOp.getValueType(); 1494 InVTNumElts = InVT.getVectorNumElements(); 1495 if (InVTNumElts == WidenNumElts) 1496 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1497 SatOp, CvtCode); 1498 } 1499 1500 if (TLI.isTypeLegal(InWidenVT)) { 1501 // Because the result and the input are different vector types, widening 1502 // the result could create a legal type but widening the input might make 1503 // it an illegal type that might lead to repeatedly splitting the input 1504 // and then widening it. To avoid this, we widen the input only if 1505 // it results in a legal type. 1506 if (WidenNumElts % InVTNumElts == 0) { 1507 // Widen the input and call convert on the widened input vector. 1508 unsigned NumConcat = WidenNumElts/InVTNumElts; 1509 SmallVector<SDValue, 16> Ops(NumConcat); 1510 Ops[0] = InOp; 1511 SDValue UndefVal = DAG.getUNDEF(InVT); 1512 for (unsigned i = 1; i != NumConcat; ++i) { 1513 Ops[i] = UndefVal; 1514 } 1515 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1516 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1517 SatOp, CvtCode); 1518 } 1519 1520 if (InVTNumElts % WidenNumElts == 0) { 1521 // Extract the input and convert the shorten input vector. 1522 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1523 DAG.getIntPtrConstant(0)); 1524 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1525 SatOp, CvtCode); 1526 } 1527 } 1528 1529 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1530 SmallVector<SDValue, 16> Ops(WidenNumElts); 1531 MVT EltVT = WidenVT.getVectorElementType(); 1532 DTyOp = DAG.getValueType(EltVT); 1533 STyOp = DAG.getValueType(InEltVT); 1534 1535 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1536 unsigned i; 1537 for (i=0; i < MinElts; ++i) { 1538 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1539 DAG.getIntPtrConstant(i)); 1540 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1541 SatOp, CvtCode); 1542 } 1543 1544 SDValue UndefVal = DAG.getUNDEF(EltVT); 1545 for (; i < WidenNumElts; ++i) 1546 Ops[i] = UndefVal; 1547 1548 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1549} 1550 1551SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1552 MVT VT = N->getValueType(0); 1553 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1554 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1555 SDValue InOp = N->getOperand(0); 1556 SDValue Idx = N->getOperand(1); 1557 DebugLoc dl = N->getDebugLoc(); 1558 1559 if (getTypeAction(InOp.getValueType()) == WidenVector) 1560 InOp = GetWidenedVector(InOp); 1561 1562 MVT InVT = InOp.getValueType(); 1563 1564 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1565 if (CIdx) { 1566 unsigned IdxVal = CIdx->getZExtValue(); 1567 // Check if we can just return the input vector after widening. 1568 if (IdxVal == 0 && InVT == WidenVT) 1569 return InOp; 1570 1571 // Check if we can extract from the vector. 1572 unsigned InNumElts = InVT.getVectorNumElements(); 1573 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1574 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1575 } 1576 1577 // We could try widening the input to the right length but for now, extract 1578 // the original elements, fill the rest with undefs and build a vector. 1579 SmallVector<SDValue, 16> Ops(WidenNumElts); 1580 MVT EltVT = VT.getVectorElementType(); 1581 MVT IdxVT = Idx.getValueType(); 1582 unsigned NumElts = VT.getVectorNumElements(); 1583 unsigned i; 1584 if (CIdx) { 1585 unsigned IdxVal = CIdx->getZExtValue(); 1586 for (i=0; i < NumElts; ++i) 1587 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1588 DAG.getConstant(IdxVal+i, IdxVT)); 1589 } else { 1590 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1591 for (i=1; i < NumElts; ++i) { 1592 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1593 DAG.getConstant(i, IdxVT)); 1594 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1595 } 1596 } 1597 1598 SDValue UndefVal = DAG.getUNDEF(EltVT); 1599 for (; i < WidenNumElts; ++i) 1600 Ops[i] = UndefVal; 1601 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1602} 1603 1604SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1605 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1606 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1607 InOp.getValueType(), InOp, 1608 N->getOperand(1), N->getOperand(2)); 1609} 1610 1611SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1612 LoadSDNode *LD = cast<LoadSDNode>(N); 1613 MVT WidenVT = TLI.getTypeToTransformTo(LD->getValueType(0)); 1614 MVT LdVT = LD->getMemoryVT(); 1615 DebugLoc dl = N->getDebugLoc(); 1616 assert(LdVT.isVector() && WidenVT.isVector()); 1617 1618 // Load information 1619 SDValue Chain = LD->getChain(); 1620 SDValue BasePtr = LD->getBasePtr(); 1621 int SVOffset = LD->getSrcValueOffset(); 1622 unsigned Align = LD->getAlignment(); 1623 bool isVolatile = LD->isVolatile(); 1624 const Value *SV = LD->getSrcValue(); 1625 ISD::LoadExtType ExtType = LD->getExtensionType(); 1626 1627 SDValue Result; 1628 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1629 if (ExtType != ISD::NON_EXTLOAD) { 1630 // For extension loads, we can not play the tricks of chopping legal 1631 // vector types and bit cast it to the right type. Instead, we unroll 1632 // the load and build a vector. 1633 MVT EltVT = WidenVT.getVectorElementType(); 1634 MVT LdEltVT = LdVT.getVectorElementType(); 1635 unsigned NumElts = LdVT.getVectorNumElements(); 1636 1637 // Load each element and widen 1638 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1639 SmallVector<SDValue, 16> Ops(WidenNumElts); 1640 unsigned Increment = LdEltVT.getSizeInBits() / 8; 1641 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 1642 LdEltVT, isVolatile, Align); 1643 LdChain.push_back(Ops[0].getValue(1)); 1644 unsigned i = 0, Offset = Increment; 1645 for (i=1; i < NumElts; ++i, Offset += Increment) { 1646 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1647 BasePtr, DAG.getIntPtrConstant(Offset)); 1648 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 1649 SVOffset + Offset, LdEltVT, isVolatile, Align); 1650 LdChain.push_back(Ops[i].getValue(1)); 1651 } 1652 1653 // Fill the rest with undefs 1654 SDValue UndefVal = DAG.getUNDEF(EltVT); 1655 for (; i != WidenNumElts; ++i) 1656 Ops[i] = UndefVal; 1657 1658 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 1659 } else { 1660 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 1661 unsigned int LdWidth = LdVT.getSizeInBits(); 1662 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 1663 Align, isVolatile, LdWidth, WidenVT, dl); 1664 } 1665 1666 // If we generate a single load, we can use that for the chain. Otherwise, 1667 // build a factor node to remember the multiple loads are independent and 1668 // chain to that. 1669 SDValue NewChain; 1670 if (LdChain.size() == 1) 1671 NewChain = LdChain[0]; 1672 else 1673 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 1674 LdChain.size()); 1675 1676 // Modified the chain - switch anything that used the old chain to use 1677 // the new one. 1678 ReplaceValueWith(SDValue(N, 1), Chain); 1679 1680 return Result; 1681} 1682 1683SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1684 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1685 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1686 WidenVT, N->getOperand(0)); 1687} 1688 1689SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1690 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1691 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1692 1693 SDValue Cond1 = N->getOperand(0); 1694 MVT CondVT = Cond1.getValueType(); 1695 if (CondVT.isVector()) { 1696 MVT CondEltVT = CondVT.getVectorElementType(); 1697 MVT CondWidenVT = MVT::getVectorVT(CondEltVT, WidenNumElts); 1698 if (getTypeAction(CondVT) == WidenVector) 1699 Cond1 = GetWidenedVector(Cond1); 1700 1701 if (Cond1.getValueType() != CondWidenVT) 1702 Cond1 = ModifyToType(Cond1, CondWidenVT); 1703 } 1704 1705 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1706 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1707 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1708 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1709 WidenVT, Cond1, InOp1, InOp2); 1710} 1711 1712SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1713 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1714 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1715 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1716 InOp1.getValueType(), N->getOperand(0), 1717 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1718} 1719 1720SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1721 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1722 return DAG.getUNDEF(WidenVT); 1723} 1724 1725SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1726 MVT VT = N->getValueType(0); 1727 DebugLoc dl = N->getDebugLoc(); 1728 1729 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1730 unsigned NumElts = VT.getVectorNumElements(); 1731 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1732 1733 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1734 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1735 1736 // Adjust mask based on new input vector length. 1737 SmallVector<int, 16> NewMask; 1738 for (unsigned i = 0; i != NumElts; ++i) { 1739 int Idx = N->getMaskElt(i); 1740 if (Idx < (int)NumElts) 1741 NewMask.push_back(Idx); 1742 else 1743 NewMask.push_back(Idx - NumElts + WidenNumElts); 1744 } 1745 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1746 NewMask.push_back(-1); 1747 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1748} 1749 1750SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1751 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1752 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1753 1754 SDValue InOp1 = N->getOperand(0); 1755 MVT InVT = InOp1.getValueType(); 1756 assert(InVT.isVector() && "can not widen non vector type"); 1757 MVT WidenInVT = MVT::getVectorVT(InVT.getVectorElementType(), WidenNumElts); 1758 InOp1 = GetWidenedVector(InOp1); 1759 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1760 1761 // Assume that the input and output will be widen appropriately. If not, 1762 // we will have to unroll it at some point. 1763 assert(InOp1.getValueType() == WidenInVT && 1764 InOp2.getValueType() == WidenInVT && 1765 "Input not widened to expected type!"); 1766 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1767 WidenVT, InOp1, InOp2, N->getOperand(2)); 1768} 1769 1770 1771//===----------------------------------------------------------------------===// 1772// Widen Vector Operand 1773//===----------------------------------------------------------------------===// 1774bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1775 DEBUG(cerr << "Widen node operand " << ResNo << ": "; N->dump(&DAG); 1776 cerr << "\n"); 1777 SDValue Res = SDValue(); 1778 1779 switch (N->getOpcode()) { 1780 default: 1781#ifndef NDEBUG 1782 cerr << "WidenVectorOperand op #" << ResNo << ": "; 1783 N->dump(&DAG); cerr << "\n"; 1784#endif 1785 assert(0 && "Do not know how to widen this operator's operand!"); 1786 abort(); 1787 1788 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1789 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1790 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1791 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1792 1793 case ISD::FP_ROUND: 1794 case ISD::FP_TO_SINT: 1795 case ISD::FP_TO_UINT: 1796 case ISD::SINT_TO_FP: 1797 case ISD::UINT_TO_FP: 1798 case ISD::TRUNCATE: 1799 case ISD::SIGN_EXTEND: 1800 case ISD::ZERO_EXTEND: 1801 case ISD::ANY_EXTEND: 1802 Res = WidenVecOp_Convert(N); 1803 break; 1804 } 1805 1806 // If Res is null, the sub-method took care of registering the result. 1807 if (!Res.getNode()) return false; 1808 1809 // If the result is N, the sub-method updated N in place. Tell the legalizer 1810 // core about this. 1811 if (Res.getNode() == N) 1812 return true; 1813 1814 1815 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1816 "Invalid operand expansion"); 1817 1818 ReplaceValueWith(SDValue(N, 0), Res); 1819 return false; 1820} 1821 1822SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1823 // Since the result is legal and the input is illegal, it is unlikely 1824 // that we can fix the input to a legal type so unroll the convert 1825 // into some scalar code and create a nasty build vector. 1826 MVT VT = N->getValueType(0); 1827 MVT EltVT = VT.getVectorElementType(); 1828 DebugLoc dl = N->getDebugLoc(); 1829 unsigned NumElts = VT.getVectorNumElements(); 1830 SDValue InOp = N->getOperand(0); 1831 if (getTypeAction(InOp.getValueType()) == WidenVector) 1832 InOp = GetWidenedVector(InOp); 1833 MVT InVT = InOp.getValueType(); 1834 MVT InEltVT = InVT.getVectorElementType(); 1835 1836 unsigned Opcode = N->getOpcode(); 1837 SmallVector<SDValue, 16> Ops(NumElts); 1838 for (unsigned i=0; i < NumElts; ++i) 1839 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1840 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1841 DAG.getIntPtrConstant(i))); 1842 1843 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1844} 1845 1846SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1847 MVT VT = N->getValueType(0); 1848 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1849 MVT InWidenVT = InOp.getValueType(); 1850 DebugLoc dl = N->getDebugLoc(); 1851 1852 // Check if we can convert between two legal vector types and extract. 1853 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1854 unsigned Size = VT.getSizeInBits(); 1855 if (InWidenSize % Size == 0 && !VT.isVector()) { 1856 unsigned NewNumElts = InWidenSize / Size; 1857 MVT NewVT = MVT::getVectorVT(VT, NewNumElts); 1858 if (TLI.isTypeLegal(NewVT)) { 1859 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1860 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1861 DAG.getIntPtrConstant(0)); 1862 } 1863 } 1864 1865 // Lower the bit-convert to a store/load from the stack. Create the stack 1866 // frame object. Make sure it is aligned for both the source and destination 1867 // types. 1868 SDValue FIPtr = DAG.CreateStackTemporary(InWidenVT, VT); 1869 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1870 const Value *SV = PseudoSourceValue::getFixedStack(FI); 1871 1872 // Emit a store to the stack slot. 1873 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0); 1874 1875 // Result is a load from the stack slot. 1876 return DAG.getLoad(VT, dl, Store, FIPtr, SV, 0); 1877} 1878 1879SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1880 // If the input vector is not legal, it is likely that we will not find a 1881 // legal vector of the same size. Replace the concatenate vector with a 1882 // nasty build vector. 1883 MVT VT = N->getValueType(0); 1884 MVT EltVT = VT.getVectorElementType(); 1885 DebugLoc dl = N->getDebugLoc(); 1886 unsigned NumElts = VT.getVectorNumElements(); 1887 SmallVector<SDValue, 16> Ops(NumElts); 1888 1889 MVT InVT = N->getOperand(0).getValueType(); 1890 unsigned NumInElts = InVT.getVectorNumElements(); 1891 1892 unsigned Idx = 0; 1893 unsigned NumOperands = N->getNumOperands(); 1894 for (unsigned i=0; i < NumOperands; ++i) { 1895 SDValue InOp = N->getOperand(i); 1896 if (getTypeAction(InOp.getValueType()) == WidenVector) 1897 InOp = GetWidenedVector(InOp); 1898 for (unsigned j=0; j < NumInElts; ++j) 1899 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1900 DAG.getIntPtrConstant(j)); 1901 } 1902 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1903} 1904 1905SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1906 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1907 MVT EltVT = InOp.getValueType().getVectorElementType(); 1908 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1909 EltVT, InOp, N->getOperand(1)); 1910} 1911 1912SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1913 // We have to widen the value but we want only to store the original 1914 // vector type. 1915 StoreSDNode *ST = cast<StoreSDNode>(N); 1916 SDValue Chain = ST->getChain(); 1917 SDValue BasePtr = ST->getBasePtr(); 1918 const Value *SV = ST->getSrcValue(); 1919 int SVOffset = ST->getSrcValueOffset(); 1920 unsigned Align = ST->getAlignment(); 1921 bool isVolatile = ST->isVolatile(); 1922 SDValue ValOp = GetWidenedVector(ST->getValue()); 1923 DebugLoc dl = N->getDebugLoc(); 1924 1925 MVT StVT = ST->getMemoryVT(); 1926 MVT ValVT = ValOp.getValueType(); 1927 // It must be true that we the widen vector type is bigger than where 1928 // we need to store. 1929 assert(StVT.isVector() && ValOp.getValueType().isVector()); 1930 assert(StVT.bitsLT(ValOp.getValueType())); 1931 1932 SmallVector<SDValue, 16> StChain; 1933 if (ST->isTruncatingStore()) { 1934 // For truncating stores, we can not play the tricks of chopping legal 1935 // vector types and bit cast it to the right type. Instead, we unroll 1936 // the store. 1937 MVT StEltVT = StVT.getVectorElementType(); 1938 MVT ValEltVT = ValVT.getVectorElementType(); 1939 unsigned Increment = ValEltVT.getSizeInBits() / 8; 1940 unsigned NumElts = StVT.getVectorNumElements(); 1941 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1942 DAG.getIntPtrConstant(0)); 1943 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 1944 SVOffset, StEltVT, 1945 isVolatile, Align)); 1946 unsigned Offset = Increment; 1947 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 1948 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1949 BasePtr, DAG.getIntPtrConstant(Offset)); 1950 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1951 DAG.getIntPtrConstant(0)); 1952 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 1953 SVOffset + Offset, StEltVT, 1954 isVolatile, MinAlign(Align, Offset))); 1955 } 1956 } 1957 else { 1958 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 1959 // Store value 1960 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 1961 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 1962 } 1963 if (StChain.size() == 1) 1964 return StChain[0]; 1965 else 1966 return DAG.getNode(ISD::TokenFactor, dl, 1967 MVT::Other,&StChain[0],StChain.size()); 1968} 1969 1970//===----------------------------------------------------------------------===// 1971// Vector Widening Utilities 1972//===----------------------------------------------------------------------===// 1973 1974 1975// Utility function to find a vector type and its associated element 1976// type from a preferred width and whose vector type must be the same size 1977// as the VecVT. 1978// TLI: Target lowering used to determine legal types. 1979// Width: Preferred width to store. 1980// VecVT: Vector value type whose size we must match. 1981// Returns NewVecVT and NewEltVT - the vector type and its associated 1982// element type. 1983static void FindAssocWidenVecType(const TargetLowering &TLI, unsigned Width, 1984 MVT VecVT, 1985 MVT& NewEltVT, MVT& NewVecVT) { 1986 unsigned EltWidth = Width + 1; 1987 if (TLI.isTypeLegal(VecVT)) { 1988 // We start with the preferred with, making it a power of 2 and find a 1989 // legal vector type of that width. If not, we reduce it by another of 2. 1990 // For incoming type is legal, this process will end as a vector of the 1991 // smallest loadable type should always be legal. 1992 do { 1993 assert(EltWidth > 0); 1994 EltWidth = 1 << Log2_32(EltWidth - 1); 1995 NewEltVT = MVT::getIntegerVT(EltWidth); 1996 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 1997 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts); 1998 } while (!TLI.isTypeLegal(NewVecVT) || 1999 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2000 } else { 2001 // The incoming vector type is illegal and is the result of widening 2002 // a vector to a power of 2. In this case, we will use the preferred 2003 // with as long as it is a multiple of the incoming vector length. 2004 // The legalization process will eventually make this into a legal type 2005 // and remove the illegal bit converts (which would turn to stack converts 2006 // if they are allow to exist). 2007 do { 2008 assert(EltWidth > 0); 2009 EltWidth = 1 << Log2_32(EltWidth - 1); 2010 NewEltVT = MVT::getIntegerVT(EltWidth); 2011 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2012 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts); 2013 } while (!TLI.isTypeLegal(NewEltVT) || 2014 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2015 } 2016} 2017 2018SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2019 SDValue Chain, 2020 SDValue BasePtr, 2021 const Value *SV, 2022 int SVOffset, 2023 unsigned Alignment, 2024 bool isVolatile, 2025 unsigned LdWidth, 2026 MVT ResType, 2027 DebugLoc dl) { 2028 // The strategy assumes that we can efficiently load powers of two widths. 2029 // The routines chops the vector into the largest power of 2 load and 2030 // can be inserted into a legal vector and then cast the result into the 2031 // vector type we want. This avoids unnecessary stack converts. 2032 2033 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 2034 // the load is nonvolatile, we an use a wider load for the value. 2035 2036 // Find the vector type that can load from. 2037 MVT NewEltVT, NewVecVT; 2038 unsigned NewEltVTWidth; 2039 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2040 NewEltVTWidth = NewEltVT.getSizeInBits(); 2041 2042 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 2043 isVolatile, Alignment); 2044 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2045 LdChain.push_back(LdOp.getValue(1)); 2046 2047 // Check if we can load the element with one instruction 2048 if (LdWidth == NewEltVTWidth) { 2049 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2050 } 2051 2052 unsigned Idx = 1; 2053 LdWidth -= NewEltVTWidth; 2054 unsigned Offset = 0; 2055 2056 while (LdWidth > 0) { 2057 unsigned Increment = NewEltVTWidth / 8; 2058 Offset += Increment; 2059 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2060 DAG.getIntPtrConstant(Increment)); 2061 2062 if (LdWidth < NewEltVTWidth) { 2063 // Our current type we are using is too large, use a smaller size by 2064 // using a smaller power of 2 2065 unsigned oNewEltVTWidth = NewEltVTWidth; 2066 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2067 NewEltVTWidth = NewEltVT.getSizeInBits(); 2068 // Readjust position and vector position based on new load type 2069 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2070 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2071 } 2072 2073 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 2074 SVOffset+Offset, isVolatile, 2075 MinAlign(Alignment, Offset)); 2076 LdChain.push_back(LdOp.getValue(1)); 2077 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 2078 DAG.getIntPtrConstant(Idx++)); 2079 2080 LdWidth -= NewEltVTWidth; 2081 } 2082 2083 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2084} 2085 2086void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2087 SDValue Chain, 2088 SDValue BasePtr, 2089 const Value *SV, 2090 int SVOffset, 2091 unsigned Alignment, 2092 bool isVolatile, 2093 SDValue ValOp, 2094 unsigned StWidth, 2095 DebugLoc dl) { 2096 // Breaks the stores into a series of power of 2 width stores. For any 2097 // width, we convert the vector to the vector of element size that we 2098 // want to store. This avoids requiring a stack convert. 2099 2100 // Find a width of the element type we can store with 2101 MVT WidenVT = ValOp.getValueType(); 2102 MVT NewEltVT, NewVecVT; 2103 2104 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2105 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 2106 2107 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2108 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2109 DAG.getIntPtrConstant(0)); 2110 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 2111 isVolatile, Alignment); 2112 StChain.push_back(StOp); 2113 2114 // Check if we are done 2115 if (StWidth == NewEltVTWidth) { 2116 return; 2117 } 2118 2119 unsigned Idx = 1; 2120 StWidth -= NewEltVTWidth; 2121 unsigned Offset = 0; 2122 2123 while (StWidth > 0) { 2124 unsigned Increment = NewEltVTWidth / 8; 2125 Offset += Increment; 2126 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2127 DAG.getIntPtrConstant(Increment)); 2128 2129 if (StWidth < NewEltVTWidth) { 2130 // Our current type we are using is too large, use a smaller size by 2131 // using a smaller power of 2 2132 unsigned oNewEltVTWidth = NewEltVTWidth; 2133 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2134 NewEltVTWidth = NewEltVT.getSizeInBits(); 2135 // Readjust position and vector position based on new load type 2136 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2137 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2138 } 2139 2140 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2141 DAG.getIntPtrConstant(Idx++)); 2142 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2143 SVOffset + Offset, isVolatile, 2144 MinAlign(Alignment, Offset))); 2145 StWidth -= NewEltVTWidth; 2146 } 2147} 2148 2149/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2150/// input vector must have the same element type as NVT. 2151SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, MVT NVT) { 2152 // Note that InOp might have been widened so it might already have 2153 // the right width or it might need be narrowed. 2154 MVT InVT = InOp.getValueType(); 2155 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2156 "input and widen element type must match"); 2157 DebugLoc dl = InOp.getDebugLoc(); 2158 2159 // Check if InOp already has the right width. 2160 if (InVT == NVT) 2161 return InOp; 2162 2163 unsigned InNumElts = InVT.getVectorNumElements(); 2164 unsigned WidenNumElts = NVT.getVectorNumElements(); 2165 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2166 unsigned NumConcat = WidenNumElts / InNumElts; 2167 SmallVector<SDValue, 16> Ops(NumConcat); 2168 SDValue UndefVal = DAG.getUNDEF(InVT); 2169 Ops[0] = InOp; 2170 for (unsigned i = 1; i != NumConcat; ++i) 2171 Ops[i] = UndefVal; 2172 2173 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2174 } 2175 2176 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2177 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2178 DAG.getIntPtrConstant(0)); 2179 2180 // Fall back to extract and build. 2181 SmallVector<SDValue, 16> Ops(WidenNumElts); 2182 MVT EltVT = NVT.getVectorElementType(); 2183 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2184 unsigned Idx; 2185 for (Idx = 0; Idx < MinNumElts; ++Idx) 2186 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2187 DAG.getIntPtrConstant(Idx)); 2188 2189 SDValue UndefVal = DAG.getUNDEF(EltVT); 2190 for ( ; Idx < WidenNumElts; ++Idx) 2191 Ops[Idx] = UndefVal; 2192 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2193} 2194