LegalizeVectorTypes.cpp revision e53009f0f4d85d1e9c3a2f902bf7fc672765d5a8
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in multiple vectors of a smaller type. For example, 19// implementing <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26using namespace llvm; 27 28//===----------------------------------------------------------------------===// 29// Result Vector Scalarization: <1 x ty> -> ty. 30//===----------------------------------------------------------------------===// 31 32void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 33 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 34 cerr << "\n"); 35 SDValue R = SDValue(); 36 37 switch (N->getOpcode()) { 38 default: 39#ifndef NDEBUG 40 cerr << "ScalarizeVectorResult #" << ResNo << ": "; 41 N->dump(&DAG); cerr << "\n"; 42#endif 43 assert(0 && "Do not know how to scalarize the result of this operator!"); 44 abort(); 45 46 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 47 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 48 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 49 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 50 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 51 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 52 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 53 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 54 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 55 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 56 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 57 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 58 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 59 60 case ISD::CTLZ: 61 case ISD::CTPOP: 62 case ISD::CTTZ: 63 case ISD::FABS: 64 case ISD::FCOS: 65 case ISD::FNEG: 66 case ISD::FP_TO_SINT: 67 case ISD::FP_TO_UINT: 68 case ISD::FSIN: 69 case ISD::FSQRT: 70 case ISD::FTRUNC: 71 case ISD::FFLOOR: 72 case ISD::FCEIL: 73 case ISD::FRINT: 74 case ISD::FNEARBYINT: 75 case ISD::UINT_TO_FP: 76 case ISD::SINT_TO_FP: 77 case ISD::TRUNCATE: 78 case ISD::SIGN_EXTEND: 79 case ISD::ZERO_EXTEND: 80 case ISD::ANY_EXTEND: 81 R = ScalarizeVecRes_UnaryOp(N); 82 break; 83 84 case ISD::ADD: 85 case ISD::AND: 86 case ISD::FADD: 87 case ISD::FDIV: 88 case ISD::FMUL: 89 case ISD::FPOW: 90 case ISD::FREM: 91 case ISD::FSUB: 92 case ISD::MUL: 93 case ISD::OR: 94 case ISD::SDIV: 95 case ISD::SREM: 96 case ISD::SUB: 97 case ISD::UDIV: 98 case ISD::UREM: 99 case ISD::XOR: 100 R = ScalarizeVecRes_BinOp(N); 101 break; 102 103 case ISD::SHL: 104 case ISD::SRA: 105 case ISD::SRL: 106 R = ScalarizeVecRes_ShiftOp(N); 107 break; 108 } 109 110 // If R is null, the sub-method took care of registering the result. 111 if (R.getNode()) 112 SetScalarizedVector(SDValue(N, ResNo), R); 113} 114 115SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 116 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 117 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 118 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 119 LHS.getValueType(), LHS, RHS); 120} 121 122SDValue DAGTypeLegalizer::ScalarizeVecRes_ShiftOp(SDNode *N) { 123 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 124 SDValue ShiftAmt = GetScalarizedVector(N->getOperand(1)); 125 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 126 LHS.getValueType(), LHS, ShiftAmt); 127} 128 129SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 130 MVT NewVT = N->getValueType(0).getVectorElementType(); 131 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 132 NewVT, N->getOperand(0)); 133} 134 135SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 136 MVT NewVT = N->getValueType(0).getVectorElementType(); 137 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 138 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 139 Op0, DAG.getValueType(NewVT), 140 DAG.getValueType(Op0.getValueType()), 141 N->getOperand(3), 142 N->getOperand(4), 143 cast<CvtRndSatSDNode>(N)->getCvtCode()); 144} 145 146SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 147 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 148 N->getValueType(0).getVectorElementType(), 149 N->getOperand(0), N->getOperand(1)); 150} 151 152SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 153 SDValue Op = GetScalarizedVector(N->getOperand(0)); 154 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 155 Op.getValueType(), Op, N->getOperand(1)); 156} 157 158SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 159 // The value to insert may have a wider type than the vector element type, 160 // so be sure to truncate it to the element type if necessary. 161 SDValue Op = N->getOperand(1); 162 MVT EltVT = N->getValueType(0).getVectorElementType(); 163 if (Op.getValueType() != EltVT) 164 // FIXME: Can this happen for floating point types? 165 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 166 return Op; 167} 168 169SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 170 assert(N->isUnindexed() && "Indexed vector load?"); 171 172 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 173 N->getExtensionType(), 174 N->getValueType(0).getVectorElementType(), 175 N->getChain(), N->getBasePtr(), 176 DAG.getUNDEF(N->getBasePtr().getValueType()), 177 N->getSrcValue(), N->getSrcValueOffset(), 178 N->getMemoryVT().getVectorElementType(), 179 N->isVolatile(), N->getAlignment()); 180 181 // Legalized the chain result - switch anything that used the old chain to 182 // use the new one. 183 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 184 return Result; 185} 186 187SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 188 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 189 MVT DestVT = N->getValueType(0).getVectorElementType(); 190 SDValue Op = GetScalarizedVector(N->getOperand(0)); 191 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 192} 193 194SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 195 // If the operand is wider than the vector element type then it is implicitly 196 // truncated. Make that explicit here. 197 MVT EltVT = N->getValueType(0).getVectorElementType(); 198 SDValue InOp = N->getOperand(0); 199 if (InOp.getValueType() != EltVT) 200 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 201 return InOp; 202} 203 204SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 205 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 206 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 207 LHS.getValueType(), N->getOperand(0), LHS, 208 GetScalarizedVector(N->getOperand(2))); 209} 210 211SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 212 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 213 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 214 N->getOperand(0), N->getOperand(1), 215 LHS, GetScalarizedVector(N->getOperand(3)), 216 N->getOperand(4)); 217} 218 219SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 220 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 221} 222 223SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 224 // Figure out if the scalar is the LHS or RHS and return it. 225 SDValue Arg = N->getOperand(2).getOperand(0); 226 if (Arg.getOpcode() == ISD::UNDEF) 227 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 228 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 229 return GetScalarizedVector(N->getOperand(Op)); 230} 231 232SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 233 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 234 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 235 MVT NVT = N->getValueType(0).getVectorElementType(); 236 MVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 237 DebugLoc dl = N->getDebugLoc(); 238 239 // Turn it into a scalar SETCC. 240 SDValue Res = DAG.getNode(ISD::SETCC, dl, SVT, LHS, RHS, N->getOperand(2)); 241 242 // VSETCC always returns a sign-extended value, while SETCC may not. The 243 // SETCC result type may not match the vector element type. Correct these. 244 if (NVT.bitsLE(SVT)) { 245 // The SETCC result type is bigger than the vector element type. 246 // Ensure the SETCC result is sign-extended. 247 if (TLI.getBooleanContents() != 248 TargetLowering::ZeroOrNegativeOneBooleanContent) 249 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, SVT, Res, 250 DAG.getValueType(MVT::i1)); 251 // Truncate to the final type. 252 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); 253 } else { 254 // The SETCC result type is smaller than the vector element type. 255 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 256 if (TLI.getBooleanContents() != 257 TargetLowering::ZeroOrNegativeOneBooleanContent) 258 Res = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Res); 259 // Sign extend to the final type. 260 return DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Res); 261 } 262} 263 264 265//===----------------------------------------------------------------------===// 266// Operand Vector Scalarization <1 x ty> -> ty. 267//===----------------------------------------------------------------------===// 268 269bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 270 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 271 cerr << "\n"); 272 SDValue Res = SDValue(); 273 274 if (Res.getNode() == 0) { 275 switch (N->getOpcode()) { 276 default: 277#ifndef NDEBUG 278 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": "; 279 N->dump(&DAG); cerr << "\n"; 280#endif 281 assert(0 && "Do not know how to scalarize this operator's operand!"); 282 abort(); 283 284 case ISD::BIT_CONVERT: 285 Res = ScalarizeVecOp_BIT_CONVERT(N); break; 286 287 case ISD::CONCAT_VECTORS: 288 Res = ScalarizeVecOp_CONCAT_VECTORS(N); break; 289 290 case ISD::EXTRACT_VECTOR_ELT: 291 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); break; 292 293 case ISD::STORE: 294 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); break; 295 } 296 } 297 298 // If the result is null, the sub-method took care of registering results etc. 299 if (!Res.getNode()) return false; 300 301 // If the result is N, the sub-method updated N in place. Tell the legalizer 302 // core about this. 303 if (Res.getNode() == N) 304 return true; 305 306 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 307 "Invalid operand expansion"); 308 309 ReplaceValueWith(SDValue(N, 0), Res); 310 return false; 311} 312 313/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 314/// to be scalarized, it must be <1 x ty>. Convert the element instead. 315SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 316 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 317 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 318 N->getValueType(0), Elt); 319} 320 321/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 322/// use a BUILD_VECTOR instead. 323SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 324 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 325 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 326 Ops[i] = GetScalarizedVector(N->getOperand(i)); 327 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 328 &Ops[0], Ops.size()); 329} 330 331/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 332/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 333/// index. 334SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 335 return GetScalarizedVector(N->getOperand(0)); 336} 337 338/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 339/// scalarized, it must be <1 x ty>. Just store the element. 340SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 341 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 342 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 343 DebugLoc dl = N->getDebugLoc(); 344 345 if (N->isTruncatingStore()) 346 return DAG.getTruncStore(N->getChain(), dl, 347 GetScalarizedVector(N->getOperand(1)), 348 N->getBasePtr(), 349 N->getSrcValue(), N->getSrcValueOffset(), 350 N->getMemoryVT().getVectorElementType(), 351 N->isVolatile(), N->getAlignment()); 352 353 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 354 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 355 N->isVolatile(), N->getAlignment()); 356} 357 358 359//===----------------------------------------------------------------------===// 360// Result Vector Splitting 361//===----------------------------------------------------------------------===// 362 363/// SplitVectorResult - This method is called when the specified result of the 364/// specified node is found to need vector splitting. At this point, the node 365/// may also have invalid operands or may have other results that need 366/// legalization, we just know that (at least) one result needs vector 367/// splitting. 368void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 369 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n"); 370 SDValue Lo, Hi; 371 372 switch (N->getOpcode()) { 373 default: 374#ifndef NDEBUG 375 cerr << "SplitVectorResult #" << ResNo << ": "; 376 N->dump(&DAG); cerr << "\n"; 377#endif 378 assert(0 && "Do not know how to split the result of this operator!"); 379 abort(); 380 381 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 382 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 383 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 384 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 385 386 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 387 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 388 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 389 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 390 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 391 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 392 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 393 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 394 case ISD::LOAD: SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);break; 395 case ISD::VECTOR_SHUFFLE: 396 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); break; 397 case ISD::VSETCC: SplitVecRes_VSETCC(N, Lo, Hi); break; 398 399 case ISD::CTTZ: 400 case ISD::CTLZ: 401 case ISD::CTPOP: 402 case ISD::FNEG: 403 case ISD::FABS: 404 case ISD::FSQRT: 405 case ISD::FSIN: 406 case ISD::FCOS: 407 case ISD::FTRUNC: 408 case ISD::FFLOOR: 409 case ISD::FCEIL: 410 case ISD::FRINT: 411 case ISD::FNEARBYINT: 412 case ISD::FP_TO_SINT: 413 case ISD::FP_TO_UINT: 414 case ISD::SINT_TO_FP: 415 case ISD::UINT_TO_FP: 416 case ISD::TRUNCATE: 417 case ISD::SIGN_EXTEND: 418 case ISD::ZERO_EXTEND: 419 case ISD::ANY_EXTEND: 420 SplitVecRes_UnaryOp(N, Lo, Hi); 421 break; 422 423 case ISD::ADD: 424 case ISD::SUB: 425 case ISD::MUL: 426 case ISD::FADD: 427 case ISD::FSUB: 428 case ISD::FMUL: 429 case ISD::SDIV: 430 case ISD::UDIV: 431 case ISD::FDIV: 432 case ISD::FPOW: 433 case ISD::AND: 434 case ISD::OR: 435 case ISD::XOR: 436 case ISD::SHL: 437 case ISD::SRA: 438 case ISD::SRL: 439 case ISD::UREM: 440 case ISD::SREM: 441 case ISD::FREM: 442 SplitVecRes_BinOp(N, Lo, Hi); 443 break; 444 } 445 446 // If Lo/Hi is null, the sub-method took care of registering results etc. 447 if (Lo.getNode()) 448 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 449} 450 451void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 452 SDValue &Hi) { 453 SDValue LHSLo, LHSHi; 454 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 455 SDValue RHSLo, RHSHi; 456 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 457 DebugLoc dl = N->getDebugLoc(); 458 459 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 460 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 461} 462 463void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 464 SDValue &Hi) { 465 // We know the result is a vector. The input may be either a vector or a 466 // scalar value. 467 MVT LoVT, HiVT; 468 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 469 DebugLoc dl = N->getDebugLoc(); 470 471 SDValue InOp = N->getOperand(0); 472 MVT InVT = InOp.getValueType(); 473 474 // Handle some special cases efficiently. 475 switch (getTypeAction(InVT)) { 476 default: 477 assert(false && "Unknown type action!"); 478 case Legal: 479 case PromoteInteger: 480 case SoftenFloat: 481 case ScalarizeVector: 482 break; 483 case ExpandInteger: 484 case ExpandFloat: 485 // A scalar to vector conversion, where the scalar needs expansion. 486 // If the vector is being split in two then we can just convert the 487 // expanded pieces. 488 if (LoVT == HiVT) { 489 GetExpandedOp(InOp, Lo, Hi); 490 if (TLI.isBigEndian()) 491 std::swap(Lo, Hi); 492 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 493 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 494 return; 495 } 496 break; 497 case SplitVector: 498 // If the input is a vector that needs to be split, convert each split 499 // piece of the input now. 500 GetSplitVector(InOp, Lo, Hi); 501 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 502 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 503 return; 504 } 505 506 // In the general case, convert the input to an integer and split it by hand. 507 MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits()); 508 MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits()); 509 if (TLI.isBigEndian()) 510 std::swap(LoIntVT, HiIntVT); 511 512 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 513 514 if (TLI.isBigEndian()) 515 std::swap(Lo, Hi); 516 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 517 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 518} 519 520void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 521 SDValue &Hi) { 522 MVT LoVT, HiVT; 523 DebugLoc dl = N->getDebugLoc(); 524 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 525 unsigned LoNumElts = LoVT.getVectorNumElements(); 526 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 527 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 528 529 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 530 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 531} 532 533void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 534 SDValue &Hi) { 535 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 536 DebugLoc dl = N->getDebugLoc(); 537 unsigned NumSubvectors = N->getNumOperands() / 2; 538 if (NumSubvectors == 1) { 539 Lo = N->getOperand(0); 540 Hi = N->getOperand(1); 541 return; 542 } 543 544 MVT LoVT, HiVT; 545 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 546 547 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 548 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 549 550 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 551 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 552} 553 554void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 555 SDValue &Hi) { 556 MVT LoVT, HiVT; 557 DebugLoc dl = N->getDebugLoc(); 558 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 559 560 SDValue DTyOpLo = DAG.getValueType(LoVT); 561 SDValue DTyOpHi = DAG.getValueType(HiVT); 562 563 SDValue RndOp = N->getOperand(3); 564 SDValue SatOp = N->getOperand(4); 565 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 566 567 // Split the input. 568 SDValue VLo, VHi; 569 MVT InVT = N->getOperand(0).getValueType(); 570 switch (getTypeAction(InVT)) { 571 default: assert(0 && "Unexpected type action!"); 572 case Legal: { 573 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 574 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 575 LoVT.getVectorNumElements()); 576 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 577 DAG.getIntPtrConstant(0)); 578 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 579 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 580 break; 581 } 582 case SplitVector: 583 GetSplitVector(N->getOperand(0), VLo, VHi); 584 break; 585 case WidenVector: { 586 // If the result needs to be split and the input needs to be widened, 587 // the two types must have different lengths. Use the widened result 588 // and extract from it to do the split. 589 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 590 SDValue InOp = GetWidenedVector(N->getOperand(0)); 591 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 592 LoVT.getVectorNumElements()); 593 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 594 DAG.getIntPtrConstant(0)); 595 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 596 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 597 break; 598 } 599 } 600 601 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 602 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 603 604 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 605 CvtCode); 606 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 607 CvtCode); 608} 609 610void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 611 SDValue &Hi) { 612 SDValue Vec = N->getOperand(0); 613 SDValue Idx = N->getOperand(1); 614 MVT IdxVT = Idx.getValueType(); 615 DebugLoc dl = N->getDebugLoc(); 616 617 MVT LoVT, HiVT; 618 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 619 // The indices are not guaranteed to be a multiple of the new vector 620 // size unless the original vector type was split in two. 621 assert(LoVT == HiVT && "Non power-of-two vectors not supported!"); 622 623 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 624 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 625 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 626 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 627} 628 629void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 630 SDValue &Hi) { 631 DebugLoc dl = N->getDebugLoc(); 632 GetSplitVector(N->getOperand(0), Lo, Hi); 633 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 634 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 635} 636 637void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 638 SDValue &Hi) { 639 SDValue Vec = N->getOperand(0); 640 SDValue Elt = N->getOperand(1); 641 SDValue Idx = N->getOperand(2); 642 DebugLoc dl = N->getDebugLoc(); 643 GetSplitVector(Vec, Lo, Hi); 644 645 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 646 unsigned IdxVal = CIdx->getZExtValue(); 647 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 648 if (IdxVal < LoNumElts) 649 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 650 Lo.getValueType(), Lo, Elt, Idx); 651 else 652 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 653 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 654 return; 655 } 656 657 // Spill the vector to the stack. 658 MVT VecVT = Vec.getValueType(); 659 MVT EltVT = VecVT.getVectorElementType(); 660 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 661 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 662 663 // Store the new element. This may be larger than the vector element type, 664 // so use a truncating store. 665 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 666 unsigned Alignment = 667 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForMVT()); 668 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 669 670 // Load the Lo part from the stack slot. 671 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 672 673 // Increment the pointer to the other part. 674 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 675 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 676 DAG.getIntPtrConstant(IncrementSize)); 677 678 // Load the Hi part from the stack slot. 679 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 680 MinAlign(Alignment, IncrementSize)); 681} 682 683void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 684 SDValue &Hi) { 685 MVT LoVT, HiVT; 686 DebugLoc dl = N->getDebugLoc(); 687 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 688 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 689 Hi = DAG.getUNDEF(HiVT); 690} 691 692void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 693 SDValue &Hi) { 694 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 695 MVT LoVT, HiVT; 696 DebugLoc dl = LD->getDebugLoc(); 697 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 698 699 ISD::LoadExtType ExtType = LD->getExtensionType(); 700 SDValue Ch = LD->getChain(); 701 SDValue Ptr = LD->getBasePtr(); 702 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 703 const Value *SV = LD->getSrcValue(); 704 int SVOffset = LD->getSrcValueOffset(); 705 MVT MemoryVT = LD->getMemoryVT(); 706 unsigned Alignment = LD->getAlignment(); 707 bool isVolatile = LD->isVolatile(); 708 709 MVT LoMemVT, HiMemVT; 710 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 711 712 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 713 SV, SVOffset, LoMemVT, isVolatile, Alignment); 714 715 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 716 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 717 DAG.getIntPtrConstant(IncrementSize)); 718 SVOffset += IncrementSize; 719 Alignment = MinAlign(Alignment, IncrementSize); 720 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 721 SV, SVOffset, HiMemVT, isVolatile, Alignment); 722 723 // Build a factor node to remember that this load is independent of the 724 // other one. 725 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 726 Hi.getValue(1)); 727 728 // Legalized the chain result - switch anything that used the old chain to 729 // use the new one. 730 ReplaceValueWith(SDValue(LD, 1), Ch); 731} 732 733void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 734 SDValue &Hi) { 735 // Get the dest types - they may not match the input types, e.g. int_to_fp. 736 MVT LoVT, HiVT; 737 DebugLoc dl = N->getDebugLoc(); 738 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 739 740 // Split the input. 741 MVT InVT = N->getOperand(0).getValueType(); 742 switch (getTypeAction(InVT)) { 743 default: assert(0 && "Unexpected type action!"); 744 case Legal: { 745 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 746 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 747 LoVT.getVectorNumElements()); 748 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 749 DAG.getIntPtrConstant(0)); 750 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 751 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 752 break; 753 } 754 case SplitVector: 755 GetSplitVector(N->getOperand(0), Lo, Hi); 756 break; 757 case WidenVector: { 758 // If the result needs to be split and the input needs to be widened, 759 // the two types must have different lengths. Use the widened result 760 // and extract from it to do the split. 761 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 762 SDValue InOp = GetWidenedVector(N->getOperand(0)); 763 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 764 LoVT.getVectorNumElements()); 765 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 766 DAG.getIntPtrConstant(0)); 767 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 768 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 769 break; 770 } 771 } 772 773 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 774 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 775} 776 777void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 778 SDValue &Lo, SDValue &Hi) { 779 // The low and high parts of the original input give four input vectors. 780 SDValue Inputs[4]; 781 DebugLoc dl = N->getDebugLoc(); 782 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 783 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 784 MVT NewVT = Inputs[0].getValueType(); 785 unsigned NewElts = NewVT.getVectorNumElements(); 786 assert(NewVT == Inputs[1].getValueType() && 787 "Non power-of-two vectors not supported!"); 788 789 // If Lo or Hi uses elements from at most two of the four input vectors, then 790 // express it as a vector shuffle of those two inputs. Otherwise extract the 791 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 792 SmallVector<int, 16> Ops; 793 for (unsigned High = 0; High < 2; ++High) { 794 SDValue &Output = High ? Hi : Lo; 795 796 // Build a shuffle mask for the output, discovering on the fly which 797 // input vectors to use as shuffle operands (recorded in InputUsed). 798 // If building a suitable shuffle vector proves too hard, then bail 799 // out with useBuildVector set. 800 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 801 unsigned FirstMaskIdx = High * NewElts; 802 bool useBuildVector = false; 803 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 804 // The mask element. This indexes into the input. 805 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 806 807 // The input vector this mask element indexes into. 808 unsigned Input = (unsigned)Idx / NewElts; 809 810 if (Input >= array_lengthof(Inputs)) { 811 // The mask element does not index into any input vector. 812 Ops.push_back(-1); 813 continue; 814 } 815 816 // Turn the index into an offset from the start of the input vector. 817 Idx -= Input * NewElts; 818 819 // Find or create a shuffle vector operand to hold this input. 820 unsigned OpNo; 821 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 822 if (InputUsed[OpNo] == Input) { 823 // This input vector is already an operand. 824 break; 825 } else if (InputUsed[OpNo] == -1U) { 826 // Create a new operand for this input vector. 827 InputUsed[OpNo] = Input; 828 break; 829 } 830 } 831 832 if (OpNo >= array_lengthof(InputUsed)) { 833 // More than two input vectors used! Give up on trying to create a 834 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 835 useBuildVector = true; 836 break; 837 } 838 839 // Add the mask index for the new shuffle vector. 840 Ops.push_back(Idx + OpNo * NewElts); 841 } 842 843 if (useBuildVector) { 844 MVT EltVT = NewVT.getVectorElementType(); 845 SmallVector<SDValue, 16> SVOps; 846 847 // Extract the input elements by hand. 848 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 849 // The mask element. This indexes into the input. 850 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 851 852 // The input vector this mask element indexes into. 853 unsigned Input = (unsigned)Idx / NewElts; 854 855 if (Input >= array_lengthof(Inputs)) { 856 // The mask element is "undef" or indexes off the end of the input. 857 SVOps.push_back(DAG.getUNDEF(EltVT)); 858 continue; 859 } 860 861 // Turn the index into an offset from the start of the input vector. 862 Idx -= Input * NewElts; 863 864 // Extract the vector element by hand. 865 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 866 Inputs[Input], DAG.getIntPtrConstant(Idx))); 867 } 868 869 // Construct the Lo/Hi output using a BUILD_VECTOR. 870 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 871 } else if (InputUsed[0] == -1U) { 872 // No input vectors were used! The result is undefined. 873 Output = DAG.getUNDEF(NewVT); 874 } else { 875 SDValue Op0 = Inputs[InputUsed[0]]; 876 // If only one input was used, use an undefined vector for the other. 877 SDValue Op1 = InputUsed[1] == -1U ? 878 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 879 // At least one input vector was used. Create a new shuffle vector. 880 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 881 } 882 883 Ops.clear(); 884 } 885} 886 887void DAGTypeLegalizer::SplitVecRes_VSETCC(SDNode *N, SDValue &Lo, 888 SDValue &Hi) { 889 MVT LoVT, HiVT; 890 DebugLoc dl = N->getDebugLoc(); 891 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 892 893 SDValue LL, LH, RL, RH; 894 GetSplitVector(N->getOperand(0), LL, LH); 895 GetSplitVector(N->getOperand(1), RL, RH); 896 897 Lo = DAG.getNode(ISD::VSETCC, dl, LoVT, LL, RL, N->getOperand(2)); 898 Hi = DAG.getNode(ISD::VSETCC, dl, HiVT, LH, RH, N->getOperand(2)); 899} 900 901 902//===----------------------------------------------------------------------===// 903// Operand Vector Splitting 904//===----------------------------------------------------------------------===// 905 906/// SplitVectorOperand - This method is called when the specified operand of the 907/// specified node is found to need vector splitting. At this point, all of the 908/// result types of the node are known to be legal, but other operands of the 909/// node may need legalization as well as the specified one. 910bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 911 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n"); 912 SDValue Res = SDValue(); 913 914 if (Res.getNode() == 0) { 915 switch (N->getOpcode()) { 916 default: 917#ifndef NDEBUG 918 cerr << "SplitVectorOperand Op #" << OpNo << ": "; 919 N->dump(&DAG); cerr << "\n"; 920#endif 921 assert(0 && "Do not know how to split this operator's operand!"); 922 abort(); 923 924 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 925 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 926 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 927 case ISD::STORE: 928 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 929 break; 930 931 case ISD::CTTZ: 932 case ISD::CTLZ: 933 case ISD::CTPOP: 934 case ISD::FP_TO_SINT: 935 case ISD::FP_TO_UINT: 936 case ISD::SINT_TO_FP: 937 case ISD::UINT_TO_FP: 938 case ISD::TRUNCATE: 939 case ISD::SIGN_EXTEND: 940 case ISD::ZERO_EXTEND: 941 case ISD::ANY_EXTEND: 942 Res = SplitVecOp_UnaryOp(N); 943 break; 944 } 945 } 946 947 // If the result is null, the sub-method took care of registering results etc. 948 if (!Res.getNode()) return false; 949 950 // If the result is N, the sub-method updated N in place. Tell the legalizer 951 // core about this. 952 if (Res.getNode() == N) 953 return true; 954 955 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 956 "Invalid operand expansion"); 957 958 ReplaceValueWith(SDValue(N, 0), Res); 959 return false; 960} 961 962SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 963 // The result has a legal vector type, but the input needs splitting. 964 MVT ResVT = N->getValueType(0); 965 SDValue Lo, Hi; 966 DebugLoc dl = N->getDebugLoc(); 967 GetSplitVector(N->getOperand(0), Lo, Hi); 968 assert(Lo.getValueType() == Hi.getValueType() && 969 "Returns legal non-power-of-two vector type?"); 970 MVT InVT = Lo.getValueType(); 971 972 MVT OutVT = MVT::getVectorVT(ResVT.getVectorElementType(), 973 InVT.getVectorNumElements()); 974 975 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 976 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 977 978 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 979} 980 981SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 982 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 983 // end up being split all the way down to individual components. Convert the 984 // split pieces into integers and reassemble. 985 SDValue Lo, Hi; 986 GetSplitVector(N->getOperand(0), Lo, Hi); 987 Lo = BitConvertToInteger(Lo); 988 Hi = BitConvertToInteger(Hi); 989 990 if (TLI.isBigEndian()) 991 std::swap(Lo, Hi); 992 993 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 994 JoinIntegers(Lo, Hi)); 995} 996 997SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 998 // We know that the extracted result type is legal. For now, assume the index 999 // is a constant. 1000 MVT SubVT = N->getValueType(0); 1001 SDValue Idx = N->getOperand(1); 1002 DebugLoc dl = N->getDebugLoc(); 1003 SDValue Lo, Hi; 1004 GetSplitVector(N->getOperand(0), Lo, Hi); 1005 1006 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1007 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1008 1009 if (IdxVal < LoElts) { 1010 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1011 "Extracted subvector crosses vector split!"); 1012 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1013 } else { 1014 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1015 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1016 } 1017} 1018 1019SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1020 SDValue Vec = N->getOperand(0); 1021 SDValue Idx = N->getOperand(1); 1022 MVT VecVT = Vec.getValueType(); 1023 1024 if (isa<ConstantSDNode>(Idx)) { 1025 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1026 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1027 1028 SDValue Lo, Hi; 1029 GetSplitVector(Vec, Lo, Hi); 1030 1031 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1032 1033 if (IdxVal < LoElts) 1034 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1035 else 1036 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1037 DAG.getConstant(IdxVal - LoElts, 1038 Idx.getValueType())); 1039 } 1040 1041 // Store the vector to the stack. 1042 MVT EltVT = VecVT.getVectorElementType(); 1043 DebugLoc dl = N->getDebugLoc(); 1044 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1045 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1046 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1047 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1048 1049 // Load back the required element. 1050 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1051 return DAG.getLoad(EltVT, dl, Store, StackPtr, SV, 0); 1052} 1053 1054SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1055 assert(N->isUnindexed() && "Indexed store of vector?"); 1056 assert(OpNo == 1 && "Can only split the stored value"); 1057 DebugLoc dl = N->getDebugLoc(); 1058 1059 bool isTruncating = N->isTruncatingStore(); 1060 SDValue Ch = N->getChain(); 1061 SDValue Ptr = N->getBasePtr(); 1062 int SVOffset = N->getSrcValueOffset(); 1063 MVT MemoryVT = N->getMemoryVT(); 1064 unsigned Alignment = N->getAlignment(); 1065 bool isVol = N->isVolatile(); 1066 SDValue Lo, Hi; 1067 GetSplitVector(N->getOperand(1), Lo, Hi); 1068 1069 MVT LoMemVT, HiMemVT; 1070 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1071 1072 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1073 1074 if (isTruncating) 1075 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1076 LoMemVT, isVol, Alignment); 1077 else 1078 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1079 isVol, Alignment); 1080 1081 // Increment the pointer to the other half. 1082 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1083 DAG.getIntPtrConstant(IncrementSize)); 1084 1085 if (isTruncating) 1086 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 1087 N->getSrcValue(), SVOffset+IncrementSize, 1088 HiMemVT, 1089 isVol, MinAlign(Alignment, IncrementSize)); 1090 else 1091 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 1092 isVol, MinAlign(Alignment, IncrementSize)); 1093 1094 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1095} 1096 1097 1098//===----------------------------------------------------------------------===// 1099// Result Vector Widening 1100//===----------------------------------------------------------------------===// 1101 1102void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1103 DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG); 1104 cerr << "\n"); 1105 SDValue Res = SDValue(); 1106 1107 switch (N->getOpcode()) { 1108 default: 1109#ifndef NDEBUG 1110 cerr << "WidenVectorResult #" << ResNo << ": "; 1111 N->dump(&DAG); cerr << "\n"; 1112#endif 1113 assert(0 && "Do not know how to widen the result of this operator!"); 1114 abort(); 1115 1116 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1117 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1118 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1119 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1120 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1121 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1122 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1123 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1124 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1125 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1126 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1127 case ISD::VECTOR_SHUFFLE: 1128 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); break; 1129 case ISD::VSETCC: Res = WidenVecRes_VSETCC(N); break; 1130 1131 case ISD::ADD: 1132 case ISD::AND: 1133 case ISD::BSWAP: 1134 case ISD::FADD: 1135 case ISD::FCOPYSIGN: 1136 case ISD::FDIV: 1137 case ISD::FMUL: 1138 case ISD::FPOW: 1139 case ISD::FPOWI: 1140 case ISD::FREM: 1141 case ISD::FSUB: 1142 case ISD::MUL: 1143 case ISD::MULHS: 1144 case ISD::MULHU: 1145 case ISD::OR: 1146 case ISD::SDIV: 1147 case ISD::SREM: 1148 case ISD::UDIV: 1149 case ISD::UREM: 1150 case ISD::SUB: 1151 case ISD::XOR: 1152 Res = WidenVecRes_Binary(N); 1153 break; 1154 1155 case ISD::SHL: 1156 case ISD::SRA: 1157 case ISD::SRL: 1158 Res = WidenVecRes_Shift(N); 1159 break; 1160 1161 case ISD::FP_ROUND: 1162 case ISD::FP_TO_SINT: 1163 case ISD::FP_TO_UINT: 1164 case ISD::SINT_TO_FP: 1165 case ISD::UINT_TO_FP: 1166 case ISD::TRUNCATE: 1167 case ISD::SIGN_EXTEND: 1168 case ISD::ZERO_EXTEND: 1169 case ISD::ANY_EXTEND: 1170 Res = WidenVecRes_Convert(N); 1171 break; 1172 1173 case ISD::CTLZ: 1174 case ISD::CTPOP: 1175 case ISD::CTTZ: 1176 case ISD::FABS: 1177 case ISD::FCOS: 1178 case ISD::FNEG: 1179 case ISD::FSIN: 1180 case ISD::FSQRT: 1181 Res = WidenVecRes_Unary(N); 1182 break; 1183 } 1184 1185 // If Res is null, the sub-method took care of registering the result. 1186 if (Res.getNode()) 1187 SetWidenedVector(SDValue(N, ResNo), Res); 1188} 1189 1190SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1191 // Binary op widening. 1192 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1193 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1194 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1195 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1196} 1197 1198SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1199 SDValue InOp = N->getOperand(0); 1200 DebugLoc dl = N->getDebugLoc(); 1201 1202 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1203 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1204 1205 MVT InVT = InOp.getValueType(); 1206 MVT InEltVT = InVT.getVectorElementType(); 1207 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts); 1208 1209 unsigned Opcode = N->getOpcode(); 1210 unsigned InVTNumElts = InVT.getVectorNumElements(); 1211 1212 if (getTypeAction(InVT) == WidenVector) { 1213 InOp = GetWidenedVector(N->getOperand(0)); 1214 InVT = InOp.getValueType(); 1215 InVTNumElts = InVT.getVectorNumElements(); 1216 if (InVTNumElts == WidenNumElts) 1217 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1218 } 1219 1220 if (TLI.isTypeLegal(InWidenVT)) { 1221 // Because the result and the input are different vector types, widening 1222 // the result could create a legal type but widening the input might make 1223 // it an illegal type that might lead to repeatedly splitting the input 1224 // and then widening it. To avoid this, we widen the input only if 1225 // it results in a legal type. 1226 if (WidenNumElts % InVTNumElts == 0) { 1227 // Widen the input and call convert on the widened input vector. 1228 unsigned NumConcat = WidenNumElts/InVTNumElts; 1229 SmallVector<SDValue, 16> Ops(NumConcat); 1230 Ops[0] = InOp; 1231 SDValue UndefVal = DAG.getUNDEF(InVT); 1232 for (unsigned i = 1; i != NumConcat; ++i) 1233 Ops[i] = UndefVal; 1234 return DAG.getNode(Opcode, dl, WidenVT, 1235 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1236 &Ops[0], NumConcat)); 1237 } 1238 1239 if (InVTNumElts % WidenNumElts == 0) { 1240 // Extract the input and convert the shorten input vector. 1241 return DAG.getNode(Opcode, dl, WidenVT, 1242 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1243 InOp, DAG.getIntPtrConstant(0))); 1244 } 1245 } 1246 1247 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1248 SmallVector<SDValue, 16> Ops(WidenNumElts); 1249 MVT EltVT = WidenVT.getVectorElementType(); 1250 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1251 unsigned i; 1252 for (i=0; i < MinElts; ++i) 1253 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1254 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1255 DAG.getIntPtrConstant(i))); 1256 1257 SDValue UndefVal = DAG.getUNDEF(EltVT); 1258 for (; i < WidenNumElts; ++i) 1259 Ops[i] = UndefVal; 1260 1261 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1262} 1263 1264SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1265 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1266 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1267 SDValue ShOp = N->getOperand(1); 1268 1269 MVT ShVT = ShOp.getValueType(); 1270 if (getTypeAction(ShVT) == WidenVector) { 1271 ShOp = GetWidenedVector(ShOp); 1272 ShVT = ShOp.getValueType(); 1273 } 1274 MVT ShWidenVT = MVT::getVectorVT(ShVT.getVectorElementType(), 1275 WidenVT.getVectorNumElements()); 1276 if (ShVT != ShWidenVT) 1277 ShOp = ModifyToType(ShOp, ShWidenVT); 1278 1279 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1280} 1281 1282SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1283 // Unary op widening. 1284 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1285 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1286 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1287} 1288 1289SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1290 SDValue InOp = N->getOperand(0); 1291 MVT InVT = InOp.getValueType(); 1292 MVT VT = N->getValueType(0); 1293 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1294 DebugLoc dl = N->getDebugLoc(); 1295 1296 switch (getTypeAction(InVT)) { 1297 default: 1298 assert(false && "Unknown type action!"); 1299 break; 1300 case Legal: 1301 break; 1302 case PromoteInteger: 1303 // If the InOp is promoted to the same size, convert it. Otherwise, 1304 // fall out of the switch and widen the promoted input. 1305 InOp = GetPromotedInteger(InOp); 1306 InVT = InOp.getValueType(); 1307 if (WidenVT.bitsEq(InVT)) 1308 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1309 break; 1310 case SoftenFloat: 1311 case ExpandInteger: 1312 case ExpandFloat: 1313 case ScalarizeVector: 1314 case SplitVector: 1315 break; 1316 case WidenVector: 1317 // If the InOp is widened to the same size, convert it. Otherwise, fall 1318 // out of the switch and widen the widened input. 1319 InOp = GetWidenedVector(InOp); 1320 InVT = InOp.getValueType(); 1321 if (WidenVT.bitsEq(InVT)) 1322 // The input widens to the same size. Convert to the widen value. 1323 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1324 break; 1325 } 1326 1327 unsigned WidenSize = WidenVT.getSizeInBits(); 1328 unsigned InSize = InVT.getSizeInBits(); 1329 if (WidenSize % InSize == 0) { 1330 // Determine new input vector type. The new input vector type will use 1331 // the same element type (if its a vector) or use the input type as a 1332 // vector. It is the same size as the type to widen to. 1333 MVT NewInVT; 1334 unsigned NewNumElts = WidenSize / InSize; 1335 if (InVT.isVector()) { 1336 MVT InEltVT = InVT.getVectorElementType(); 1337 NewInVT= MVT::getVectorVT(InEltVT, WidenSize / InEltVT.getSizeInBits()); 1338 } else { 1339 NewInVT = MVT::getVectorVT(InVT, NewNumElts); 1340 } 1341 1342 if (TLI.isTypeLegal(NewInVT)) { 1343 // Because the result and the input are different vector types, widening 1344 // the result could create a legal type but widening the input might make 1345 // it an illegal type that might lead to repeatedly splitting the input 1346 // and then widening it. To avoid this, we widen the input only if 1347 // it results in a legal type. 1348 SmallVector<SDValue, 16> Ops(NewNumElts); 1349 SDValue UndefVal = DAG.getUNDEF(InVT); 1350 Ops[0] = InOp; 1351 for (unsigned i = 1; i < NewNumElts; ++i) 1352 Ops[i] = UndefVal; 1353 1354 SDValue NewVec; 1355 if (InVT.isVector()) 1356 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1357 NewInVT, &Ops[0], NewNumElts); 1358 else 1359 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1360 NewInVT, &Ops[0], NewNumElts); 1361 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1362 } 1363 } 1364 1365 // This should occur rarely. Lower the bit-convert to a store/load 1366 // from the stack. Create the stack frame object. Make sure it is aligned 1367 // for both the source and destination types. 1368 SDValue FIPtr = DAG.CreateStackTemporary(InVT, WidenVT); 1369 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1370 const Value *SV = PseudoSourceValue::getFixedStack(FI); 1371 1372 // Emit a store to the stack slot. 1373 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0); 1374 1375 // Result is a load from the stack slot. 1376 return DAG.getLoad(WidenVT, dl, Store, FIPtr, SV, 0); 1377} 1378 1379SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1380 DebugLoc dl = N->getDebugLoc(); 1381 // Build a vector with undefined for the new nodes. 1382 MVT VT = N->getValueType(0); 1383 MVT EltVT = VT.getVectorElementType(); 1384 unsigned NumElts = VT.getVectorNumElements(); 1385 1386 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1387 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1388 1389 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1390 NewOps.reserve(WidenNumElts); 1391 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1392 NewOps.push_back(DAG.getUNDEF(EltVT)); 1393 1394 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1395} 1396 1397SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1398 MVT InVT = N->getOperand(0).getValueType(); 1399 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1400 DebugLoc dl = N->getDebugLoc(); 1401 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1402 unsigned NumOperands = N->getNumOperands(); 1403 1404 bool InputWidened = false; // Indicates we need to widen the input. 1405 if (getTypeAction(InVT) != WidenVector) { 1406 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1407 // Add undef vectors to widen to correct length. 1408 unsigned NumConcat = WidenVT.getVectorNumElements() / 1409 InVT.getVectorNumElements(); 1410 SDValue UndefVal = DAG.getUNDEF(InVT); 1411 SmallVector<SDValue, 16> Ops(NumConcat); 1412 for (unsigned i=0; i < NumOperands; ++i) 1413 Ops[i] = N->getOperand(i); 1414 for (unsigned i = NumOperands; i != NumConcat; ++i) 1415 Ops[i] = UndefVal; 1416 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1417 } 1418 } else { 1419 InputWidened = true; 1420 if (WidenVT == TLI.getTypeToTransformTo(InVT)) { 1421 // The inputs and the result are widen to the same value. 1422 unsigned i; 1423 for (i=1; i < NumOperands; ++i) 1424 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1425 break; 1426 1427 if (i > NumOperands) 1428 // Everything but the first operand is an UNDEF so just return the 1429 // widened first operand. 1430 return GetWidenedVector(N->getOperand(0)); 1431 1432 if (NumOperands == 2) { 1433 // Replace concat of two operands with a shuffle. 1434 SmallVector<int, 16> MaskOps(WidenNumElts); 1435 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1436 MaskOps[i] = i; 1437 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1438 } 1439 return DAG.getVectorShuffle(WidenVT, dl, 1440 GetWidenedVector(N->getOperand(0)), 1441 GetWidenedVector(N->getOperand(1)), 1442 &MaskOps[0]); 1443 } 1444 } 1445 } 1446 1447 // Fall back to use extracts and build vector. 1448 MVT EltVT = WidenVT.getVectorElementType(); 1449 unsigned NumInElts = InVT.getVectorNumElements(); 1450 SmallVector<SDValue, 16> Ops(WidenNumElts); 1451 unsigned Idx = 0; 1452 for (unsigned i=0; i < NumOperands; ++i) { 1453 SDValue InOp = N->getOperand(i); 1454 if (InputWidened) 1455 InOp = GetWidenedVector(InOp); 1456 for (unsigned j=0; j < NumInElts; ++j) 1457 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1458 DAG.getIntPtrConstant(j)); 1459 } 1460 SDValue UndefVal = DAG.getUNDEF(EltVT); 1461 for (; Idx < WidenNumElts; ++Idx) 1462 Ops[Idx] = UndefVal; 1463 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1464} 1465 1466SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1467 DebugLoc dl = N->getDebugLoc(); 1468 SDValue InOp = N->getOperand(0); 1469 SDValue RndOp = N->getOperand(3); 1470 SDValue SatOp = N->getOperand(4); 1471 1472 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1473 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1474 1475 MVT InVT = InOp.getValueType(); 1476 MVT InEltVT = InVT.getVectorElementType(); 1477 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts); 1478 1479 SDValue DTyOp = DAG.getValueType(WidenVT); 1480 SDValue STyOp = DAG.getValueType(InWidenVT); 1481 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1482 1483 unsigned InVTNumElts = InVT.getVectorNumElements(); 1484 if (getTypeAction(InVT) == WidenVector) { 1485 InOp = GetWidenedVector(InOp); 1486 InVT = InOp.getValueType(); 1487 InVTNumElts = InVT.getVectorNumElements(); 1488 if (InVTNumElts == WidenNumElts) 1489 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1490 SatOp, CvtCode); 1491 } 1492 1493 if (TLI.isTypeLegal(InWidenVT)) { 1494 // Because the result and the input are different vector types, widening 1495 // the result could create a legal type but widening the input might make 1496 // it an illegal type that might lead to repeatedly splitting the input 1497 // and then widening it. To avoid this, we widen the input only if 1498 // it results in a legal type. 1499 if (WidenNumElts % InVTNumElts == 0) { 1500 // Widen the input and call convert on the widened input vector. 1501 unsigned NumConcat = WidenNumElts/InVTNumElts; 1502 SmallVector<SDValue, 16> Ops(NumConcat); 1503 Ops[0] = InOp; 1504 SDValue UndefVal = DAG.getUNDEF(InVT); 1505 for (unsigned i = 1; i != NumConcat; ++i) { 1506 Ops[i] = UndefVal; 1507 } 1508 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1509 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1510 SatOp, CvtCode); 1511 } 1512 1513 if (InVTNumElts % WidenNumElts == 0) { 1514 // Extract the input and convert the shorten input vector. 1515 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1516 DAG.getIntPtrConstant(0)); 1517 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1518 SatOp, CvtCode); 1519 } 1520 } 1521 1522 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1523 SmallVector<SDValue, 16> Ops(WidenNumElts); 1524 MVT EltVT = WidenVT.getVectorElementType(); 1525 DTyOp = DAG.getValueType(EltVT); 1526 STyOp = DAG.getValueType(InEltVT); 1527 1528 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1529 unsigned i; 1530 for (i=0; i < MinElts; ++i) { 1531 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1532 DAG.getIntPtrConstant(i)); 1533 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1534 SatOp, CvtCode); 1535 } 1536 1537 SDValue UndefVal = DAG.getUNDEF(EltVT); 1538 for (; i < WidenNumElts; ++i) 1539 Ops[i] = UndefVal; 1540 1541 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1542} 1543 1544SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1545 MVT VT = N->getValueType(0); 1546 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1547 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1548 SDValue InOp = N->getOperand(0); 1549 SDValue Idx = N->getOperand(1); 1550 DebugLoc dl = N->getDebugLoc(); 1551 1552 if (getTypeAction(InOp.getValueType()) == WidenVector) 1553 InOp = GetWidenedVector(InOp); 1554 1555 MVT InVT = InOp.getValueType(); 1556 1557 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1558 if (CIdx) { 1559 unsigned IdxVal = CIdx->getZExtValue(); 1560 // Check if we can just return the input vector after widening. 1561 if (IdxVal == 0 && InVT == WidenVT) 1562 return InOp; 1563 1564 // Check if we can extract from the vector. 1565 unsigned InNumElts = InVT.getVectorNumElements(); 1566 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1567 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1568 } 1569 1570 // We could try widening the input to the right length but for now, extract 1571 // the original elements, fill the rest with undefs and build a vector. 1572 SmallVector<SDValue, 16> Ops(WidenNumElts); 1573 MVT EltVT = VT.getVectorElementType(); 1574 MVT IdxVT = Idx.getValueType(); 1575 unsigned NumElts = VT.getVectorNumElements(); 1576 unsigned i; 1577 if (CIdx) { 1578 unsigned IdxVal = CIdx->getZExtValue(); 1579 for (i=0; i < NumElts; ++i) 1580 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1581 DAG.getConstant(IdxVal+i, IdxVT)); 1582 } else { 1583 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1584 for (i=1; i < NumElts; ++i) { 1585 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1586 DAG.getConstant(i, IdxVT)); 1587 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1588 } 1589 } 1590 1591 SDValue UndefVal = DAG.getUNDEF(EltVT); 1592 for (; i < WidenNumElts; ++i) 1593 Ops[i] = UndefVal; 1594 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1595} 1596 1597SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1598 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1599 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1600 InOp.getValueType(), InOp, 1601 N->getOperand(1), N->getOperand(2)); 1602} 1603 1604SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1605 LoadSDNode *LD = cast<LoadSDNode>(N); 1606 MVT WidenVT = TLI.getTypeToTransformTo(LD->getValueType(0)); 1607 MVT LdVT = LD->getMemoryVT(); 1608 DebugLoc dl = N->getDebugLoc(); 1609 assert(LdVT.isVector() && WidenVT.isVector()); 1610 1611 // Load information 1612 SDValue Chain = LD->getChain(); 1613 SDValue BasePtr = LD->getBasePtr(); 1614 int SVOffset = LD->getSrcValueOffset(); 1615 unsigned Align = LD->getAlignment(); 1616 bool isVolatile = LD->isVolatile(); 1617 const Value *SV = LD->getSrcValue(); 1618 ISD::LoadExtType ExtType = LD->getExtensionType(); 1619 1620 SDValue Result; 1621 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1622 if (ExtType != ISD::NON_EXTLOAD) { 1623 // For extension loads, we can not play the tricks of chopping legal 1624 // vector types and bit cast it to the right type. Instead, we unroll 1625 // the load and build a vector. 1626 MVT EltVT = WidenVT.getVectorElementType(); 1627 MVT LdEltVT = LdVT.getVectorElementType(); 1628 unsigned NumElts = LdVT.getVectorNumElements(); 1629 1630 // Load each element and widen 1631 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1632 SmallVector<SDValue, 16> Ops(WidenNumElts); 1633 unsigned Increment = LdEltVT.getSizeInBits() / 8; 1634 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 1635 LdEltVT, isVolatile, Align); 1636 LdChain.push_back(Ops[0].getValue(1)); 1637 unsigned i = 0, Offset = Increment; 1638 for (i=1; i < NumElts; ++i, Offset += Increment) { 1639 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1640 BasePtr, DAG.getIntPtrConstant(Offset)); 1641 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 1642 SVOffset + Offset, LdEltVT, isVolatile, Align); 1643 LdChain.push_back(Ops[i].getValue(1)); 1644 } 1645 1646 // Fill the rest with undefs 1647 SDValue UndefVal = DAG.getUNDEF(EltVT); 1648 for (; i != WidenNumElts; ++i) 1649 Ops[i] = UndefVal; 1650 1651 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 1652 } else { 1653 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 1654 unsigned int LdWidth = LdVT.getSizeInBits(); 1655 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 1656 Align, isVolatile, LdWidth, WidenVT, dl); 1657 } 1658 1659 // If we generate a single load, we can use that for the chain. Otherwise, 1660 // build a factor node to remember the multiple loads are independent and 1661 // chain to that. 1662 SDValue NewChain; 1663 if (LdChain.size() == 1) 1664 NewChain = LdChain[0]; 1665 else 1666 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 1667 LdChain.size()); 1668 1669 // Modified the chain - switch anything that used the old chain to use 1670 // the new one. 1671 ReplaceValueWith(SDValue(N, 1), Chain); 1672 1673 return Result; 1674} 1675 1676SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1677 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1678 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1679 WidenVT, N->getOperand(0)); 1680} 1681 1682SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1683 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1684 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1685 1686 SDValue Cond1 = N->getOperand(0); 1687 MVT CondVT = Cond1.getValueType(); 1688 if (CondVT.isVector()) { 1689 MVT CondEltVT = CondVT.getVectorElementType(); 1690 MVT CondWidenVT = MVT::getVectorVT(CondEltVT, WidenNumElts); 1691 if (getTypeAction(CondVT) == WidenVector) 1692 Cond1 = GetWidenedVector(Cond1); 1693 1694 if (Cond1.getValueType() != CondWidenVT) 1695 Cond1 = ModifyToType(Cond1, CondWidenVT); 1696 } 1697 1698 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1699 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1700 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1701 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1702 WidenVT, Cond1, InOp1, InOp2); 1703} 1704 1705SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1706 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1707 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1708 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1709 InOp1.getValueType(), N->getOperand(0), 1710 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1711} 1712 1713SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1714 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1715 return DAG.getUNDEF(WidenVT); 1716} 1717 1718SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1719 MVT VT = N->getValueType(0); 1720 DebugLoc dl = N->getDebugLoc(); 1721 1722 MVT WidenVT = TLI.getTypeToTransformTo(VT); 1723 unsigned NumElts = VT.getVectorNumElements(); 1724 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1725 1726 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1727 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1728 1729 // Adjust mask based on new input vector length. 1730 SmallVector<int, 16> NewMask; 1731 for (unsigned i = 0; i != NumElts; ++i) { 1732 int Idx = N->getMaskElt(i); 1733 if (Idx < (int)NumElts) 1734 NewMask.push_back(Idx); 1735 else 1736 NewMask.push_back(Idx - NumElts + WidenNumElts); 1737 } 1738 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1739 NewMask.push_back(-1); 1740 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1741} 1742 1743SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1744 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0)); 1745 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1746 1747 SDValue InOp1 = N->getOperand(0); 1748 MVT InVT = InOp1.getValueType(); 1749 assert(InVT.isVector() && "can not widen non vector type"); 1750 MVT WidenInVT = MVT::getVectorVT(InVT.getVectorElementType(), WidenNumElts); 1751 InOp1 = GetWidenedVector(InOp1); 1752 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1753 1754 // Assume that the input and output will be widen appropriately. If not, 1755 // we will have to unroll it at some point. 1756 assert(InOp1.getValueType() == WidenInVT && 1757 InOp2.getValueType() == WidenInVT && 1758 "Input not widened to expected type!"); 1759 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1760 WidenVT, InOp1, InOp2, N->getOperand(2)); 1761} 1762 1763 1764//===----------------------------------------------------------------------===// 1765// Widen Vector Operand 1766//===----------------------------------------------------------------------===// 1767bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1768 DEBUG(cerr << "Widen node operand " << ResNo << ": "; N->dump(&DAG); 1769 cerr << "\n"); 1770 SDValue Res = SDValue(); 1771 1772 switch (N->getOpcode()) { 1773 default: 1774#ifndef NDEBUG 1775 cerr << "WidenVectorOperand op #" << ResNo << ": "; 1776 N->dump(&DAG); cerr << "\n"; 1777#endif 1778 assert(0 && "Do not know how to widen this operator's operand!"); 1779 abort(); 1780 1781 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1782 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1783 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1784 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1785 1786 case ISD::FP_ROUND: 1787 case ISD::FP_TO_SINT: 1788 case ISD::FP_TO_UINT: 1789 case ISD::SINT_TO_FP: 1790 case ISD::UINT_TO_FP: 1791 case ISD::TRUNCATE: 1792 case ISD::SIGN_EXTEND: 1793 case ISD::ZERO_EXTEND: 1794 case ISD::ANY_EXTEND: 1795 Res = WidenVecOp_Convert(N); 1796 break; 1797 } 1798 1799 // If Res is null, the sub-method took care of registering the result. 1800 if (!Res.getNode()) return false; 1801 1802 // If the result is N, the sub-method updated N in place. Tell the legalizer 1803 // core about this. 1804 if (Res.getNode() == N) 1805 return true; 1806 1807 1808 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1809 "Invalid operand expansion"); 1810 1811 ReplaceValueWith(SDValue(N, 0), Res); 1812 return false; 1813} 1814 1815SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1816 // Since the result is legal and the input is illegal, it is unlikely 1817 // that we can fix the input to a legal type so unroll the convert 1818 // into some scalar code and create a nasty build vector. 1819 MVT VT = N->getValueType(0); 1820 MVT EltVT = VT.getVectorElementType(); 1821 DebugLoc dl = N->getDebugLoc(); 1822 unsigned NumElts = VT.getVectorNumElements(); 1823 SDValue InOp = N->getOperand(0); 1824 if (getTypeAction(InOp.getValueType()) == WidenVector) 1825 InOp = GetWidenedVector(InOp); 1826 MVT InVT = InOp.getValueType(); 1827 MVT InEltVT = InVT.getVectorElementType(); 1828 1829 unsigned Opcode = N->getOpcode(); 1830 SmallVector<SDValue, 16> Ops(NumElts); 1831 for (unsigned i=0; i < NumElts; ++i) 1832 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1833 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1834 DAG.getIntPtrConstant(i))); 1835 1836 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1837} 1838 1839SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1840 MVT VT = N->getValueType(0); 1841 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1842 MVT InWidenVT = InOp.getValueType(); 1843 DebugLoc dl = N->getDebugLoc(); 1844 1845 // Check if we can convert between two legal vector types and extract. 1846 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1847 unsigned Size = VT.getSizeInBits(); 1848 if (InWidenSize % Size == 0 && !VT.isVector()) { 1849 unsigned NewNumElts = InWidenSize / Size; 1850 MVT NewVT = MVT::getVectorVT(VT, NewNumElts); 1851 if (TLI.isTypeLegal(NewVT)) { 1852 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1853 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1854 DAG.getIntPtrConstant(0)); 1855 } 1856 } 1857 1858 // Lower the bit-convert to a store/load from the stack. Create the stack 1859 // frame object. Make sure it is aligned for both the source and destination 1860 // types. 1861 SDValue FIPtr = DAG.CreateStackTemporary(InWidenVT, VT); 1862 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1863 const Value *SV = PseudoSourceValue::getFixedStack(FI); 1864 1865 // Emit a store to the stack slot. 1866 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0); 1867 1868 // Result is a load from the stack slot. 1869 return DAG.getLoad(VT, dl, Store, FIPtr, SV, 0); 1870} 1871 1872SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1873 // If the input vector is not legal, it is likely that we will not find a 1874 // legal vector of the same size. Replace the concatenate vector with a 1875 // nasty build vector. 1876 MVT VT = N->getValueType(0); 1877 MVT EltVT = VT.getVectorElementType(); 1878 DebugLoc dl = N->getDebugLoc(); 1879 unsigned NumElts = VT.getVectorNumElements(); 1880 SmallVector<SDValue, 16> Ops(NumElts); 1881 1882 MVT InVT = N->getOperand(0).getValueType(); 1883 unsigned NumInElts = InVT.getVectorNumElements(); 1884 1885 unsigned Idx = 0; 1886 unsigned NumOperands = N->getNumOperands(); 1887 for (unsigned i=0; i < NumOperands; ++i) { 1888 SDValue InOp = N->getOperand(i); 1889 if (getTypeAction(InOp.getValueType()) == WidenVector) 1890 InOp = GetWidenedVector(InOp); 1891 for (unsigned j=0; j < NumInElts; ++j) 1892 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1893 DAG.getIntPtrConstant(j)); 1894 } 1895 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1896} 1897 1898SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1899 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1900 MVT EltVT = InOp.getValueType().getVectorElementType(); 1901 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1902 EltVT, InOp, N->getOperand(1)); 1903} 1904 1905SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1906 // We have to widen the value but we want only to store the original 1907 // vector type. 1908 StoreSDNode *ST = cast<StoreSDNode>(N); 1909 SDValue Chain = ST->getChain(); 1910 SDValue BasePtr = ST->getBasePtr(); 1911 const Value *SV = ST->getSrcValue(); 1912 int SVOffset = ST->getSrcValueOffset(); 1913 unsigned Align = ST->getAlignment(); 1914 bool isVolatile = ST->isVolatile(); 1915 SDValue ValOp = GetWidenedVector(ST->getValue()); 1916 DebugLoc dl = N->getDebugLoc(); 1917 1918 MVT StVT = ST->getMemoryVT(); 1919 MVT ValVT = ValOp.getValueType(); 1920 // It must be true that we the widen vector type is bigger than where 1921 // we need to store. 1922 assert(StVT.isVector() && ValOp.getValueType().isVector()); 1923 assert(StVT.bitsLT(ValOp.getValueType())); 1924 1925 SmallVector<SDValue, 16> StChain; 1926 if (ST->isTruncatingStore()) { 1927 // For truncating stores, we can not play the tricks of chopping legal 1928 // vector types and bit cast it to the right type. Instead, we unroll 1929 // the store. 1930 MVT StEltVT = StVT.getVectorElementType(); 1931 MVT ValEltVT = ValVT.getVectorElementType(); 1932 unsigned Increment = ValEltVT.getSizeInBits() / 8; 1933 unsigned NumElts = StVT.getVectorNumElements(); 1934 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1935 DAG.getIntPtrConstant(0)); 1936 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 1937 SVOffset, StEltVT, 1938 isVolatile, Align)); 1939 unsigned Offset = Increment; 1940 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 1941 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1942 BasePtr, DAG.getIntPtrConstant(Offset)); 1943 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1944 DAG.getIntPtrConstant(0)); 1945 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 1946 SVOffset + Offset, StEltVT, 1947 isVolatile, MinAlign(Align, Offset))); 1948 } 1949 } 1950 else { 1951 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 1952 // Store value 1953 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 1954 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 1955 } 1956 if (StChain.size() == 1) 1957 return StChain[0]; 1958 else 1959 return DAG.getNode(ISD::TokenFactor, dl, 1960 MVT::Other,&StChain[0],StChain.size()); 1961} 1962 1963//===----------------------------------------------------------------------===// 1964// Vector Widening Utilities 1965//===----------------------------------------------------------------------===// 1966 1967 1968// Utility function to find a vector type and its associated element 1969// type from a preferred width and whose vector type must be the same size 1970// as the VecVT. 1971// TLI: Target lowering used to determine legal types. 1972// Width: Preferred width to store. 1973// VecVT: Vector value type whose size we must match. 1974// Returns NewVecVT and NewEltVT - the vector type and its associated 1975// element type. 1976static void FindAssocWidenVecType(const TargetLowering &TLI, unsigned Width, 1977 MVT VecVT, 1978 MVT& NewEltVT, MVT& NewVecVT) { 1979 unsigned EltWidth = Width + 1; 1980 if (TLI.isTypeLegal(VecVT)) { 1981 // We start with the preferred with, making it a power of 2 and find a 1982 // legal vector type of that width. If not, we reduce it by another of 2. 1983 // For incoming type is legal, this process will end as a vector of the 1984 // smallest loadable type should always be legal. 1985 do { 1986 assert(EltWidth > 0); 1987 EltWidth = 1 << Log2_32(EltWidth - 1); 1988 NewEltVT = MVT::getIntegerVT(EltWidth); 1989 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 1990 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts); 1991 } while (!TLI.isTypeLegal(NewVecVT) || 1992 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 1993 } else { 1994 // The incoming vector type is illegal and is the result of widening 1995 // a vector to a power of 2. In this case, we will use the preferred 1996 // with as long as it is a multiple of the incoming vector length. 1997 // The legalization process will eventually make this into a legal type 1998 // and remove the illegal bit converts (which would turn to stack converts 1999 // if they are allow to exist). 2000 do { 2001 assert(EltWidth > 0); 2002 EltWidth = 1 << Log2_32(EltWidth - 1); 2003 NewEltVT = MVT::getIntegerVT(EltWidth); 2004 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2005 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts); 2006 } while (!TLI.isTypeLegal(NewEltVT) || 2007 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2008 } 2009} 2010 2011SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2012 SDValue Chain, 2013 SDValue BasePtr, 2014 const Value *SV, 2015 int SVOffset, 2016 unsigned Alignment, 2017 bool isVolatile, 2018 unsigned LdWidth, 2019 MVT ResType, 2020 DebugLoc dl) { 2021 // The strategy assumes that we can efficiently load powers of two widths. 2022 // The routines chops the vector into the largest power of 2 load and 2023 // can be inserted into a legal vector and then cast the result into the 2024 // vector type we want. This avoids unnecessary stack converts. 2025 2026 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 2027 // the load is nonvolatile, we an use a wider load for the value. 2028 2029 // Find the vector type that can load from. 2030 MVT NewEltVT, NewVecVT; 2031 unsigned NewEltVTWidth; 2032 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2033 NewEltVTWidth = NewEltVT.getSizeInBits(); 2034 2035 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 2036 isVolatile, Alignment); 2037 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2038 LdChain.push_back(LdOp.getValue(1)); 2039 2040 // Check if we can load the element with one instruction 2041 if (LdWidth == NewEltVTWidth) { 2042 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2043 } 2044 2045 unsigned Idx = 1; 2046 LdWidth -= NewEltVTWidth; 2047 unsigned Offset = 0; 2048 2049 while (LdWidth > 0) { 2050 unsigned Increment = NewEltVTWidth / 8; 2051 Offset += Increment; 2052 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2053 DAG.getIntPtrConstant(Increment)); 2054 2055 if (LdWidth < NewEltVTWidth) { 2056 // Our current type we are using is too large, use a smaller size by 2057 // using a smaller power of 2 2058 unsigned oNewEltVTWidth = NewEltVTWidth; 2059 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2060 NewEltVTWidth = NewEltVT.getSizeInBits(); 2061 // Readjust position and vector position based on new load type 2062 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2063 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2064 } 2065 2066 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 2067 SVOffset+Offset, isVolatile, 2068 MinAlign(Alignment, Offset)); 2069 LdChain.push_back(LdOp.getValue(1)); 2070 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 2071 DAG.getIntPtrConstant(Idx++)); 2072 2073 LdWidth -= NewEltVTWidth; 2074 } 2075 2076 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2077} 2078 2079void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2080 SDValue Chain, 2081 SDValue BasePtr, 2082 const Value *SV, 2083 int SVOffset, 2084 unsigned Alignment, 2085 bool isVolatile, 2086 SDValue ValOp, 2087 unsigned StWidth, 2088 DebugLoc dl) { 2089 // Breaks the stores into a series of power of 2 width stores. For any 2090 // width, we convert the vector to the vector of element size that we 2091 // want to store. This avoids requiring a stack convert. 2092 2093 // Find a width of the element type we can store with 2094 MVT WidenVT = ValOp.getValueType(); 2095 MVT NewEltVT, NewVecVT; 2096 2097 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2098 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 2099 2100 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2101 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2102 DAG.getIntPtrConstant(0)); 2103 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 2104 isVolatile, Alignment); 2105 StChain.push_back(StOp); 2106 2107 // Check if we are done 2108 if (StWidth == NewEltVTWidth) { 2109 return; 2110 } 2111 2112 unsigned Idx = 1; 2113 StWidth -= NewEltVTWidth; 2114 unsigned Offset = 0; 2115 2116 while (StWidth > 0) { 2117 unsigned Increment = NewEltVTWidth / 8; 2118 Offset += Increment; 2119 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2120 DAG.getIntPtrConstant(Increment)); 2121 2122 if (StWidth < NewEltVTWidth) { 2123 // Our current type we are using is too large, use a smaller size by 2124 // using a smaller power of 2 2125 unsigned oNewEltVTWidth = NewEltVTWidth; 2126 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2127 NewEltVTWidth = NewEltVT.getSizeInBits(); 2128 // Readjust position and vector position based on new load type 2129 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2130 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2131 } 2132 2133 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2134 DAG.getIntPtrConstant(Idx++)); 2135 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2136 SVOffset + Offset, isVolatile, 2137 MinAlign(Alignment, Offset))); 2138 StWidth -= NewEltVTWidth; 2139 } 2140} 2141 2142/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2143/// input vector must have the same element type as NVT. 2144SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, MVT NVT) { 2145 // Note that InOp might have been widened so it might already have 2146 // the right width or it might need be narrowed. 2147 MVT InVT = InOp.getValueType(); 2148 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2149 "input and widen element type must match"); 2150 DebugLoc dl = InOp.getDebugLoc(); 2151 2152 // Check if InOp already has the right width. 2153 if (InVT == NVT) 2154 return InOp; 2155 2156 unsigned InNumElts = InVT.getVectorNumElements(); 2157 unsigned WidenNumElts = NVT.getVectorNumElements(); 2158 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2159 unsigned NumConcat = WidenNumElts / InNumElts; 2160 SmallVector<SDValue, 16> Ops(NumConcat); 2161 SDValue UndefVal = DAG.getUNDEF(InVT); 2162 Ops[0] = InOp; 2163 for (unsigned i = 1; i != NumConcat; ++i) 2164 Ops[i] = UndefVal; 2165 2166 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2167 } 2168 2169 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2170 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2171 DAG.getIntPtrConstant(0)); 2172 2173 // Fall back to extract and build. 2174 SmallVector<SDValue, 16> Ops(WidenNumElts); 2175 MVT EltVT = NVT.getVectorElementType(); 2176 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2177 unsigned Idx; 2178 for (Idx = 0; Idx < MinNumElts; ++Idx) 2179 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2180 DAG.getIntPtrConstant(Idx)); 2181 2182 SDValue UndefVal = DAG.getUNDEF(EltVT); 2183 for ( ; Idx < WidenNumElts; ++Idx) 2184 Ops[Idx] = UndefVal; 2185 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2186} 2187