LegalizeVectorTypes.cpp revision ff89dcb06fbd103373436e2d0ae85f252fae2254
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in two vectors of half the size. For example, implementing 19// <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26#include "llvm/Support/ErrorHandling.h" 27#include "llvm/Support/raw_ostream.h" 28using namespace llvm; 29 30//===----------------------------------------------------------------------===// 31// Result Vector Scalarization: <1 x ty> -> ty. 32//===----------------------------------------------------------------------===// 33 34void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 35 DEBUG(errs() << "Scalarize node result " << ResNo << ": "; 36 N->dump(&DAG); 37 errs() << "\n"); 38 SDValue R = SDValue(); 39 40 switch (N->getOpcode()) { 41 default: 42#ifndef NDEBUG 43 errs() << "ScalarizeVectorResult #" << ResNo << ": "; 44 N->dump(&DAG); 45 errs() << "\n"; 46#endif 47 llvm_unreachable("Do not know how to scalarize the result of this operator!"); 48 49 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 50 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 51 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 52 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 53 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 54 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 55 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 56 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 57 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 58 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 59 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 60 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 61 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 62 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 63 64 case ISD::CTLZ: 65 case ISD::CTPOP: 66 case ISD::CTTZ: 67 case ISD::FABS: 68 case ISD::FCOS: 69 case ISD::FNEG: 70 case ISD::FP_TO_SINT: 71 case ISD::FP_TO_UINT: 72 case ISD::FSIN: 73 case ISD::FSQRT: 74 case ISD::FTRUNC: 75 case ISD::FFLOOR: 76 case ISD::FCEIL: 77 case ISD::FRINT: 78 case ISD::FNEARBYINT: 79 case ISD::UINT_TO_FP: 80 case ISD::SINT_TO_FP: 81 case ISD::TRUNCATE: 82 case ISD::SIGN_EXTEND: 83 case ISD::ZERO_EXTEND: 84 case ISD::ANY_EXTEND: 85 R = ScalarizeVecRes_UnaryOp(N); 86 break; 87 88 case ISD::ADD: 89 case ISD::AND: 90 case ISD::FADD: 91 case ISD::FDIV: 92 case ISD::FMUL: 93 case ISD::FPOW: 94 case ISD::FREM: 95 case ISD::FSUB: 96 case ISD::MUL: 97 case ISD::OR: 98 case ISD::SDIV: 99 case ISD::SREM: 100 case ISD::SUB: 101 case ISD::UDIV: 102 case ISD::UREM: 103 case ISD::XOR: 104 case ISD::SHL: 105 case ISD::SRA: 106 case ISD::SRL: 107 R = ScalarizeVecRes_BinOp(N); 108 break; 109 } 110 111 // If R is null, the sub-method took care of registering the result. 112 if (R.getNode()) 113 SetScalarizedVector(SDValue(N, ResNo), R); 114} 115 116SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 117 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 118 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 119 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 120 LHS.getValueType(), LHS, RHS); 121} 122 123SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 124 EVT NewVT = N->getValueType(0).getVectorElementType(); 125 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 126 NewVT, N->getOperand(0)); 127} 128 129SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 130 EVT NewVT = N->getValueType(0).getVectorElementType(); 131 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 132 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 133 Op0, DAG.getValueType(NewVT), 134 DAG.getValueType(Op0.getValueType()), 135 N->getOperand(3), 136 N->getOperand(4), 137 cast<CvtRndSatSDNode>(N)->getCvtCode()); 138} 139 140SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 141 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 142 N->getValueType(0).getVectorElementType(), 143 N->getOperand(0), N->getOperand(1)); 144} 145 146SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 147 SDValue Op = GetScalarizedVector(N->getOperand(0)); 148 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 149 Op.getValueType(), Op, N->getOperand(1)); 150} 151 152SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 153 // The value to insert may have a wider type than the vector element type, 154 // so be sure to truncate it to the element type if necessary. 155 SDValue Op = N->getOperand(1); 156 EVT EltVT = N->getValueType(0).getVectorElementType(); 157 if (Op.getValueType() != EltVT) 158 // FIXME: Can this happen for floating point types? 159 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 160 return Op; 161} 162 163SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 164 assert(N->isUnindexed() && "Indexed vector load?"); 165 166 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 167 N->getExtensionType(), 168 N->getValueType(0).getVectorElementType(), 169 N->getChain(), N->getBasePtr(), 170 DAG.getUNDEF(N->getBasePtr().getValueType()), 171 N->getSrcValue(), N->getSrcValueOffset(), 172 N->getMemoryVT().getVectorElementType(), 173 N->isVolatile(), N->getOriginalAlignment()); 174 175 // Legalized the chain result - switch anything that used the old chain to 176 // use the new one. 177 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 178 return Result; 179} 180 181SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 182 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 183 EVT DestVT = N->getValueType(0).getVectorElementType(); 184 SDValue Op = GetScalarizedVector(N->getOperand(0)); 185 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 186} 187 188SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 189 // If the operand is wider than the vector element type then it is implicitly 190 // truncated. Make that explicit here. 191 EVT EltVT = N->getValueType(0).getVectorElementType(); 192 SDValue InOp = N->getOperand(0); 193 if (InOp.getValueType() != EltVT) 194 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 195 return InOp; 196} 197 198SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 199 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 200 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 201 LHS.getValueType(), N->getOperand(0), LHS, 202 GetScalarizedVector(N->getOperand(2))); 203} 204 205SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 206 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 207 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 208 N->getOperand(0), N->getOperand(1), 209 LHS, GetScalarizedVector(N->getOperand(3)), 210 N->getOperand(4)); 211} 212 213SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 214 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 215 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 216 DebugLoc DL = N->getDebugLoc(); 217 218 // Turn it into a scalar SETCC. 219 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 220} 221 222SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 223 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 224} 225 226SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 227 // Figure out if the scalar is the LHS or RHS and return it. 228 SDValue Arg = N->getOperand(2).getOperand(0); 229 if (Arg.getOpcode() == ISD::UNDEF) 230 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 231 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 232 return GetScalarizedVector(N->getOperand(Op)); 233} 234 235SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 236 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 237 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 238 EVT NVT = N->getValueType(0).getVectorElementType(); 239 EVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 240 DebugLoc DL = N->getDebugLoc(); 241 242 // Turn it into a scalar SETCC. 243 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 244 245 // VSETCC always returns a sign-extended value, while SETCC may not. The 246 // SETCC result type may not match the vector element type. Correct these. 247 if (NVT.bitsLE(SVT)) { 248 // The SETCC result type is bigger than the vector element type. 249 // Ensure the SETCC result is sign-extended. 250 if (TLI.getBooleanContents() != 251 TargetLowering::ZeroOrNegativeOneBooleanContent) 252 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 253 DAG.getValueType(MVT::i1)); 254 // Truncate to the final type. 255 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 256 } 257 258 // The SETCC result type is smaller than the vector element type. 259 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 260 if (TLI.getBooleanContents() != 261 TargetLowering::ZeroOrNegativeOneBooleanContent) 262 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 263 // Sign extend to the final type. 264 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 265} 266 267 268//===----------------------------------------------------------------------===// 269// Operand Vector Scalarization <1 x ty> -> ty. 270//===----------------------------------------------------------------------===// 271 272bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 273 DEBUG(errs() << "Scalarize node operand " << OpNo << ": "; 274 N->dump(&DAG); 275 errs() << "\n"); 276 SDValue Res = SDValue(); 277 278 if (Res.getNode() == 0) { 279 switch (N->getOpcode()) { 280 default: 281#ifndef NDEBUG 282 errs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 283 N->dump(&DAG); 284 errs() << "\n"; 285#endif 286 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 287 case ISD::BIT_CONVERT: 288 Res = ScalarizeVecOp_BIT_CONVERT(N); 289 break; 290 case ISD::CONCAT_VECTORS: 291 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 292 break; 293 case ISD::EXTRACT_VECTOR_ELT: 294 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 295 break; 296 case ISD::STORE: 297 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 298 break; 299 } 300 } 301 302 // If the result is null, the sub-method took care of registering results etc. 303 if (!Res.getNode()) return false; 304 305 // If the result is N, the sub-method updated N in place. Tell the legalizer 306 // core about this. 307 if (Res.getNode() == N) 308 return true; 309 310 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 311 "Invalid operand expansion"); 312 313 ReplaceValueWith(SDValue(N, 0), Res); 314 return false; 315} 316 317/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 318/// to be scalarized, it must be <1 x ty>. Convert the element instead. 319SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 320 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 321 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 322 N->getValueType(0), Elt); 323} 324 325/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 326/// use a BUILD_VECTOR instead. 327SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 328 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 329 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 330 Ops[i] = GetScalarizedVector(N->getOperand(i)); 331 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 332 &Ops[0], Ops.size()); 333} 334 335/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 336/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 337/// index. 338SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 339 SDValue Res = GetScalarizedVector(N->getOperand(0)); 340 if (Res.getValueType() != N->getValueType(0)) 341 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 342 Res); 343 return Res; 344} 345 346/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 347/// scalarized, it must be <1 x ty>. Just store the element. 348SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 349 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 350 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 351 DebugLoc dl = N->getDebugLoc(); 352 353 if (N->isTruncatingStore()) 354 return DAG.getTruncStore(N->getChain(), dl, 355 GetScalarizedVector(N->getOperand(1)), 356 N->getBasePtr(), 357 N->getSrcValue(), N->getSrcValueOffset(), 358 N->getMemoryVT().getVectorElementType(), 359 N->isVolatile(), N->getAlignment()); 360 361 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 362 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 363 N->isVolatile(), N->getOriginalAlignment()); 364} 365 366 367//===----------------------------------------------------------------------===// 368// Result Vector Splitting 369//===----------------------------------------------------------------------===// 370 371/// SplitVectorResult - This method is called when the specified result of the 372/// specified node is found to need vector splitting. At this point, the node 373/// may also have invalid operands or may have other results that need 374/// legalization, we just know that (at least) one result needs vector 375/// splitting. 376void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 377 DEBUG(errs() << "Split node result: "; 378 N->dump(&DAG); 379 errs() << "\n"); 380 SDValue Lo, Hi; 381 382 switch (N->getOpcode()) { 383 default: 384#ifndef NDEBUG 385 errs() << "SplitVectorResult #" << ResNo << ": "; 386 N->dump(&DAG); 387 errs() << "\n"; 388#endif 389 llvm_unreachable("Do not know how to split the result of this operator!"); 390 391 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 392 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 393 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 394 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 395 396 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 397 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 398 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 399 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 400 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 401 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 402 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 403 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 404 case ISD::LOAD: 405 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 406 break; 407 case ISD::SETCC: 408 case ISD::VSETCC: 409 SplitVecRes_SETCC(N, Lo, Hi); 410 break; 411 case ISD::VECTOR_SHUFFLE: 412 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 413 break; 414 415 case ISD::CTTZ: 416 case ISD::CTLZ: 417 case ISD::CTPOP: 418 case ISD::FNEG: 419 case ISD::FABS: 420 case ISD::FSQRT: 421 case ISD::FSIN: 422 case ISD::FCOS: 423 case ISD::FTRUNC: 424 case ISD::FFLOOR: 425 case ISD::FCEIL: 426 case ISD::FRINT: 427 case ISD::FNEARBYINT: 428 case ISD::FP_TO_SINT: 429 case ISD::FP_TO_UINT: 430 case ISD::SINT_TO_FP: 431 case ISD::UINT_TO_FP: 432 case ISD::TRUNCATE: 433 case ISD::SIGN_EXTEND: 434 case ISD::ZERO_EXTEND: 435 case ISD::ANY_EXTEND: 436 SplitVecRes_UnaryOp(N, Lo, Hi); 437 break; 438 439 case ISD::ADD: 440 case ISD::SUB: 441 case ISD::MUL: 442 case ISD::FADD: 443 case ISD::FSUB: 444 case ISD::FMUL: 445 case ISD::SDIV: 446 case ISD::UDIV: 447 case ISD::FDIV: 448 case ISD::FPOW: 449 case ISD::AND: 450 case ISD::OR: 451 case ISD::XOR: 452 case ISD::SHL: 453 case ISD::SRA: 454 case ISD::SRL: 455 case ISD::UREM: 456 case ISD::SREM: 457 case ISD::FREM: 458 SplitVecRes_BinOp(N, Lo, Hi); 459 break; 460 } 461 462 // If Lo/Hi is null, the sub-method took care of registering results etc. 463 if (Lo.getNode()) 464 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 465} 466 467void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 468 SDValue &Hi) { 469 SDValue LHSLo, LHSHi; 470 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 471 SDValue RHSLo, RHSHi; 472 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 473 DebugLoc dl = N->getDebugLoc(); 474 475 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 476 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 477} 478 479void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 480 SDValue &Hi) { 481 // We know the result is a vector. The input may be either a vector or a 482 // scalar value. 483 EVT LoVT, HiVT; 484 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 485 DebugLoc dl = N->getDebugLoc(); 486 487 SDValue InOp = N->getOperand(0); 488 EVT InVT = InOp.getValueType(); 489 490 // Handle some special cases efficiently. 491 switch (getTypeAction(InVT)) { 492 default: 493 assert(false && "Unknown type action!"); 494 case Legal: 495 case PromoteInteger: 496 case SoftenFloat: 497 case ScalarizeVector: 498 break; 499 case ExpandInteger: 500 case ExpandFloat: 501 // A scalar to vector conversion, where the scalar needs expansion. 502 // If the vector is being split in two then we can just convert the 503 // expanded pieces. 504 if (LoVT == HiVT) { 505 GetExpandedOp(InOp, Lo, Hi); 506 if (TLI.isBigEndian()) 507 std::swap(Lo, Hi); 508 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 509 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 510 return; 511 } 512 break; 513 case SplitVector: 514 // If the input is a vector that needs to be split, convert each split 515 // piece of the input now. 516 GetSplitVector(InOp, Lo, Hi); 517 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 518 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 519 return; 520 } 521 522 // In the general case, convert the input to an integer and split it by hand. 523 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 524 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 525 if (TLI.isBigEndian()) 526 std::swap(LoIntVT, HiIntVT); 527 528 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 529 530 if (TLI.isBigEndian()) 531 std::swap(Lo, Hi); 532 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 533 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 534} 535 536void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 537 SDValue &Hi) { 538 EVT LoVT, HiVT; 539 DebugLoc dl = N->getDebugLoc(); 540 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 541 unsigned LoNumElts = LoVT.getVectorNumElements(); 542 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 543 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 544 545 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 546 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 547} 548 549void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 550 SDValue &Hi) { 551 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 552 DebugLoc dl = N->getDebugLoc(); 553 unsigned NumSubvectors = N->getNumOperands() / 2; 554 if (NumSubvectors == 1) { 555 Lo = N->getOperand(0); 556 Hi = N->getOperand(1); 557 return; 558 } 559 560 EVT LoVT, HiVT; 561 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 562 563 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 564 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 565 566 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 567 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 568} 569 570void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 571 SDValue &Hi) { 572 EVT LoVT, HiVT; 573 DebugLoc dl = N->getDebugLoc(); 574 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 575 576 SDValue DTyOpLo = DAG.getValueType(LoVT); 577 SDValue DTyOpHi = DAG.getValueType(HiVT); 578 579 SDValue RndOp = N->getOperand(3); 580 SDValue SatOp = N->getOperand(4); 581 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 582 583 // Split the input. 584 SDValue VLo, VHi; 585 EVT InVT = N->getOperand(0).getValueType(); 586 switch (getTypeAction(InVT)) { 587 default: llvm_unreachable("Unexpected type action!"); 588 case Legal: { 589 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 590 LoVT.getVectorNumElements()); 591 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 592 DAG.getIntPtrConstant(0)); 593 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 594 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 595 break; 596 } 597 case SplitVector: 598 GetSplitVector(N->getOperand(0), VLo, VHi); 599 break; 600 case WidenVector: { 601 // If the result needs to be split and the input needs to be widened, 602 // the two types must have different lengths. Use the widened result 603 // and extract from it to do the split. 604 SDValue InOp = GetWidenedVector(N->getOperand(0)); 605 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 606 LoVT.getVectorNumElements()); 607 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 608 DAG.getIntPtrConstant(0)); 609 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 610 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 611 break; 612 } 613 } 614 615 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 616 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 617 618 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 619 CvtCode); 620 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 621 CvtCode); 622} 623 624void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 625 SDValue &Hi) { 626 SDValue Vec = N->getOperand(0); 627 SDValue Idx = N->getOperand(1); 628 EVT IdxVT = Idx.getValueType(); 629 DebugLoc dl = N->getDebugLoc(); 630 631 EVT LoVT, HiVT; 632 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 633 634 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 635 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 636 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 637 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 638} 639 640void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 641 SDValue &Hi) { 642 DebugLoc dl = N->getDebugLoc(); 643 GetSplitVector(N->getOperand(0), Lo, Hi); 644 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 645 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 646} 647 648void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 649 SDValue &Hi) { 650 SDValue Vec = N->getOperand(0); 651 SDValue Elt = N->getOperand(1); 652 SDValue Idx = N->getOperand(2); 653 DebugLoc dl = N->getDebugLoc(); 654 GetSplitVector(Vec, Lo, Hi); 655 656 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 657 unsigned IdxVal = CIdx->getZExtValue(); 658 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 659 if (IdxVal < LoNumElts) 660 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 661 Lo.getValueType(), Lo, Elt, Idx); 662 else 663 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 664 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 665 return; 666 } 667 668 // Spill the vector to the stack. 669 EVT VecVT = Vec.getValueType(); 670 EVT EltVT = VecVT.getVectorElementType(); 671 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 672 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 673 674 // Store the new element. This may be larger than the vector element type, 675 // so use a truncating store. 676 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 677 unsigned Alignment = 678 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext())); 679 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 680 681 // Load the Lo part from the stack slot. 682 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 683 684 // Increment the pointer to the other part. 685 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 686 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 687 DAG.getIntPtrConstant(IncrementSize)); 688 689 // Load the Hi part from the stack slot. 690 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 691 MinAlign(Alignment, IncrementSize)); 692} 693 694void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 695 SDValue &Hi) { 696 EVT LoVT, HiVT; 697 DebugLoc dl = N->getDebugLoc(); 698 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 699 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 700 Hi = DAG.getUNDEF(HiVT); 701} 702 703void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 704 SDValue &Hi) { 705 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 706 EVT LoVT, HiVT; 707 DebugLoc dl = LD->getDebugLoc(); 708 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 709 710 ISD::LoadExtType ExtType = LD->getExtensionType(); 711 SDValue Ch = LD->getChain(); 712 SDValue Ptr = LD->getBasePtr(); 713 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 714 const Value *SV = LD->getSrcValue(); 715 int SVOffset = LD->getSrcValueOffset(); 716 EVT MemoryVT = LD->getMemoryVT(); 717 unsigned Alignment = LD->getOriginalAlignment(); 718 bool isVolatile = LD->isVolatile(); 719 720 EVT LoMemVT, HiMemVT; 721 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 722 723 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 724 SV, SVOffset, LoMemVT, isVolatile, Alignment); 725 726 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 727 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 728 DAG.getIntPtrConstant(IncrementSize)); 729 SVOffset += IncrementSize; 730 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 731 SV, SVOffset, HiMemVT, isVolatile, Alignment); 732 733 // Build a factor node to remember that this load is independent of the 734 // other one. 735 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 736 Hi.getValue(1)); 737 738 // Legalized the chain result - switch anything that used the old chain to 739 // use the new one. 740 ReplaceValueWith(SDValue(LD, 1), Ch); 741} 742 743void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 744 EVT LoVT, HiVT; 745 DebugLoc DL = N->getDebugLoc(); 746 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 747 748 // Split the input. 749 EVT InVT = N->getOperand(0).getValueType(); 750 SDValue LL, LH, RL, RH; 751 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 752 LoVT.getVectorNumElements()); 753 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 754 DAG.getIntPtrConstant(0)); 755 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 756 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 757 758 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 759 DAG.getIntPtrConstant(0)); 760 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 761 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 762 763 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 764 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 765} 766 767void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 768 SDValue &Hi) { 769 // Get the dest types - they may not match the input types, e.g. int_to_fp. 770 EVT LoVT, HiVT; 771 DebugLoc dl = N->getDebugLoc(); 772 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 773 774 // Split the input. 775 EVT InVT = N->getOperand(0).getValueType(); 776 switch (getTypeAction(InVT)) { 777 default: llvm_unreachable("Unexpected type action!"); 778 case Legal: { 779 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 780 LoVT.getVectorNumElements()); 781 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 782 DAG.getIntPtrConstant(0)); 783 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 784 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 785 break; 786 } 787 case SplitVector: 788 GetSplitVector(N->getOperand(0), Lo, Hi); 789 break; 790 case WidenVector: { 791 // If the result needs to be split and the input needs to be widened, 792 // the two types must have different lengths. Use the widened result 793 // and extract from it to do the split. 794 SDValue InOp = GetWidenedVector(N->getOperand(0)); 795 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 796 LoVT.getVectorNumElements()); 797 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 798 DAG.getIntPtrConstant(0)); 799 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 800 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 801 break; 802 } 803 } 804 805 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 806 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 807} 808 809void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 810 SDValue &Lo, SDValue &Hi) { 811 // The low and high parts of the original input give four input vectors. 812 SDValue Inputs[4]; 813 DebugLoc dl = N->getDebugLoc(); 814 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 815 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 816 EVT NewVT = Inputs[0].getValueType(); 817 unsigned NewElts = NewVT.getVectorNumElements(); 818 819 // If Lo or Hi uses elements from at most two of the four input vectors, then 820 // express it as a vector shuffle of those two inputs. Otherwise extract the 821 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 822 SmallVector<int, 16> Ops; 823 for (unsigned High = 0; High < 2; ++High) { 824 SDValue &Output = High ? Hi : Lo; 825 826 // Build a shuffle mask for the output, discovering on the fly which 827 // input vectors to use as shuffle operands (recorded in InputUsed). 828 // If building a suitable shuffle vector proves too hard, then bail 829 // out with useBuildVector set. 830 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 831 unsigned FirstMaskIdx = High * NewElts; 832 bool useBuildVector = false; 833 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 834 // The mask element. This indexes into the input. 835 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 836 837 // The input vector this mask element indexes into. 838 unsigned Input = (unsigned)Idx / NewElts; 839 840 if (Input >= array_lengthof(Inputs)) { 841 // The mask element does not index into any input vector. 842 Ops.push_back(-1); 843 continue; 844 } 845 846 // Turn the index into an offset from the start of the input vector. 847 Idx -= Input * NewElts; 848 849 // Find or create a shuffle vector operand to hold this input. 850 unsigned OpNo; 851 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 852 if (InputUsed[OpNo] == Input) { 853 // This input vector is already an operand. 854 break; 855 } else if (InputUsed[OpNo] == -1U) { 856 // Create a new operand for this input vector. 857 InputUsed[OpNo] = Input; 858 break; 859 } 860 } 861 862 if (OpNo >= array_lengthof(InputUsed)) { 863 // More than two input vectors used! Give up on trying to create a 864 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 865 useBuildVector = true; 866 break; 867 } 868 869 // Add the mask index for the new shuffle vector. 870 Ops.push_back(Idx + OpNo * NewElts); 871 } 872 873 if (useBuildVector) { 874 EVT EltVT = NewVT.getVectorElementType(); 875 SmallVector<SDValue, 16> SVOps; 876 877 // Extract the input elements by hand. 878 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 879 // The mask element. This indexes into the input. 880 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 881 882 // The input vector this mask element indexes into. 883 unsigned Input = (unsigned)Idx / NewElts; 884 885 if (Input >= array_lengthof(Inputs)) { 886 // The mask element is "undef" or indexes off the end of the input. 887 SVOps.push_back(DAG.getUNDEF(EltVT)); 888 continue; 889 } 890 891 // Turn the index into an offset from the start of the input vector. 892 Idx -= Input * NewElts; 893 894 // Extract the vector element by hand. 895 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 896 Inputs[Input], DAG.getIntPtrConstant(Idx))); 897 } 898 899 // Construct the Lo/Hi output using a BUILD_VECTOR. 900 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 901 } else if (InputUsed[0] == -1U) { 902 // No input vectors were used! The result is undefined. 903 Output = DAG.getUNDEF(NewVT); 904 } else { 905 SDValue Op0 = Inputs[InputUsed[0]]; 906 // If only one input was used, use an undefined vector for the other. 907 SDValue Op1 = InputUsed[1] == -1U ? 908 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 909 // At least one input vector was used. Create a new shuffle vector. 910 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 911 } 912 913 Ops.clear(); 914 } 915} 916 917 918//===----------------------------------------------------------------------===// 919// Operand Vector Splitting 920//===----------------------------------------------------------------------===// 921 922/// SplitVectorOperand - This method is called when the specified operand of the 923/// specified node is found to need vector splitting. At this point, all of the 924/// result types of the node are known to be legal, but other operands of the 925/// node may need legalization as well as the specified one. 926bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 927 DEBUG(errs() << "Split node operand: "; 928 N->dump(&DAG); 929 errs() << "\n"); 930 SDValue Res = SDValue(); 931 932 if (Res.getNode() == 0) { 933 switch (N->getOpcode()) { 934 default: 935#ifndef NDEBUG 936 errs() << "SplitVectorOperand Op #" << OpNo << ": "; 937 N->dump(&DAG); 938 errs() << "\n"; 939#endif 940 llvm_unreachable("Do not know how to split this operator's operand!"); 941 942 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 943 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 944 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 945 case ISD::STORE: 946 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 947 break; 948 949 case ISD::CTTZ: 950 case ISD::CTLZ: 951 case ISD::CTPOP: 952 case ISD::FP_TO_SINT: 953 case ISD::FP_TO_UINT: 954 case ISD::SINT_TO_FP: 955 case ISD::UINT_TO_FP: 956 case ISD::TRUNCATE: 957 case ISD::SIGN_EXTEND: 958 case ISD::ZERO_EXTEND: 959 case ISD::ANY_EXTEND: 960 Res = SplitVecOp_UnaryOp(N); 961 break; 962 } 963 } 964 965 // If the result is null, the sub-method took care of registering results etc. 966 if (!Res.getNode()) return false; 967 968 // If the result is N, the sub-method updated N in place. Tell the legalizer 969 // core about this. 970 if (Res.getNode() == N) 971 return true; 972 973 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 974 "Invalid operand expansion"); 975 976 ReplaceValueWith(SDValue(N, 0), Res); 977 return false; 978} 979 980SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 981 // The result has a legal vector type, but the input needs splitting. 982 EVT ResVT = N->getValueType(0); 983 SDValue Lo, Hi; 984 DebugLoc dl = N->getDebugLoc(); 985 GetSplitVector(N->getOperand(0), Lo, Hi); 986 EVT InVT = Lo.getValueType(); 987 988 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 989 InVT.getVectorNumElements()); 990 991 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 992 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 993 994 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 995} 996 997SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 998 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 999 // end up being split all the way down to individual components. Convert the 1000 // split pieces into integers and reassemble. 1001 SDValue Lo, Hi; 1002 GetSplitVector(N->getOperand(0), Lo, Hi); 1003 Lo = BitConvertToInteger(Lo); 1004 Hi = BitConvertToInteger(Hi); 1005 1006 if (TLI.isBigEndian()) 1007 std::swap(Lo, Hi); 1008 1009 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 1010 JoinIntegers(Lo, Hi)); 1011} 1012 1013SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1014 // We know that the extracted result type is legal. For now, assume the index 1015 // is a constant. 1016 EVT SubVT = N->getValueType(0); 1017 SDValue Idx = N->getOperand(1); 1018 DebugLoc dl = N->getDebugLoc(); 1019 SDValue Lo, Hi; 1020 GetSplitVector(N->getOperand(0), Lo, Hi); 1021 1022 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1023 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1024 1025 if (IdxVal < LoElts) { 1026 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1027 "Extracted subvector crosses vector split!"); 1028 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1029 } else { 1030 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1031 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1032 } 1033} 1034 1035SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1036 SDValue Vec = N->getOperand(0); 1037 SDValue Idx = N->getOperand(1); 1038 EVT VecVT = Vec.getValueType(); 1039 1040 if (isa<ConstantSDNode>(Idx)) { 1041 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1042 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1043 1044 SDValue Lo, Hi; 1045 GetSplitVector(Vec, Lo, Hi); 1046 1047 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1048 1049 if (IdxVal < LoElts) 1050 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1051 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1052 DAG.getConstant(IdxVal - LoElts, 1053 Idx.getValueType())); 1054 } 1055 1056 // Store the vector to the stack. 1057 EVT EltVT = VecVT.getVectorElementType(); 1058 DebugLoc dl = N->getDebugLoc(); 1059 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1060 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1061 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1062 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1063 1064 // Load back the required element. 1065 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1066 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1067 SV, 0, EltVT); 1068} 1069 1070SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1071 assert(N->isUnindexed() && "Indexed store of vector?"); 1072 assert(OpNo == 1 && "Can only split the stored value"); 1073 DebugLoc dl = N->getDebugLoc(); 1074 1075 bool isTruncating = N->isTruncatingStore(); 1076 SDValue Ch = N->getChain(); 1077 SDValue Ptr = N->getBasePtr(); 1078 int SVOffset = N->getSrcValueOffset(); 1079 EVT MemoryVT = N->getMemoryVT(); 1080 unsigned Alignment = N->getOriginalAlignment(); 1081 bool isVol = N->isVolatile(); 1082 SDValue Lo, Hi; 1083 GetSplitVector(N->getOperand(1), Lo, Hi); 1084 1085 EVT LoMemVT, HiMemVT; 1086 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1087 1088 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1089 1090 if (isTruncating) 1091 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1092 LoMemVT, isVol, Alignment); 1093 else 1094 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1095 isVol, Alignment); 1096 1097 // Increment the pointer to the other half. 1098 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1099 DAG.getIntPtrConstant(IncrementSize)); 1100 SVOffset += IncrementSize; 1101 1102 if (isTruncating) 1103 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1104 HiMemVT, isVol, Alignment); 1105 else 1106 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1107 isVol, Alignment); 1108 1109 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1110} 1111 1112 1113//===----------------------------------------------------------------------===// 1114// Result Vector Widening 1115//===----------------------------------------------------------------------===// 1116 1117void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1118 DEBUG(errs() << "Widen node result " << ResNo << ": "; 1119 N->dump(&DAG); 1120 errs() << "\n"); 1121 SDValue Res = SDValue(); 1122 1123 switch (N->getOpcode()) { 1124 default: 1125#ifndef NDEBUG 1126 errs() << "WidenVectorResult #" << ResNo << ": "; 1127 N->dump(&DAG); 1128 errs() << "\n"; 1129#endif 1130 llvm_unreachable("Do not know how to widen the result of this operator!"); 1131 1132 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1133 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1134 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1135 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1136 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1137 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1138 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1139 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1140 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1141 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1142 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1143 case ISD::VECTOR_SHUFFLE: 1144 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1145 break; 1146 case ISD::VSETCC: 1147 Res = WidenVecRes_VSETCC(N); 1148 break; 1149 1150 case ISD::ADD: 1151 case ISD::AND: 1152 case ISD::BSWAP: 1153 case ISD::FADD: 1154 case ISD::FCOPYSIGN: 1155 case ISD::FDIV: 1156 case ISD::FMUL: 1157 case ISD::FPOW: 1158 case ISD::FPOWI: 1159 case ISD::FREM: 1160 case ISD::FSUB: 1161 case ISD::MUL: 1162 case ISD::MULHS: 1163 case ISD::MULHU: 1164 case ISD::OR: 1165 case ISD::SDIV: 1166 case ISD::SREM: 1167 case ISD::UDIV: 1168 case ISD::UREM: 1169 case ISD::SUB: 1170 case ISD::XOR: 1171 Res = WidenVecRes_Binary(N); 1172 break; 1173 1174 case ISD::SHL: 1175 case ISD::SRA: 1176 case ISD::SRL: 1177 Res = WidenVecRes_Shift(N); 1178 break; 1179 1180 case ISD::FP_ROUND: 1181 case ISD::FP_TO_SINT: 1182 case ISD::FP_TO_UINT: 1183 case ISD::SINT_TO_FP: 1184 case ISD::UINT_TO_FP: 1185 case ISD::TRUNCATE: 1186 case ISD::SIGN_EXTEND: 1187 case ISD::ZERO_EXTEND: 1188 case ISD::ANY_EXTEND: 1189 Res = WidenVecRes_Convert(N); 1190 break; 1191 1192 case ISD::CTLZ: 1193 case ISD::CTPOP: 1194 case ISD::CTTZ: 1195 case ISD::FABS: 1196 case ISD::FCOS: 1197 case ISD::FNEG: 1198 case ISD::FSIN: 1199 case ISD::FSQRT: 1200 Res = WidenVecRes_Unary(N); 1201 break; 1202 } 1203 1204 // If Res is null, the sub-method took care of registering the result. 1205 if (Res.getNode()) 1206 SetWidenedVector(SDValue(N, ResNo), Res); 1207} 1208 1209SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1210 // Binary op widening. 1211 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1212 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1213 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1214 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1215} 1216 1217SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1218 SDValue InOp = N->getOperand(0); 1219 DebugLoc dl = N->getDebugLoc(); 1220 1221 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1222 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1223 1224 EVT InVT = InOp.getValueType(); 1225 EVT InEltVT = InVT.getVectorElementType(); 1226 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1227 1228 unsigned Opcode = N->getOpcode(); 1229 unsigned InVTNumElts = InVT.getVectorNumElements(); 1230 1231 if (getTypeAction(InVT) == WidenVector) { 1232 InOp = GetWidenedVector(N->getOperand(0)); 1233 InVT = InOp.getValueType(); 1234 InVTNumElts = InVT.getVectorNumElements(); 1235 if (InVTNumElts == WidenNumElts) 1236 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1237 } 1238 1239 if (TLI.isTypeLegal(InWidenVT)) { 1240 // Because the result and the input are different vector types, widening 1241 // the result could create a legal type but widening the input might make 1242 // it an illegal type that might lead to repeatedly splitting the input 1243 // and then widening it. To avoid this, we widen the input only if 1244 // it results in a legal type. 1245 if (WidenNumElts % InVTNumElts == 0) { 1246 // Widen the input and call convert on the widened input vector. 1247 unsigned NumConcat = WidenNumElts/InVTNumElts; 1248 SmallVector<SDValue, 16> Ops(NumConcat); 1249 Ops[0] = InOp; 1250 SDValue UndefVal = DAG.getUNDEF(InVT); 1251 for (unsigned i = 1; i != NumConcat; ++i) 1252 Ops[i] = UndefVal; 1253 return DAG.getNode(Opcode, dl, WidenVT, 1254 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1255 &Ops[0], NumConcat)); 1256 } 1257 1258 if (InVTNumElts % WidenNumElts == 0) { 1259 // Extract the input and convert the shorten input vector. 1260 return DAG.getNode(Opcode, dl, WidenVT, 1261 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1262 InOp, DAG.getIntPtrConstant(0))); 1263 } 1264 } 1265 1266 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1267 SmallVector<SDValue, 16> Ops(WidenNumElts); 1268 EVT EltVT = WidenVT.getVectorElementType(); 1269 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1270 unsigned i; 1271 for (i=0; i < MinElts; ++i) 1272 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1273 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1274 DAG.getIntPtrConstant(i))); 1275 1276 SDValue UndefVal = DAG.getUNDEF(EltVT); 1277 for (; i < WidenNumElts; ++i) 1278 Ops[i] = UndefVal; 1279 1280 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1281} 1282 1283SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1284 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1285 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1286 SDValue ShOp = N->getOperand(1); 1287 1288 EVT ShVT = ShOp.getValueType(); 1289 if (getTypeAction(ShVT) == WidenVector) { 1290 ShOp = GetWidenedVector(ShOp); 1291 ShVT = ShOp.getValueType(); 1292 } 1293 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(), 1294 WidenVT.getVectorNumElements()); 1295 if (ShVT != ShWidenVT) 1296 ShOp = ModifyToType(ShOp, ShWidenVT); 1297 1298 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1299} 1300 1301SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1302 // Unary op widening. 1303 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1304 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1305 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1306} 1307 1308SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1309 SDValue InOp = N->getOperand(0); 1310 EVT InVT = InOp.getValueType(); 1311 EVT VT = N->getValueType(0); 1312 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1313 DebugLoc dl = N->getDebugLoc(); 1314 1315 switch (getTypeAction(InVT)) { 1316 default: 1317 assert(false && "Unknown type action!"); 1318 break; 1319 case Legal: 1320 break; 1321 case PromoteInteger: 1322 // If the InOp is promoted to the same size, convert it. Otherwise, 1323 // fall out of the switch and widen the promoted input. 1324 InOp = GetPromotedInteger(InOp); 1325 InVT = InOp.getValueType(); 1326 if (WidenVT.bitsEq(InVT)) 1327 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1328 break; 1329 case SoftenFloat: 1330 case ExpandInteger: 1331 case ExpandFloat: 1332 case ScalarizeVector: 1333 case SplitVector: 1334 break; 1335 case WidenVector: 1336 // If the InOp is widened to the same size, convert it. Otherwise, fall 1337 // out of the switch and widen the widened input. 1338 InOp = GetWidenedVector(InOp); 1339 InVT = InOp.getValueType(); 1340 if (WidenVT.bitsEq(InVT)) 1341 // The input widens to the same size. Convert to the widen value. 1342 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1343 break; 1344 } 1345 1346 unsigned WidenSize = WidenVT.getSizeInBits(); 1347 unsigned InSize = InVT.getSizeInBits(); 1348 if (WidenSize % InSize == 0) { 1349 // Determine new input vector type. The new input vector type will use 1350 // the same element type (if its a vector) or use the input type as a 1351 // vector. It is the same size as the type to widen to. 1352 EVT NewInVT; 1353 unsigned NewNumElts = WidenSize / InSize; 1354 if (InVT.isVector()) { 1355 EVT InEltVT = InVT.getVectorElementType(); 1356 NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits()); 1357 } else { 1358 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1359 } 1360 1361 if (TLI.isTypeLegal(NewInVT)) { 1362 // Because the result and the input are different vector types, widening 1363 // the result could create a legal type but widening the input might make 1364 // it an illegal type that might lead to repeatedly splitting the input 1365 // and then widening it. To avoid this, we widen the input only if 1366 // it results in a legal type. 1367 SmallVector<SDValue, 16> Ops(NewNumElts); 1368 SDValue UndefVal = DAG.getUNDEF(InVT); 1369 Ops[0] = InOp; 1370 for (unsigned i = 1; i < NewNumElts; ++i) 1371 Ops[i] = UndefVal; 1372 1373 SDValue NewVec; 1374 if (InVT.isVector()) 1375 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1376 NewInVT, &Ops[0], NewNumElts); 1377 else 1378 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1379 NewInVT, &Ops[0], NewNumElts); 1380 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1381 } 1382 } 1383 1384 return CreateStackStoreLoad(InOp, WidenVT); 1385} 1386 1387SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1388 DebugLoc dl = N->getDebugLoc(); 1389 // Build a vector with undefined for the new nodes. 1390 EVT VT = N->getValueType(0); 1391 EVT EltVT = VT.getVectorElementType(); 1392 unsigned NumElts = VT.getVectorNumElements(); 1393 1394 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1395 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1396 1397 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1398 NewOps.reserve(WidenNumElts); 1399 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1400 NewOps.push_back(DAG.getUNDEF(EltVT)); 1401 1402 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1403} 1404 1405SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1406 EVT InVT = N->getOperand(0).getValueType(); 1407 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1408 DebugLoc dl = N->getDebugLoc(); 1409 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1410 unsigned NumOperands = N->getNumOperands(); 1411 1412 bool InputWidened = false; // Indicates we need to widen the input. 1413 if (getTypeAction(InVT) != WidenVector) { 1414 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1415 // Add undef vectors to widen to correct length. 1416 unsigned NumConcat = WidenVT.getVectorNumElements() / 1417 InVT.getVectorNumElements(); 1418 SDValue UndefVal = DAG.getUNDEF(InVT); 1419 SmallVector<SDValue, 16> Ops(NumConcat); 1420 for (unsigned i=0; i < NumOperands; ++i) 1421 Ops[i] = N->getOperand(i); 1422 for (unsigned i = NumOperands; i != NumConcat; ++i) 1423 Ops[i] = UndefVal; 1424 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1425 } 1426 } else { 1427 InputWidened = true; 1428 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1429 // The inputs and the result are widen to the same value. 1430 unsigned i; 1431 for (i=1; i < NumOperands; ++i) 1432 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1433 break; 1434 1435 if (i > NumOperands) 1436 // Everything but the first operand is an UNDEF so just return the 1437 // widened first operand. 1438 return GetWidenedVector(N->getOperand(0)); 1439 1440 if (NumOperands == 2) { 1441 // Replace concat of two operands with a shuffle. 1442 SmallVector<int, 16> MaskOps(WidenNumElts); 1443 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1444 MaskOps[i] = i; 1445 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1446 } 1447 return DAG.getVectorShuffle(WidenVT, dl, 1448 GetWidenedVector(N->getOperand(0)), 1449 GetWidenedVector(N->getOperand(1)), 1450 &MaskOps[0]); 1451 } 1452 } 1453 } 1454 1455 // Fall back to use extracts and build vector. 1456 EVT EltVT = WidenVT.getVectorElementType(); 1457 unsigned NumInElts = InVT.getVectorNumElements(); 1458 SmallVector<SDValue, 16> Ops(WidenNumElts); 1459 unsigned Idx = 0; 1460 for (unsigned i=0; i < NumOperands; ++i) { 1461 SDValue InOp = N->getOperand(i); 1462 if (InputWidened) 1463 InOp = GetWidenedVector(InOp); 1464 for (unsigned j=0; j < NumInElts; ++j) 1465 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1466 DAG.getIntPtrConstant(j)); 1467 } 1468 SDValue UndefVal = DAG.getUNDEF(EltVT); 1469 for (; Idx < WidenNumElts; ++Idx) 1470 Ops[Idx] = UndefVal; 1471 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1472} 1473 1474SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1475 DebugLoc dl = N->getDebugLoc(); 1476 SDValue InOp = N->getOperand(0); 1477 SDValue RndOp = N->getOperand(3); 1478 SDValue SatOp = N->getOperand(4); 1479 1480 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1481 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1482 1483 EVT InVT = InOp.getValueType(); 1484 EVT InEltVT = InVT.getVectorElementType(); 1485 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1486 1487 SDValue DTyOp = DAG.getValueType(WidenVT); 1488 SDValue STyOp = DAG.getValueType(InWidenVT); 1489 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1490 1491 unsigned InVTNumElts = InVT.getVectorNumElements(); 1492 if (getTypeAction(InVT) == WidenVector) { 1493 InOp = GetWidenedVector(InOp); 1494 InVT = InOp.getValueType(); 1495 InVTNumElts = InVT.getVectorNumElements(); 1496 if (InVTNumElts == WidenNumElts) 1497 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1498 SatOp, CvtCode); 1499 } 1500 1501 if (TLI.isTypeLegal(InWidenVT)) { 1502 // Because the result and the input are different vector types, widening 1503 // the result could create a legal type but widening the input might make 1504 // it an illegal type that might lead to repeatedly splitting the input 1505 // and then widening it. To avoid this, we widen the input only if 1506 // it results in a legal type. 1507 if (WidenNumElts % InVTNumElts == 0) { 1508 // Widen the input and call convert on the widened input vector. 1509 unsigned NumConcat = WidenNumElts/InVTNumElts; 1510 SmallVector<SDValue, 16> Ops(NumConcat); 1511 Ops[0] = InOp; 1512 SDValue UndefVal = DAG.getUNDEF(InVT); 1513 for (unsigned i = 1; i != NumConcat; ++i) { 1514 Ops[i] = UndefVal; 1515 } 1516 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1517 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1518 SatOp, CvtCode); 1519 } 1520 1521 if (InVTNumElts % WidenNumElts == 0) { 1522 // Extract the input and convert the shorten input vector. 1523 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1524 DAG.getIntPtrConstant(0)); 1525 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1526 SatOp, CvtCode); 1527 } 1528 } 1529 1530 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1531 SmallVector<SDValue, 16> Ops(WidenNumElts); 1532 EVT EltVT = WidenVT.getVectorElementType(); 1533 DTyOp = DAG.getValueType(EltVT); 1534 STyOp = DAG.getValueType(InEltVT); 1535 1536 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1537 unsigned i; 1538 for (i=0; i < MinElts; ++i) { 1539 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1540 DAG.getIntPtrConstant(i)); 1541 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1542 SatOp, CvtCode); 1543 } 1544 1545 SDValue UndefVal = DAG.getUNDEF(EltVT); 1546 for (; i < WidenNumElts; ++i) 1547 Ops[i] = UndefVal; 1548 1549 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1550} 1551 1552SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1553 EVT VT = N->getValueType(0); 1554 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1555 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1556 SDValue InOp = N->getOperand(0); 1557 SDValue Idx = N->getOperand(1); 1558 DebugLoc dl = N->getDebugLoc(); 1559 1560 if (getTypeAction(InOp.getValueType()) == WidenVector) 1561 InOp = GetWidenedVector(InOp); 1562 1563 EVT InVT = InOp.getValueType(); 1564 1565 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1566 if (CIdx) { 1567 unsigned IdxVal = CIdx->getZExtValue(); 1568 // Check if we can just return the input vector after widening. 1569 if (IdxVal == 0 && InVT == WidenVT) 1570 return InOp; 1571 1572 // Check if we can extract from the vector. 1573 unsigned InNumElts = InVT.getVectorNumElements(); 1574 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1575 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1576 } 1577 1578 // We could try widening the input to the right length but for now, extract 1579 // the original elements, fill the rest with undefs and build a vector. 1580 SmallVector<SDValue, 16> Ops(WidenNumElts); 1581 EVT EltVT = VT.getVectorElementType(); 1582 EVT IdxVT = Idx.getValueType(); 1583 unsigned NumElts = VT.getVectorNumElements(); 1584 unsigned i; 1585 if (CIdx) { 1586 unsigned IdxVal = CIdx->getZExtValue(); 1587 for (i=0; i < NumElts; ++i) 1588 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1589 DAG.getConstant(IdxVal+i, IdxVT)); 1590 } else { 1591 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1592 for (i=1; i < NumElts; ++i) { 1593 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1594 DAG.getConstant(i, IdxVT)); 1595 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1596 } 1597 } 1598 1599 SDValue UndefVal = DAG.getUNDEF(EltVT); 1600 for (; i < WidenNumElts; ++i) 1601 Ops[i] = UndefVal; 1602 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1603} 1604 1605SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1606 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1607 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1608 InOp.getValueType(), InOp, 1609 N->getOperand(1), N->getOperand(2)); 1610} 1611 1612SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1613 LoadSDNode *LD = cast<LoadSDNode>(N); 1614 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1615 EVT LdVT = LD->getMemoryVT(); 1616 DebugLoc dl = N->getDebugLoc(); 1617 assert(LdVT.isVector() && WidenVT.isVector()); 1618 1619 // Load information 1620 SDValue Chain = LD->getChain(); 1621 SDValue BasePtr = LD->getBasePtr(); 1622 int SVOffset = LD->getSrcValueOffset(); 1623 unsigned Align = LD->getAlignment(); 1624 bool isVolatile = LD->isVolatile(); 1625 const Value *SV = LD->getSrcValue(); 1626 ISD::LoadExtType ExtType = LD->getExtensionType(); 1627 1628 SDValue Result; 1629 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1630 if (ExtType != ISD::NON_EXTLOAD) { 1631 // For extension loads, we can not play the tricks of chopping legal 1632 // vector types and bit cast it to the right type. Instead, we unroll 1633 // the load and build a vector. 1634 EVT EltVT = WidenVT.getVectorElementType(); 1635 EVT LdEltVT = LdVT.getVectorElementType(); 1636 unsigned NumElts = LdVT.getVectorNumElements(); 1637 1638 // Load each element and widen 1639 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1640 SmallVector<SDValue, 16> Ops(WidenNumElts); 1641 unsigned Increment = LdEltVT.getSizeInBits() / 8; 1642 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 1643 LdEltVT, isVolatile, Align); 1644 LdChain.push_back(Ops[0].getValue(1)); 1645 unsigned i = 0, Offset = Increment; 1646 for (i=1; i < NumElts; ++i, Offset += Increment) { 1647 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1648 BasePtr, DAG.getIntPtrConstant(Offset)); 1649 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 1650 SVOffset + Offset, LdEltVT, isVolatile, Align); 1651 LdChain.push_back(Ops[i].getValue(1)); 1652 } 1653 1654 // Fill the rest with undefs 1655 SDValue UndefVal = DAG.getUNDEF(EltVT); 1656 for (; i != WidenNumElts; ++i) 1657 Ops[i] = UndefVal; 1658 1659 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 1660 } else { 1661 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 1662 unsigned int LdWidth = LdVT.getSizeInBits(); 1663 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 1664 Align, isVolatile, LdWidth, WidenVT, dl); 1665 } 1666 1667 // If we generate a single load, we can use that for the chain. Otherwise, 1668 // build a factor node to remember the multiple loads are independent and 1669 // chain to that. 1670 SDValue NewChain; 1671 if (LdChain.size() == 1) 1672 NewChain = LdChain[0]; 1673 else 1674 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 1675 LdChain.size()); 1676 1677 // Modified the chain - switch anything that used the old chain to use 1678 // the new one. 1679 ReplaceValueWith(SDValue(N, 1), NewChain); 1680 1681 return Result; 1682} 1683 1684SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1685 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1686 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1687 WidenVT, N->getOperand(0)); 1688} 1689 1690SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1691 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1692 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1693 1694 SDValue Cond1 = N->getOperand(0); 1695 EVT CondVT = Cond1.getValueType(); 1696 if (CondVT.isVector()) { 1697 EVT CondEltVT = CondVT.getVectorElementType(); 1698 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts); 1699 if (getTypeAction(CondVT) == WidenVector) 1700 Cond1 = GetWidenedVector(Cond1); 1701 1702 if (Cond1.getValueType() != CondWidenVT) 1703 Cond1 = ModifyToType(Cond1, CondWidenVT); 1704 } 1705 1706 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1707 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1708 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1709 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1710 WidenVT, Cond1, InOp1, InOp2); 1711} 1712 1713SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1714 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1715 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1716 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1717 InOp1.getValueType(), N->getOperand(0), 1718 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1719} 1720 1721SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1722 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1723 return DAG.getUNDEF(WidenVT); 1724} 1725 1726SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1727 EVT VT = N->getValueType(0); 1728 DebugLoc dl = N->getDebugLoc(); 1729 1730 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1731 unsigned NumElts = VT.getVectorNumElements(); 1732 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1733 1734 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1735 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1736 1737 // Adjust mask based on new input vector length. 1738 SmallVector<int, 16> NewMask; 1739 for (unsigned i = 0; i != NumElts; ++i) { 1740 int Idx = N->getMaskElt(i); 1741 if (Idx < (int)NumElts) 1742 NewMask.push_back(Idx); 1743 else 1744 NewMask.push_back(Idx - NumElts + WidenNumElts); 1745 } 1746 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1747 NewMask.push_back(-1); 1748 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1749} 1750 1751SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1752 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1753 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1754 1755 SDValue InOp1 = N->getOperand(0); 1756 EVT InVT = InOp1.getValueType(); 1757 assert(InVT.isVector() && "can not widen non vector type"); 1758 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); 1759 InOp1 = GetWidenedVector(InOp1); 1760 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1761 1762 // Assume that the input and output will be widen appropriately. If not, 1763 // we will have to unroll it at some point. 1764 assert(InOp1.getValueType() == WidenInVT && 1765 InOp2.getValueType() == WidenInVT && 1766 "Input not widened to expected type!"); 1767 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1768 WidenVT, InOp1, InOp2, N->getOperand(2)); 1769} 1770 1771 1772//===----------------------------------------------------------------------===// 1773// Widen Vector Operand 1774//===----------------------------------------------------------------------===// 1775bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1776 DEBUG(errs() << "Widen node operand " << ResNo << ": "; 1777 N->dump(&DAG); 1778 errs() << "\n"); 1779 SDValue Res = SDValue(); 1780 1781 switch (N->getOpcode()) { 1782 default: 1783#ifndef NDEBUG 1784 errs() << "WidenVectorOperand op #" << ResNo << ": "; 1785 N->dump(&DAG); 1786 errs() << "\n"; 1787#endif 1788 llvm_unreachable("Do not know how to widen this operator's operand!"); 1789 1790 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1791 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1792 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 1793 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1794 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1795 1796 case ISD::FP_ROUND: 1797 case ISD::FP_TO_SINT: 1798 case ISD::FP_TO_UINT: 1799 case ISD::SINT_TO_FP: 1800 case ISD::UINT_TO_FP: 1801 case ISD::TRUNCATE: 1802 case ISD::SIGN_EXTEND: 1803 case ISD::ZERO_EXTEND: 1804 case ISD::ANY_EXTEND: 1805 Res = WidenVecOp_Convert(N); 1806 break; 1807 } 1808 1809 // If Res is null, the sub-method took care of registering the result. 1810 if (!Res.getNode()) return false; 1811 1812 // If the result is N, the sub-method updated N in place. Tell the legalizer 1813 // core about this. 1814 if (Res.getNode() == N) 1815 return true; 1816 1817 1818 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1819 "Invalid operand expansion"); 1820 1821 ReplaceValueWith(SDValue(N, 0), Res); 1822 return false; 1823} 1824 1825SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1826 // Since the result is legal and the input is illegal, it is unlikely 1827 // that we can fix the input to a legal type so unroll the convert 1828 // into some scalar code and create a nasty build vector. 1829 EVT VT = N->getValueType(0); 1830 EVT EltVT = VT.getVectorElementType(); 1831 DebugLoc dl = N->getDebugLoc(); 1832 unsigned NumElts = VT.getVectorNumElements(); 1833 SDValue InOp = N->getOperand(0); 1834 if (getTypeAction(InOp.getValueType()) == WidenVector) 1835 InOp = GetWidenedVector(InOp); 1836 EVT InVT = InOp.getValueType(); 1837 EVT InEltVT = InVT.getVectorElementType(); 1838 1839 unsigned Opcode = N->getOpcode(); 1840 SmallVector<SDValue, 16> Ops(NumElts); 1841 for (unsigned i=0; i < NumElts; ++i) 1842 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1843 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1844 DAG.getIntPtrConstant(i))); 1845 1846 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1847} 1848 1849SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1850 EVT VT = N->getValueType(0); 1851 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1852 EVT InWidenVT = InOp.getValueType(); 1853 DebugLoc dl = N->getDebugLoc(); 1854 1855 // Check if we can convert between two legal vector types and extract. 1856 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1857 unsigned Size = VT.getSizeInBits(); 1858 if (InWidenSize % Size == 0 && !VT.isVector()) { 1859 unsigned NewNumElts = InWidenSize / Size; 1860 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 1861 if (TLI.isTypeLegal(NewVT)) { 1862 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1863 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1864 DAG.getIntPtrConstant(0)); 1865 } 1866 } 1867 1868 return CreateStackStoreLoad(InOp, VT); 1869} 1870 1871SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1872 // If the input vector is not legal, it is likely that we will not find a 1873 // legal vector of the same size. Replace the concatenate vector with a 1874 // nasty build vector. 1875 EVT VT = N->getValueType(0); 1876 EVT EltVT = VT.getVectorElementType(); 1877 DebugLoc dl = N->getDebugLoc(); 1878 unsigned NumElts = VT.getVectorNumElements(); 1879 SmallVector<SDValue, 16> Ops(NumElts); 1880 1881 EVT InVT = N->getOperand(0).getValueType(); 1882 unsigned NumInElts = InVT.getVectorNumElements(); 1883 1884 unsigned Idx = 0; 1885 unsigned NumOperands = N->getNumOperands(); 1886 for (unsigned i=0; i < NumOperands; ++i) { 1887 SDValue InOp = N->getOperand(i); 1888 if (getTypeAction(InOp.getValueType()) == WidenVector) 1889 InOp = GetWidenedVector(InOp); 1890 for (unsigned j=0; j < NumInElts; ++j) 1891 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1892 DAG.getIntPtrConstant(j)); 1893 } 1894 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1895} 1896 1897SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1898 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1899 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), 1900 N->getValueType(0), InOp, N->getOperand(1)); 1901} 1902 1903SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1904 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1905 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1906 N->getValueType(0), InOp, N->getOperand(1)); 1907} 1908 1909SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1910 // We have to widen the value but we want only to store the original 1911 // vector type. 1912 StoreSDNode *ST = cast<StoreSDNode>(N); 1913 SDValue Chain = ST->getChain(); 1914 SDValue BasePtr = ST->getBasePtr(); 1915 const Value *SV = ST->getSrcValue(); 1916 int SVOffset = ST->getSrcValueOffset(); 1917 unsigned Align = ST->getAlignment(); 1918 bool isVolatile = ST->isVolatile(); 1919 SDValue ValOp = GetWidenedVector(ST->getValue()); 1920 DebugLoc dl = N->getDebugLoc(); 1921 1922 EVT StVT = ST->getMemoryVT(); 1923 EVT ValVT = ValOp.getValueType(); 1924 // It must be true that we the widen vector type is bigger than where 1925 // we need to store. 1926 assert(StVT.isVector() && ValOp.getValueType().isVector()); 1927 assert(StVT.bitsLT(ValOp.getValueType())); 1928 1929 SmallVector<SDValue, 16> StChain; 1930 if (ST->isTruncatingStore()) { 1931 // For truncating stores, we can not play the tricks of chopping legal 1932 // vector types and bit cast it to the right type. Instead, we unroll 1933 // the store. 1934 EVT StEltVT = StVT.getVectorElementType(); 1935 EVT ValEltVT = ValVT.getVectorElementType(); 1936 unsigned Increment = ValEltVT.getSizeInBits() / 8; 1937 unsigned NumElts = StVT.getVectorNumElements(); 1938 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1939 DAG.getIntPtrConstant(0)); 1940 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 1941 SVOffset, StEltVT, 1942 isVolatile, Align)); 1943 unsigned Offset = Increment; 1944 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 1945 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1946 BasePtr, DAG.getIntPtrConstant(Offset)); 1947 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1948 DAG.getIntPtrConstant(0)); 1949 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 1950 SVOffset + Offset, StEltVT, 1951 isVolatile, MinAlign(Align, Offset))); 1952 } 1953 } 1954 else { 1955 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 1956 // Store value 1957 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 1958 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 1959 } 1960 if (StChain.size() == 1) 1961 return StChain[0]; 1962 else 1963 return DAG.getNode(ISD::TokenFactor, dl, 1964 MVT::Other,&StChain[0],StChain.size()); 1965} 1966 1967//===----------------------------------------------------------------------===// 1968// Vector Widening Utilities 1969//===----------------------------------------------------------------------===// 1970 1971 1972// Utility function to find a vector type and its associated element 1973// type from a preferred width and whose vector type must be the same size 1974// as the VecVT. 1975// TLI: Target lowering used to determine legal types. 1976// Width: Preferred width to store. 1977// VecVT: Vector value type whose size we must match. 1978// Returns NewVecVT and NewEltVT - the vector type and its associated 1979// element type. 1980static void FindAssocWidenVecType(SelectionDAG& DAG, 1981 const TargetLowering &TLI, unsigned Width, 1982 EVT VecVT, 1983 EVT& NewEltVT, EVT& NewVecVT) { 1984 unsigned EltWidth = Width + 1; 1985 if (TLI.isTypeLegal(VecVT)) { 1986 // We start with the preferred with, making it a power of 2 and find a 1987 // legal vector type of that width. If not, we reduce it by another of 2. 1988 // For incoming type is legal, this process will end as a vector of the 1989 // smallest loadable type should always be legal. 1990 do { 1991 assert(EltWidth > 0); 1992 EltWidth = 1 << Log2_32(EltWidth - 1); 1993 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 1994 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 1995 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 1996 } while (!TLI.isTypeLegal(NewVecVT) || 1997 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 1998 } else { 1999 // The incoming vector type is illegal and is the result of widening 2000 // a vector to a power of 2. In this case, we will use the preferred 2001 // with as long as it is a multiple of the incoming vector length. 2002 // The legalization process will eventually make this into a legal type 2003 // and remove the illegal bit converts (which would turn to stack converts 2004 // if they are allow to exist). 2005 do { 2006 assert(EltWidth > 0); 2007 EltWidth = 1 << Log2_32(EltWidth - 1); 2008 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 2009 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2010 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 2011 } while (!TLI.isTypeLegal(NewEltVT) || 2012 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2013 } 2014} 2015 2016SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2017 SDValue Chain, 2018 SDValue BasePtr, 2019 const Value *SV, 2020 int SVOffset, 2021 unsigned Alignment, 2022 bool isVolatile, 2023 unsigned LdWidth, 2024 EVT ResType, 2025 DebugLoc dl) { 2026 // The strategy assumes that we can efficiently load powers of two widths. 2027 // The routines chops the vector into the largest power of 2 load and 2028 // can be inserted into a legal vector and then cast the result into the 2029 // vector type we want. This avoids unnecessary stack converts. 2030 2031 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 2032 // the load is nonvolatile, we an use a wider load for the value. 2033 2034 // Find the vector type that can load from. 2035 EVT NewEltVT, NewVecVT; 2036 unsigned NewEltVTWidth; 2037 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2038 NewEltVTWidth = NewEltVT.getSizeInBits(); 2039 2040 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 2041 isVolatile, Alignment); 2042 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2043 LdChain.push_back(LdOp.getValue(1)); 2044 2045 // Check if we can load the element with one instruction 2046 if (LdWidth == NewEltVTWidth) { 2047 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2048 } 2049 2050 unsigned Idx = 1; 2051 LdWidth -= NewEltVTWidth; 2052 unsigned Offset = 0; 2053 2054 while (LdWidth > 0) { 2055 unsigned Increment = NewEltVTWidth / 8; 2056 Offset += Increment; 2057 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2058 DAG.getIntPtrConstant(Increment)); 2059 2060 if (LdWidth < NewEltVTWidth) { 2061 // Our current type we are using is too large, use a smaller size by 2062 // using a smaller power of 2 2063 unsigned oNewEltVTWidth = NewEltVTWidth; 2064 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2065 NewEltVTWidth = NewEltVT.getSizeInBits(); 2066 // Readjust position and vector position based on new load type 2067 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2068 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2069 } 2070 2071 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 2072 SVOffset+Offset, isVolatile, 2073 MinAlign(Alignment, Offset)); 2074 LdChain.push_back(LdOp.getValue(1)); 2075 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 2076 DAG.getIntPtrConstant(Idx++)); 2077 2078 LdWidth -= NewEltVTWidth; 2079 } 2080 2081 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2082} 2083 2084void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2085 SDValue Chain, 2086 SDValue BasePtr, 2087 const Value *SV, 2088 int SVOffset, 2089 unsigned Alignment, 2090 bool isVolatile, 2091 SDValue ValOp, 2092 unsigned StWidth, 2093 DebugLoc dl) { 2094 // Breaks the stores into a series of power of 2 width stores. For any 2095 // width, we convert the vector to the vector of element size that we 2096 // want to store. This avoids requiring a stack convert. 2097 2098 // Find a width of the element type we can store with 2099 EVT WidenVT = ValOp.getValueType(); 2100 EVT NewEltVT, NewVecVT; 2101 2102 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2103 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 2104 2105 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2106 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2107 DAG.getIntPtrConstant(0)); 2108 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 2109 isVolatile, Alignment); 2110 StChain.push_back(StOp); 2111 2112 // Check if we are done 2113 if (StWidth == NewEltVTWidth) { 2114 return; 2115 } 2116 2117 unsigned Idx = 1; 2118 StWidth -= NewEltVTWidth; 2119 unsigned Offset = 0; 2120 2121 while (StWidth > 0) { 2122 unsigned Increment = NewEltVTWidth / 8; 2123 Offset += Increment; 2124 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2125 DAG.getIntPtrConstant(Increment)); 2126 2127 if (StWidth < NewEltVTWidth) { 2128 // Our current type we are using is too large, use a smaller size by 2129 // using a smaller power of 2 2130 unsigned oNewEltVTWidth = NewEltVTWidth; 2131 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2132 NewEltVTWidth = NewEltVT.getSizeInBits(); 2133 // Readjust position and vector position based on new load type 2134 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2135 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2136 } 2137 2138 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2139 DAG.getIntPtrConstant(Idx++)); 2140 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2141 SVOffset + Offset, isVolatile, 2142 MinAlign(Alignment, Offset))); 2143 StWidth -= NewEltVTWidth; 2144 } 2145} 2146 2147/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2148/// input vector must have the same element type as NVT. 2149SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2150 // Note that InOp might have been widened so it might already have 2151 // the right width or it might need be narrowed. 2152 EVT InVT = InOp.getValueType(); 2153 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2154 "input and widen element type must match"); 2155 DebugLoc dl = InOp.getDebugLoc(); 2156 2157 // Check if InOp already has the right width. 2158 if (InVT == NVT) 2159 return InOp; 2160 2161 unsigned InNumElts = InVT.getVectorNumElements(); 2162 unsigned WidenNumElts = NVT.getVectorNumElements(); 2163 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2164 unsigned NumConcat = WidenNumElts / InNumElts; 2165 SmallVector<SDValue, 16> Ops(NumConcat); 2166 SDValue UndefVal = DAG.getUNDEF(InVT); 2167 Ops[0] = InOp; 2168 for (unsigned i = 1; i != NumConcat; ++i) 2169 Ops[i] = UndefVal; 2170 2171 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2172 } 2173 2174 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2175 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2176 DAG.getIntPtrConstant(0)); 2177 2178 // Fall back to extract and build. 2179 SmallVector<SDValue, 16> Ops(WidenNumElts); 2180 EVT EltVT = NVT.getVectorElementType(); 2181 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2182 unsigned Idx; 2183 for (Idx = 0; Idx < MinNumElts; ++Idx) 2184 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2185 DAG.getIntPtrConstant(Idx)); 2186 2187 SDValue UndefVal = DAG.getUNDEF(EltVT); 2188 for ( ; Idx < WidenNumElts; ++Idx) 2189 Ops[Idx] = UndefVal; 2190 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2191} 2192