LegalizeVectorTypes.cpp revision a8ff0dddb04db54c7c54d3f84babe100e577fc49
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in two vectors of half the size. For example, implementing 19// <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26#include "llvm/Support/ErrorHandling.h" 27#include "llvm/Support/raw_ostream.h" 28using namespace llvm; 29 30//===----------------------------------------------------------------------===// 31// Result Vector Scalarization: <1 x ty> -> ty. 32//===----------------------------------------------------------------------===// 33 34void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 35 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 36 N->dump(&DAG); 37 dbgs() << "\n"); 38 SDValue R = SDValue(); 39 40 switch (N->getOpcode()) { 41 default: 42#ifndef NDEBUG 43 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 44 N->dump(&DAG); 45 dbgs() << "\n"; 46#endif 47 llvm_unreachable("Do not know how to scalarize the result of this operator!"); 48 49 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 50 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 51 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 52 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 53 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 54 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 55 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 56 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 57 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 58 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 59 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 60 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 61 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 62 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 63 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 64 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 65 66 case ISD::CTLZ: 67 case ISD::CTPOP: 68 case ISD::CTTZ: 69 case ISD::FABS: 70 case ISD::FCOS: 71 case ISD::FNEG: 72 case ISD::FP_TO_SINT: 73 case ISD::FP_TO_UINT: 74 case ISD::FSIN: 75 case ISD::FSQRT: 76 case ISD::FTRUNC: 77 case ISD::FFLOOR: 78 case ISD::FCEIL: 79 case ISD::FRINT: 80 case ISD::FNEARBYINT: 81 case ISD::UINT_TO_FP: 82 case ISD::SINT_TO_FP: 83 case ISD::TRUNCATE: 84 case ISD::SIGN_EXTEND: 85 case ISD::ZERO_EXTEND: 86 case ISD::ANY_EXTEND: 87 R = ScalarizeVecRes_UnaryOp(N); 88 break; 89 90 case ISD::ADD: 91 case ISD::AND: 92 case ISD::FADD: 93 case ISD::FDIV: 94 case ISD::FMUL: 95 case ISD::FPOW: 96 case ISD::FREM: 97 case ISD::FSUB: 98 case ISD::MUL: 99 case ISD::OR: 100 case ISD::SDIV: 101 case ISD::SREM: 102 case ISD::SUB: 103 case ISD::UDIV: 104 case ISD::UREM: 105 case ISD::XOR: 106 case ISD::SHL: 107 case ISD::SRA: 108 case ISD::SRL: 109 R = ScalarizeVecRes_BinOp(N); 110 break; 111 } 112 113 // If R is null, the sub-method took care of registering the result. 114 if (R.getNode()) 115 SetScalarizedVector(SDValue(N, ResNo), R); 116} 117 118SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 119 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 120 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 121 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 122 LHS.getValueType(), LHS, RHS); 123} 124 125SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 126 EVT NewVT = N->getValueType(0).getVectorElementType(); 127 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 128 NewVT, N->getOperand(0)); 129} 130 131SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 132 EVT NewVT = N->getValueType(0).getVectorElementType(); 133 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 134 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 135 Op0, DAG.getValueType(NewVT), 136 DAG.getValueType(Op0.getValueType()), 137 N->getOperand(3), 138 N->getOperand(4), 139 cast<CvtRndSatSDNode>(N)->getCvtCode()); 140} 141 142SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 143 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 144 N->getValueType(0).getVectorElementType(), 145 N->getOperand(0), N->getOperand(1)); 146} 147 148SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 149 SDValue Op = GetScalarizedVector(N->getOperand(0)); 150 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 151 Op.getValueType(), Op, N->getOperand(1)); 152} 153 154SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 155 // The value to insert may have a wider type than the vector element type, 156 // so be sure to truncate it to the element type if necessary. 157 SDValue Op = N->getOperand(1); 158 EVT EltVT = N->getValueType(0).getVectorElementType(); 159 if (Op.getValueType() != EltVT) 160 // FIXME: Can this happen for floating point types? 161 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 162 return Op; 163} 164 165SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 166 assert(N->isUnindexed() && "Indexed vector load?"); 167 168 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 169 N->getExtensionType(), 170 N->getValueType(0).getVectorElementType(), 171 N->getChain(), N->getBasePtr(), 172 DAG.getUNDEF(N->getBasePtr().getValueType()), 173 N->getSrcValue(), N->getSrcValueOffset(), 174 N->getMemoryVT().getVectorElementType(), 175 N->isVolatile(), N->getOriginalAlignment()); 176 177 // Legalized the chain result - switch anything that used the old chain to 178 // use the new one. 179 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 180 return Result; 181} 182 183SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 184 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 185 EVT DestVT = N->getValueType(0).getVectorElementType(); 186 SDValue Op = GetScalarizedVector(N->getOperand(0)); 187 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 188} 189 190SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 191 EVT EltVT = N->getValueType(0).getVectorElementType(); 192 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 193 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 194 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), EltVT, 195 LHS, DAG.getValueType(ExtVT)); 196} 197 198SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 199 // If the operand is wider than the vector element type then it is implicitly 200 // truncated. Make that explicit here. 201 EVT EltVT = N->getValueType(0).getVectorElementType(); 202 SDValue InOp = N->getOperand(0); 203 if (InOp.getValueType() != EltVT) 204 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 205 return InOp; 206} 207 208SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 209 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 210 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 211 LHS.getValueType(), N->getOperand(0), LHS, 212 GetScalarizedVector(N->getOperand(2))); 213} 214 215SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 216 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 217 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 218 N->getOperand(0), N->getOperand(1), 219 LHS, GetScalarizedVector(N->getOperand(3)), 220 N->getOperand(4)); 221} 222 223SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 224 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 225 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 226 DebugLoc DL = N->getDebugLoc(); 227 228 // Turn it into a scalar SETCC. 229 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 230} 231 232SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 233 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 234} 235 236SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 237 // Figure out if the scalar is the LHS or RHS and return it. 238 SDValue Arg = N->getOperand(2).getOperand(0); 239 if (Arg.getOpcode() == ISD::UNDEF) 240 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 241 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 242 return GetScalarizedVector(N->getOperand(Op)); 243} 244 245SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 246 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 247 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 248 EVT NVT = N->getValueType(0).getVectorElementType(); 249 EVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 250 DebugLoc DL = N->getDebugLoc(); 251 252 // Turn it into a scalar SETCC. 253 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 254 255 // VSETCC always returns a sign-extended value, while SETCC may not. The 256 // SETCC result type may not match the vector element type. Correct these. 257 if (NVT.bitsLE(SVT)) { 258 // The SETCC result type is bigger than the vector element type. 259 // Ensure the SETCC result is sign-extended. 260 if (TLI.getBooleanContents() != 261 TargetLowering::ZeroOrNegativeOneBooleanContent) 262 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 263 DAG.getValueType(MVT::i1)); 264 // Truncate to the final type. 265 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 266 } 267 268 // The SETCC result type is smaller than the vector element type. 269 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 270 if (TLI.getBooleanContents() != 271 TargetLowering::ZeroOrNegativeOneBooleanContent) 272 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 273 // Sign extend to the final type. 274 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 275} 276 277 278//===----------------------------------------------------------------------===// 279// Operand Vector Scalarization <1 x ty> -> ty. 280//===----------------------------------------------------------------------===// 281 282bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 283 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 284 N->dump(&DAG); 285 dbgs() << "\n"); 286 SDValue Res = SDValue(); 287 288 if (Res.getNode() == 0) { 289 switch (N->getOpcode()) { 290 default: 291#ifndef NDEBUG 292 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 293 N->dump(&DAG); 294 dbgs() << "\n"; 295#endif 296 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 297 case ISD::BIT_CONVERT: 298 Res = ScalarizeVecOp_BIT_CONVERT(N); 299 break; 300 case ISD::CONCAT_VECTORS: 301 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 302 break; 303 case ISD::EXTRACT_VECTOR_ELT: 304 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 305 break; 306 case ISD::STORE: 307 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 308 break; 309 } 310 } 311 312 // If the result is null, the sub-method took care of registering results etc. 313 if (!Res.getNode()) return false; 314 315 // If the result is N, the sub-method updated N in place. Tell the legalizer 316 // core about this. 317 if (Res.getNode() == N) 318 return true; 319 320 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 321 "Invalid operand expansion"); 322 323 ReplaceValueWith(SDValue(N, 0), Res); 324 return false; 325} 326 327/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 328/// to be scalarized, it must be <1 x ty>. Convert the element instead. 329SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 330 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 331 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 332 N->getValueType(0), Elt); 333} 334 335/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 336/// use a BUILD_VECTOR instead. 337SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 338 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 339 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 340 Ops[i] = GetScalarizedVector(N->getOperand(i)); 341 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 342 &Ops[0], Ops.size()); 343} 344 345/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 346/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 347/// index. 348SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 349 SDValue Res = GetScalarizedVector(N->getOperand(0)); 350 if (Res.getValueType() != N->getValueType(0)) 351 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 352 Res); 353 return Res; 354} 355 356/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 357/// scalarized, it must be <1 x ty>. Just store the element. 358SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 359 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 360 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 361 DebugLoc dl = N->getDebugLoc(); 362 363 if (N->isTruncatingStore()) 364 return DAG.getTruncStore(N->getChain(), dl, 365 GetScalarizedVector(N->getOperand(1)), 366 N->getBasePtr(), 367 N->getSrcValue(), N->getSrcValueOffset(), 368 N->getMemoryVT().getVectorElementType(), 369 N->isVolatile(), N->getAlignment()); 370 371 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 372 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 373 N->isVolatile(), N->getOriginalAlignment()); 374} 375 376 377//===----------------------------------------------------------------------===// 378// Result Vector Splitting 379//===----------------------------------------------------------------------===// 380 381/// SplitVectorResult - This method is called when the specified result of the 382/// specified node is found to need vector splitting. At this point, the node 383/// may also have invalid operands or may have other results that need 384/// legalization, we just know that (at least) one result needs vector 385/// splitting. 386void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 387 DEBUG(dbgs() << "Split node result: "; 388 N->dump(&DAG); 389 dbgs() << "\n"); 390 SDValue Lo, Hi; 391 392 switch (N->getOpcode()) { 393 default: 394#ifndef NDEBUG 395 dbgs() << "SplitVectorResult #" << ResNo << ": "; 396 N->dump(&DAG); 397 dbgs() << "\n"; 398#endif 399 llvm_unreachable("Do not know how to split the result of this operator!"); 400 401 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 402 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 403 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 404 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 405 406 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 407 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 408 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 409 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 410 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 411 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 412 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 413 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 414 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 415 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 416 case ISD::LOAD: 417 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 418 break; 419 case ISD::SETCC: 420 case ISD::VSETCC: 421 SplitVecRes_SETCC(N, Lo, Hi); 422 break; 423 case ISD::VECTOR_SHUFFLE: 424 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 425 break; 426 427 case ISD::CTTZ: 428 case ISD::CTLZ: 429 case ISD::CTPOP: 430 case ISD::FNEG: 431 case ISD::FABS: 432 case ISD::FSQRT: 433 case ISD::FSIN: 434 case ISD::FCOS: 435 case ISD::FTRUNC: 436 case ISD::FFLOOR: 437 case ISD::FCEIL: 438 case ISD::FRINT: 439 case ISD::FNEARBYINT: 440 case ISD::FP_TO_SINT: 441 case ISD::FP_TO_UINT: 442 case ISD::SINT_TO_FP: 443 case ISD::UINT_TO_FP: 444 case ISD::TRUNCATE: 445 case ISD::SIGN_EXTEND: 446 case ISD::ZERO_EXTEND: 447 case ISD::ANY_EXTEND: 448 SplitVecRes_UnaryOp(N, Lo, Hi); 449 break; 450 451 case ISD::ADD: 452 case ISD::SUB: 453 case ISD::MUL: 454 case ISD::FADD: 455 case ISD::FSUB: 456 case ISD::FMUL: 457 case ISD::SDIV: 458 case ISD::UDIV: 459 case ISD::FDIV: 460 case ISD::FPOW: 461 case ISD::AND: 462 case ISD::OR: 463 case ISD::XOR: 464 case ISD::SHL: 465 case ISD::SRA: 466 case ISD::SRL: 467 case ISD::UREM: 468 case ISD::SREM: 469 case ISD::FREM: 470 SplitVecRes_BinOp(N, Lo, Hi); 471 break; 472 } 473 474 // If Lo/Hi is null, the sub-method took care of registering results etc. 475 if (Lo.getNode()) 476 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 477} 478 479void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 480 SDValue &Hi) { 481 SDValue LHSLo, LHSHi; 482 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 483 SDValue RHSLo, RHSHi; 484 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 485 DebugLoc dl = N->getDebugLoc(); 486 487 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 488 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 489} 490 491void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 492 SDValue &Hi) { 493 // We know the result is a vector. The input may be either a vector or a 494 // scalar value. 495 EVT LoVT, HiVT; 496 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 497 DebugLoc dl = N->getDebugLoc(); 498 499 SDValue InOp = N->getOperand(0); 500 EVT InVT = InOp.getValueType(); 501 502 // Handle some special cases efficiently. 503 switch (getTypeAction(InVT)) { 504 default: 505 assert(false && "Unknown type action!"); 506 case Legal: 507 case PromoteInteger: 508 case SoftenFloat: 509 case ScalarizeVector: 510 break; 511 case ExpandInteger: 512 case ExpandFloat: 513 // A scalar to vector conversion, where the scalar needs expansion. 514 // If the vector is being split in two then we can just convert the 515 // expanded pieces. 516 if (LoVT == HiVT) { 517 GetExpandedOp(InOp, Lo, Hi); 518 if (TLI.isBigEndian()) 519 std::swap(Lo, Hi); 520 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 521 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 522 return; 523 } 524 break; 525 case SplitVector: 526 // If the input is a vector that needs to be split, convert each split 527 // piece of the input now. 528 GetSplitVector(InOp, Lo, Hi); 529 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 530 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 531 return; 532 } 533 534 // In the general case, convert the input to an integer and split it by hand. 535 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 536 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 537 if (TLI.isBigEndian()) 538 std::swap(LoIntVT, HiIntVT); 539 540 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 541 542 if (TLI.isBigEndian()) 543 std::swap(Lo, Hi); 544 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 545 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 546} 547 548void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 549 SDValue &Hi) { 550 EVT LoVT, HiVT; 551 DebugLoc dl = N->getDebugLoc(); 552 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 553 unsigned LoNumElts = LoVT.getVectorNumElements(); 554 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 555 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 556 557 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 558 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 559} 560 561void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 562 SDValue &Hi) { 563 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 564 DebugLoc dl = N->getDebugLoc(); 565 unsigned NumSubvectors = N->getNumOperands() / 2; 566 if (NumSubvectors == 1) { 567 Lo = N->getOperand(0); 568 Hi = N->getOperand(1); 569 return; 570 } 571 572 EVT LoVT, HiVT; 573 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 574 575 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 576 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 577 578 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 579 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 580} 581 582void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 583 SDValue &Hi) { 584 EVT LoVT, HiVT; 585 DebugLoc dl = N->getDebugLoc(); 586 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 587 588 SDValue DTyOpLo = DAG.getValueType(LoVT); 589 SDValue DTyOpHi = DAG.getValueType(HiVT); 590 591 SDValue RndOp = N->getOperand(3); 592 SDValue SatOp = N->getOperand(4); 593 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 594 595 // Split the input. 596 SDValue VLo, VHi; 597 EVT InVT = N->getOperand(0).getValueType(); 598 switch (getTypeAction(InVT)) { 599 default: llvm_unreachable("Unexpected type action!"); 600 case Legal: { 601 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 602 LoVT.getVectorNumElements()); 603 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 604 DAG.getIntPtrConstant(0)); 605 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 606 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 607 break; 608 } 609 case SplitVector: 610 GetSplitVector(N->getOperand(0), VLo, VHi); 611 break; 612 case WidenVector: { 613 // If the result needs to be split and the input needs to be widened, 614 // the two types must have different lengths. Use the widened result 615 // and extract from it to do the split. 616 SDValue InOp = GetWidenedVector(N->getOperand(0)); 617 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 618 LoVT.getVectorNumElements()); 619 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 620 DAG.getIntPtrConstant(0)); 621 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 622 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 623 break; 624 } 625 } 626 627 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 628 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 629 630 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 631 CvtCode); 632 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 633 CvtCode); 634} 635 636void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 637 SDValue &Hi) { 638 SDValue Vec = N->getOperand(0); 639 SDValue Idx = N->getOperand(1); 640 EVT IdxVT = Idx.getValueType(); 641 DebugLoc dl = N->getDebugLoc(); 642 643 EVT LoVT, HiVT; 644 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 645 646 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 647 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 648 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 649 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 650} 651 652void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 653 SDValue &Hi) { 654 DebugLoc dl = N->getDebugLoc(); 655 GetSplitVector(N->getOperand(0), Lo, Hi); 656 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 657 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 658} 659 660void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 661 SDValue &Hi) { 662 SDValue LHSLo, LHSHi; 663 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 664 DebugLoc dl = N->getDebugLoc(); 665 666 EVT LoVT, HiVT; 667 GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT(), LoVT, HiVT); 668 669 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 670 DAG.getValueType(LoVT)); 671 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 672 DAG.getValueType(HiVT)); 673} 674 675void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 676 SDValue &Hi) { 677 SDValue Vec = N->getOperand(0); 678 SDValue Elt = N->getOperand(1); 679 SDValue Idx = N->getOperand(2); 680 DebugLoc dl = N->getDebugLoc(); 681 GetSplitVector(Vec, Lo, Hi); 682 683 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 684 unsigned IdxVal = CIdx->getZExtValue(); 685 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 686 if (IdxVal < LoNumElts) 687 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 688 Lo.getValueType(), Lo, Elt, Idx); 689 else 690 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 691 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 692 return; 693 } 694 695 // Spill the vector to the stack. 696 EVT VecVT = Vec.getValueType(); 697 EVT EltVT = VecVT.getVectorElementType(); 698 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 699 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 700 701 // Store the new element. This may be larger than the vector element type, 702 // so use a truncating store. 703 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 704 unsigned Alignment = 705 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext())); 706 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 707 708 // Load the Lo part from the stack slot. 709 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 710 711 // Increment the pointer to the other part. 712 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 713 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 714 DAG.getIntPtrConstant(IncrementSize)); 715 716 // Load the Hi part from the stack slot. 717 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 718 MinAlign(Alignment, IncrementSize)); 719} 720 721void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 722 SDValue &Hi) { 723 EVT LoVT, HiVT; 724 DebugLoc dl = N->getDebugLoc(); 725 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 726 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 727 Hi = DAG.getUNDEF(HiVT); 728} 729 730void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 731 SDValue &Hi) { 732 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 733 EVT LoVT, HiVT; 734 DebugLoc dl = LD->getDebugLoc(); 735 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 736 737 ISD::LoadExtType ExtType = LD->getExtensionType(); 738 SDValue Ch = LD->getChain(); 739 SDValue Ptr = LD->getBasePtr(); 740 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 741 const Value *SV = LD->getSrcValue(); 742 int SVOffset = LD->getSrcValueOffset(); 743 EVT MemoryVT = LD->getMemoryVT(); 744 unsigned Alignment = LD->getOriginalAlignment(); 745 bool isVolatile = LD->isVolatile(); 746 747 EVT LoMemVT, HiMemVT; 748 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 749 750 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 751 SV, SVOffset, LoMemVT, isVolatile, Alignment); 752 753 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 754 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 755 DAG.getIntPtrConstant(IncrementSize)); 756 SVOffset += IncrementSize; 757 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 758 SV, SVOffset, HiMemVT, isVolatile, Alignment); 759 760 // Build a factor node to remember that this load is independent of the 761 // other one. 762 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 763 Hi.getValue(1)); 764 765 // Legalized the chain result - switch anything that used the old chain to 766 // use the new one. 767 ReplaceValueWith(SDValue(LD, 1), Ch); 768} 769 770void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 771 EVT LoVT, HiVT; 772 DebugLoc DL = N->getDebugLoc(); 773 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 774 775 // Split the input. 776 EVT InVT = N->getOperand(0).getValueType(); 777 SDValue LL, LH, RL, RH; 778 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 779 LoVT.getVectorNumElements()); 780 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 781 DAG.getIntPtrConstant(0)); 782 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 783 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 784 785 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 786 DAG.getIntPtrConstant(0)); 787 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 788 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 789 790 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 791 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 792} 793 794void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 795 SDValue &Hi) { 796 // Get the dest types - they may not match the input types, e.g. int_to_fp. 797 EVT LoVT, HiVT; 798 DebugLoc dl = N->getDebugLoc(); 799 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 800 801 // Split the input. 802 EVT InVT = N->getOperand(0).getValueType(); 803 switch (getTypeAction(InVT)) { 804 default: llvm_unreachable("Unexpected type action!"); 805 case Legal: { 806 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 807 LoVT.getVectorNumElements()); 808 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 809 DAG.getIntPtrConstant(0)); 810 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 811 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 812 break; 813 } 814 case SplitVector: 815 GetSplitVector(N->getOperand(0), Lo, Hi); 816 break; 817 case WidenVector: { 818 // If the result needs to be split and the input needs to be widened, 819 // the two types must have different lengths. Use the widened result 820 // and extract from it to do the split. 821 SDValue InOp = GetWidenedVector(N->getOperand(0)); 822 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 823 LoVT.getVectorNumElements()); 824 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 825 DAG.getIntPtrConstant(0)); 826 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 827 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 828 break; 829 } 830 } 831 832 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 833 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 834} 835 836void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 837 SDValue &Lo, SDValue &Hi) { 838 // The low and high parts of the original input give four input vectors. 839 SDValue Inputs[4]; 840 DebugLoc dl = N->getDebugLoc(); 841 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 842 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 843 EVT NewVT = Inputs[0].getValueType(); 844 unsigned NewElts = NewVT.getVectorNumElements(); 845 846 // If Lo or Hi uses elements from at most two of the four input vectors, then 847 // express it as a vector shuffle of those two inputs. Otherwise extract the 848 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 849 SmallVector<int, 16> Ops; 850 for (unsigned High = 0; High < 2; ++High) { 851 SDValue &Output = High ? Hi : Lo; 852 853 // Build a shuffle mask for the output, discovering on the fly which 854 // input vectors to use as shuffle operands (recorded in InputUsed). 855 // If building a suitable shuffle vector proves too hard, then bail 856 // out with useBuildVector set. 857 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 858 unsigned FirstMaskIdx = High * NewElts; 859 bool useBuildVector = false; 860 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 861 // The mask element. This indexes into the input. 862 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 863 864 // The input vector this mask element indexes into. 865 unsigned Input = (unsigned)Idx / NewElts; 866 867 if (Input >= array_lengthof(Inputs)) { 868 // The mask element does not index into any input vector. 869 Ops.push_back(-1); 870 continue; 871 } 872 873 // Turn the index into an offset from the start of the input vector. 874 Idx -= Input * NewElts; 875 876 // Find or create a shuffle vector operand to hold this input. 877 unsigned OpNo; 878 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 879 if (InputUsed[OpNo] == Input) { 880 // This input vector is already an operand. 881 break; 882 } else if (InputUsed[OpNo] == -1U) { 883 // Create a new operand for this input vector. 884 InputUsed[OpNo] = Input; 885 break; 886 } 887 } 888 889 if (OpNo >= array_lengthof(InputUsed)) { 890 // More than two input vectors used! Give up on trying to create a 891 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 892 useBuildVector = true; 893 break; 894 } 895 896 // Add the mask index for the new shuffle vector. 897 Ops.push_back(Idx + OpNo * NewElts); 898 } 899 900 if (useBuildVector) { 901 EVT EltVT = NewVT.getVectorElementType(); 902 SmallVector<SDValue, 16> SVOps; 903 904 // Extract the input elements by hand. 905 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 906 // The mask element. This indexes into the input. 907 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 908 909 // The input vector this mask element indexes into. 910 unsigned Input = (unsigned)Idx / NewElts; 911 912 if (Input >= array_lengthof(Inputs)) { 913 // The mask element is "undef" or indexes off the end of the input. 914 SVOps.push_back(DAG.getUNDEF(EltVT)); 915 continue; 916 } 917 918 // Turn the index into an offset from the start of the input vector. 919 Idx -= Input * NewElts; 920 921 // Extract the vector element by hand. 922 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 923 Inputs[Input], DAG.getIntPtrConstant(Idx))); 924 } 925 926 // Construct the Lo/Hi output using a BUILD_VECTOR. 927 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 928 } else if (InputUsed[0] == -1U) { 929 // No input vectors were used! The result is undefined. 930 Output = DAG.getUNDEF(NewVT); 931 } else { 932 SDValue Op0 = Inputs[InputUsed[0]]; 933 // If only one input was used, use an undefined vector for the other. 934 SDValue Op1 = InputUsed[1] == -1U ? 935 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 936 // At least one input vector was used. Create a new shuffle vector. 937 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 938 } 939 940 Ops.clear(); 941 } 942} 943 944 945//===----------------------------------------------------------------------===// 946// Operand Vector Splitting 947//===----------------------------------------------------------------------===// 948 949/// SplitVectorOperand - This method is called when the specified operand of the 950/// specified node is found to need vector splitting. At this point, all of the 951/// result types of the node are known to be legal, but other operands of the 952/// node may need legalization as well as the specified one. 953bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 954 DEBUG(dbgs() << "Split node operand: "; 955 N->dump(&DAG); 956 dbgs() << "\n"); 957 SDValue Res = SDValue(); 958 959 if (Res.getNode() == 0) { 960 switch (N->getOpcode()) { 961 default: 962#ifndef NDEBUG 963 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 964 N->dump(&DAG); 965 dbgs() << "\n"; 966#endif 967 llvm_unreachable("Do not know how to split this operator's operand!"); 968 969 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 970 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 971 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 972 case ISD::STORE: 973 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 974 break; 975 976 case ISD::CTTZ: 977 case ISD::CTLZ: 978 case ISD::CTPOP: 979 case ISD::FP_TO_SINT: 980 case ISD::FP_TO_UINT: 981 case ISD::SINT_TO_FP: 982 case ISD::UINT_TO_FP: 983 case ISD::TRUNCATE: 984 case ISD::SIGN_EXTEND: 985 case ISD::ZERO_EXTEND: 986 case ISD::ANY_EXTEND: 987 Res = SplitVecOp_UnaryOp(N); 988 break; 989 } 990 } 991 992 // If the result is null, the sub-method took care of registering results etc. 993 if (!Res.getNode()) return false; 994 995 // If the result is N, the sub-method updated N in place. Tell the legalizer 996 // core about this. 997 if (Res.getNode() == N) 998 return true; 999 1000 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1001 "Invalid operand expansion"); 1002 1003 ReplaceValueWith(SDValue(N, 0), Res); 1004 return false; 1005} 1006 1007SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1008 // The result has a legal vector type, but the input needs splitting. 1009 EVT ResVT = N->getValueType(0); 1010 SDValue Lo, Hi; 1011 DebugLoc dl = N->getDebugLoc(); 1012 GetSplitVector(N->getOperand(0), Lo, Hi); 1013 EVT InVT = Lo.getValueType(); 1014 1015 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1016 InVT.getVectorNumElements()); 1017 1018 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1019 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1020 1021 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1022} 1023 1024SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 1025 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 1026 // end up being split all the way down to individual components. Convert the 1027 // split pieces into integers and reassemble. 1028 SDValue Lo, Hi; 1029 GetSplitVector(N->getOperand(0), Lo, Hi); 1030 Lo = BitConvertToInteger(Lo); 1031 Hi = BitConvertToInteger(Hi); 1032 1033 if (TLI.isBigEndian()) 1034 std::swap(Lo, Hi); 1035 1036 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 1037 JoinIntegers(Lo, Hi)); 1038} 1039 1040SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1041 // We know that the extracted result type is legal. For now, assume the index 1042 // is a constant. 1043 EVT SubVT = N->getValueType(0); 1044 SDValue Idx = N->getOperand(1); 1045 DebugLoc dl = N->getDebugLoc(); 1046 SDValue Lo, Hi; 1047 GetSplitVector(N->getOperand(0), Lo, Hi); 1048 1049 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1050 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1051 1052 if (IdxVal < LoElts) { 1053 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1054 "Extracted subvector crosses vector split!"); 1055 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1056 } else { 1057 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1058 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1059 } 1060} 1061 1062SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1063 SDValue Vec = N->getOperand(0); 1064 SDValue Idx = N->getOperand(1); 1065 EVT VecVT = Vec.getValueType(); 1066 1067 if (isa<ConstantSDNode>(Idx)) { 1068 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1069 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1070 1071 SDValue Lo, Hi; 1072 GetSplitVector(Vec, Lo, Hi); 1073 1074 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1075 1076 if (IdxVal < LoElts) 1077 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1078 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1079 DAG.getConstant(IdxVal - LoElts, 1080 Idx.getValueType())); 1081 } 1082 1083 // Store the vector to the stack. 1084 EVT EltVT = VecVT.getVectorElementType(); 1085 DebugLoc dl = N->getDebugLoc(); 1086 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1087 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1088 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1089 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1090 1091 // Load back the required element. 1092 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1093 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1094 SV, 0, EltVT); 1095} 1096 1097SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1098 assert(N->isUnindexed() && "Indexed store of vector?"); 1099 assert(OpNo == 1 && "Can only split the stored value"); 1100 DebugLoc dl = N->getDebugLoc(); 1101 1102 bool isTruncating = N->isTruncatingStore(); 1103 SDValue Ch = N->getChain(); 1104 SDValue Ptr = N->getBasePtr(); 1105 int SVOffset = N->getSrcValueOffset(); 1106 EVT MemoryVT = N->getMemoryVT(); 1107 unsigned Alignment = N->getOriginalAlignment(); 1108 bool isVol = N->isVolatile(); 1109 SDValue Lo, Hi; 1110 GetSplitVector(N->getOperand(1), Lo, Hi); 1111 1112 EVT LoMemVT, HiMemVT; 1113 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1114 1115 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1116 1117 if (isTruncating) 1118 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1119 LoMemVT, isVol, Alignment); 1120 else 1121 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1122 isVol, Alignment); 1123 1124 // Increment the pointer to the other half. 1125 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1126 DAG.getIntPtrConstant(IncrementSize)); 1127 SVOffset += IncrementSize; 1128 1129 if (isTruncating) 1130 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1131 HiMemVT, isVol, Alignment); 1132 else 1133 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1134 isVol, Alignment); 1135 1136 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1137} 1138 1139 1140//===----------------------------------------------------------------------===// 1141// Result Vector Widening 1142//===----------------------------------------------------------------------===// 1143 1144void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1145 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1146 N->dump(&DAG); 1147 dbgs() << "\n"); 1148 1149 // See if the target wants to custom widen this node. 1150 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1151 return; 1152 1153 SDValue Res = SDValue(); 1154 switch (N->getOpcode()) { 1155 default: 1156#ifndef NDEBUG 1157 dbgs() << "WidenVectorResult #" << ResNo << ": "; 1158 N->dump(&DAG); 1159 dbgs() << "\n"; 1160#endif 1161 llvm_unreachable("Do not know how to widen the result of this operator!"); 1162 1163 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1164 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1165 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1166 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1167 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1168 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1169 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1170 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1171 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1172 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1173 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1174 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1175 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1176 case ISD::VECTOR_SHUFFLE: 1177 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1178 break; 1179 case ISD::VSETCC: 1180 Res = WidenVecRes_VSETCC(N); 1181 break; 1182 1183 case ISD::ADD: 1184 case ISD::AND: 1185 case ISD::BSWAP: 1186 case ISD::FADD: 1187 case ISD::FCOPYSIGN: 1188 case ISD::FDIV: 1189 case ISD::FMUL: 1190 case ISD::FPOW: 1191 case ISD::FPOWI: 1192 case ISD::FREM: 1193 case ISD::FSUB: 1194 case ISD::MUL: 1195 case ISD::MULHS: 1196 case ISD::MULHU: 1197 case ISD::OR: 1198 case ISD::SDIV: 1199 case ISD::SREM: 1200 case ISD::UDIV: 1201 case ISD::UREM: 1202 case ISD::SUB: 1203 case ISD::XOR: 1204 Res = WidenVecRes_Binary(N); 1205 break; 1206 1207 case ISD::SHL: 1208 case ISD::SRA: 1209 case ISD::SRL: 1210 Res = WidenVecRes_Shift(N); 1211 break; 1212 1213 case ISD::FP_ROUND: 1214 case ISD::FP_TO_SINT: 1215 case ISD::FP_TO_UINT: 1216 case ISD::SINT_TO_FP: 1217 case ISD::UINT_TO_FP: 1218 case ISD::TRUNCATE: 1219 case ISD::SIGN_EXTEND: 1220 case ISD::ZERO_EXTEND: 1221 case ISD::ANY_EXTEND: 1222 Res = WidenVecRes_Convert(N); 1223 break; 1224 1225 case ISD::CTLZ: 1226 case ISD::CTPOP: 1227 case ISD::CTTZ: 1228 case ISD::FABS: 1229 case ISD::FCOS: 1230 case ISD::FNEG: 1231 case ISD::FSIN: 1232 case ISD::FSQRT: 1233 Res = WidenVecRes_Unary(N); 1234 break; 1235 } 1236 1237 // If Res is null, the sub-method took care of registering the result. 1238 if (Res.getNode()) 1239 SetWidenedVector(SDValue(N, ResNo), Res); 1240} 1241 1242SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1243 // Binary op widening. 1244 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1245 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1246 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1247 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1248} 1249 1250SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1251 SDValue InOp = N->getOperand(0); 1252 DebugLoc dl = N->getDebugLoc(); 1253 1254 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1255 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1256 1257 EVT InVT = InOp.getValueType(); 1258 EVT InEltVT = InVT.getVectorElementType(); 1259 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1260 1261 unsigned Opcode = N->getOpcode(); 1262 unsigned InVTNumElts = InVT.getVectorNumElements(); 1263 1264 if (getTypeAction(InVT) == WidenVector) { 1265 InOp = GetWidenedVector(N->getOperand(0)); 1266 InVT = InOp.getValueType(); 1267 InVTNumElts = InVT.getVectorNumElements(); 1268 if (InVTNumElts == WidenNumElts) 1269 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1270 } 1271 1272 if (TLI.isTypeLegal(InWidenVT)) { 1273 // Because the result and the input are different vector types, widening 1274 // the result could create a legal type but widening the input might make 1275 // it an illegal type that might lead to repeatedly splitting the input 1276 // and then widening it. To avoid this, we widen the input only if 1277 // it results in a legal type. 1278 if (WidenNumElts % InVTNumElts == 0) { 1279 // Widen the input and call convert on the widened input vector. 1280 unsigned NumConcat = WidenNumElts/InVTNumElts; 1281 SmallVector<SDValue, 16> Ops(NumConcat); 1282 Ops[0] = InOp; 1283 SDValue UndefVal = DAG.getUNDEF(InVT); 1284 for (unsigned i = 1; i != NumConcat; ++i) 1285 Ops[i] = UndefVal; 1286 return DAG.getNode(Opcode, dl, WidenVT, 1287 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1288 &Ops[0], NumConcat)); 1289 } 1290 1291 if (InVTNumElts % WidenNumElts == 0) { 1292 // Extract the input and convert the shorten input vector. 1293 return DAG.getNode(Opcode, dl, WidenVT, 1294 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1295 InOp, DAG.getIntPtrConstant(0))); 1296 } 1297 } 1298 1299 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1300 SmallVector<SDValue, 16> Ops(WidenNumElts); 1301 EVT EltVT = WidenVT.getVectorElementType(); 1302 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1303 unsigned i; 1304 for (i=0; i < MinElts; ++i) 1305 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1306 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1307 DAG.getIntPtrConstant(i))); 1308 1309 SDValue UndefVal = DAG.getUNDEF(EltVT); 1310 for (; i < WidenNumElts; ++i) 1311 Ops[i] = UndefVal; 1312 1313 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1314} 1315 1316SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1317 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1318 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1319 SDValue ShOp = N->getOperand(1); 1320 1321 EVT ShVT = ShOp.getValueType(); 1322 if (getTypeAction(ShVT) == WidenVector) { 1323 ShOp = GetWidenedVector(ShOp); 1324 ShVT = ShOp.getValueType(); 1325 } 1326 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(), 1327 WidenVT.getVectorNumElements()); 1328 if (ShVT != ShWidenVT) 1329 ShOp = ModifyToType(ShOp, ShWidenVT); 1330 1331 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1332} 1333 1334SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1335 // Unary op widening. 1336 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1337 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1338 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1339} 1340 1341SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1342 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1343 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1344 cast<VTSDNode>(N->getOperand(1))->getVT() 1345 .getVectorElementType(), 1346 WidenVT.getVectorNumElements()); 1347 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1348 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 1349 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1350} 1351 1352SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1353 SDValue InOp = N->getOperand(0); 1354 EVT InVT = InOp.getValueType(); 1355 EVT VT = N->getValueType(0); 1356 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1357 DebugLoc dl = N->getDebugLoc(); 1358 1359 switch (getTypeAction(InVT)) { 1360 default: 1361 assert(false && "Unknown type action!"); 1362 break; 1363 case Legal: 1364 break; 1365 case PromoteInteger: 1366 // If the InOp is promoted to the same size, convert it. Otherwise, 1367 // fall out of the switch and widen the promoted input. 1368 InOp = GetPromotedInteger(InOp); 1369 InVT = InOp.getValueType(); 1370 if (WidenVT.bitsEq(InVT)) 1371 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1372 break; 1373 case SoftenFloat: 1374 case ExpandInteger: 1375 case ExpandFloat: 1376 case ScalarizeVector: 1377 case SplitVector: 1378 break; 1379 case WidenVector: 1380 // If the InOp is widened to the same size, convert it. Otherwise, fall 1381 // out of the switch and widen the widened input. 1382 InOp = GetWidenedVector(InOp); 1383 InVT = InOp.getValueType(); 1384 if (WidenVT.bitsEq(InVT)) 1385 // The input widens to the same size. Convert to the widen value. 1386 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1387 break; 1388 } 1389 1390 unsigned WidenSize = WidenVT.getSizeInBits(); 1391 unsigned InSize = InVT.getSizeInBits(); 1392 if (WidenSize % InSize == 0) { 1393 // Determine new input vector type. The new input vector type will use 1394 // the same element type (if its a vector) or use the input type as a 1395 // vector. It is the same size as the type to widen to. 1396 EVT NewInVT; 1397 unsigned NewNumElts = WidenSize / InSize; 1398 if (InVT.isVector()) { 1399 EVT InEltVT = InVT.getVectorElementType(); 1400 NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits()); 1401 } else { 1402 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1403 } 1404 1405 if (TLI.isTypeLegal(NewInVT)) { 1406 // Because the result and the input are different vector types, widening 1407 // the result could create a legal type but widening the input might make 1408 // it an illegal type that might lead to repeatedly splitting the input 1409 // and then widening it. To avoid this, we widen the input only if 1410 // it results in a legal type. 1411 SmallVector<SDValue, 16> Ops(NewNumElts); 1412 SDValue UndefVal = DAG.getUNDEF(InVT); 1413 Ops[0] = InOp; 1414 for (unsigned i = 1; i < NewNumElts; ++i) 1415 Ops[i] = UndefVal; 1416 1417 SDValue NewVec; 1418 if (InVT.isVector()) 1419 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1420 NewInVT, &Ops[0], NewNumElts); 1421 else 1422 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1423 NewInVT, &Ops[0], NewNumElts); 1424 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1425 } 1426 } 1427 1428 return CreateStackStoreLoad(InOp, WidenVT); 1429} 1430 1431SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1432 DebugLoc dl = N->getDebugLoc(); 1433 // Build a vector with undefined for the new nodes. 1434 EVT VT = N->getValueType(0); 1435 EVT EltVT = VT.getVectorElementType(); 1436 unsigned NumElts = VT.getVectorNumElements(); 1437 1438 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1439 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1440 1441 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1442 NewOps.reserve(WidenNumElts); 1443 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1444 NewOps.push_back(DAG.getUNDEF(EltVT)); 1445 1446 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1447} 1448 1449SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1450 EVT InVT = N->getOperand(0).getValueType(); 1451 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1452 DebugLoc dl = N->getDebugLoc(); 1453 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1454 unsigned NumOperands = N->getNumOperands(); 1455 1456 bool InputWidened = false; // Indicates we need to widen the input. 1457 if (getTypeAction(InVT) != WidenVector) { 1458 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1459 // Add undef vectors to widen to correct length. 1460 unsigned NumConcat = WidenVT.getVectorNumElements() / 1461 InVT.getVectorNumElements(); 1462 SDValue UndefVal = DAG.getUNDEF(InVT); 1463 SmallVector<SDValue, 16> Ops(NumConcat); 1464 for (unsigned i=0; i < NumOperands; ++i) 1465 Ops[i] = N->getOperand(i); 1466 for (unsigned i = NumOperands; i != NumConcat; ++i) 1467 Ops[i] = UndefVal; 1468 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1469 } 1470 } else { 1471 InputWidened = true; 1472 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1473 // The inputs and the result are widen to the same value. 1474 unsigned i; 1475 for (i=1; i < NumOperands; ++i) 1476 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1477 break; 1478 1479 if (i > NumOperands) 1480 // Everything but the first operand is an UNDEF so just return the 1481 // widened first operand. 1482 return GetWidenedVector(N->getOperand(0)); 1483 1484 if (NumOperands == 2) { 1485 // Replace concat of two operands with a shuffle. 1486 SmallVector<int, 16> MaskOps(WidenNumElts); 1487 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1488 MaskOps[i] = i; 1489 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1490 } 1491 return DAG.getVectorShuffle(WidenVT, dl, 1492 GetWidenedVector(N->getOperand(0)), 1493 GetWidenedVector(N->getOperand(1)), 1494 &MaskOps[0]); 1495 } 1496 } 1497 } 1498 1499 // Fall back to use extracts and build vector. 1500 EVT EltVT = WidenVT.getVectorElementType(); 1501 unsigned NumInElts = InVT.getVectorNumElements(); 1502 SmallVector<SDValue, 16> Ops(WidenNumElts); 1503 unsigned Idx = 0; 1504 for (unsigned i=0; i < NumOperands; ++i) { 1505 SDValue InOp = N->getOperand(i); 1506 if (InputWidened) 1507 InOp = GetWidenedVector(InOp); 1508 for (unsigned j=0; j < NumInElts; ++j) 1509 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1510 DAG.getIntPtrConstant(j)); 1511 } 1512 SDValue UndefVal = DAG.getUNDEF(EltVT); 1513 for (; Idx < WidenNumElts; ++Idx) 1514 Ops[Idx] = UndefVal; 1515 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1516} 1517 1518SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1519 DebugLoc dl = N->getDebugLoc(); 1520 SDValue InOp = N->getOperand(0); 1521 SDValue RndOp = N->getOperand(3); 1522 SDValue SatOp = N->getOperand(4); 1523 1524 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1525 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1526 1527 EVT InVT = InOp.getValueType(); 1528 EVT InEltVT = InVT.getVectorElementType(); 1529 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1530 1531 SDValue DTyOp = DAG.getValueType(WidenVT); 1532 SDValue STyOp = DAG.getValueType(InWidenVT); 1533 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1534 1535 unsigned InVTNumElts = InVT.getVectorNumElements(); 1536 if (getTypeAction(InVT) == WidenVector) { 1537 InOp = GetWidenedVector(InOp); 1538 InVT = InOp.getValueType(); 1539 InVTNumElts = InVT.getVectorNumElements(); 1540 if (InVTNumElts == WidenNumElts) 1541 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1542 SatOp, CvtCode); 1543 } 1544 1545 if (TLI.isTypeLegal(InWidenVT)) { 1546 // Because the result and the input are different vector types, widening 1547 // the result could create a legal type but widening the input might make 1548 // it an illegal type that might lead to repeatedly splitting the input 1549 // and then widening it. To avoid this, we widen the input only if 1550 // it results in a legal type. 1551 if (WidenNumElts % InVTNumElts == 0) { 1552 // Widen the input and call convert on the widened input vector. 1553 unsigned NumConcat = WidenNumElts/InVTNumElts; 1554 SmallVector<SDValue, 16> Ops(NumConcat); 1555 Ops[0] = InOp; 1556 SDValue UndefVal = DAG.getUNDEF(InVT); 1557 for (unsigned i = 1; i != NumConcat; ++i) { 1558 Ops[i] = UndefVal; 1559 } 1560 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1561 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1562 SatOp, CvtCode); 1563 } 1564 1565 if (InVTNumElts % WidenNumElts == 0) { 1566 // Extract the input and convert the shorten input vector. 1567 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1568 DAG.getIntPtrConstant(0)); 1569 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1570 SatOp, CvtCode); 1571 } 1572 } 1573 1574 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1575 SmallVector<SDValue, 16> Ops(WidenNumElts); 1576 EVT EltVT = WidenVT.getVectorElementType(); 1577 DTyOp = DAG.getValueType(EltVT); 1578 STyOp = DAG.getValueType(InEltVT); 1579 1580 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1581 unsigned i; 1582 for (i=0; i < MinElts; ++i) { 1583 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1584 DAG.getIntPtrConstant(i)); 1585 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1586 SatOp, CvtCode); 1587 } 1588 1589 SDValue UndefVal = DAG.getUNDEF(EltVT); 1590 for (; i < WidenNumElts; ++i) 1591 Ops[i] = UndefVal; 1592 1593 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1594} 1595 1596SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1597 EVT VT = N->getValueType(0); 1598 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1599 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1600 SDValue InOp = N->getOperand(0); 1601 SDValue Idx = N->getOperand(1); 1602 DebugLoc dl = N->getDebugLoc(); 1603 1604 if (getTypeAction(InOp.getValueType()) == WidenVector) 1605 InOp = GetWidenedVector(InOp); 1606 1607 EVT InVT = InOp.getValueType(); 1608 1609 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1610 if (CIdx) { 1611 unsigned IdxVal = CIdx->getZExtValue(); 1612 // Check if we can just return the input vector after widening. 1613 if (IdxVal == 0 && InVT == WidenVT) 1614 return InOp; 1615 1616 // Check if we can extract from the vector. 1617 unsigned InNumElts = InVT.getVectorNumElements(); 1618 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1619 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1620 } 1621 1622 // We could try widening the input to the right length but for now, extract 1623 // the original elements, fill the rest with undefs and build a vector. 1624 SmallVector<SDValue, 16> Ops(WidenNumElts); 1625 EVT EltVT = VT.getVectorElementType(); 1626 EVT IdxVT = Idx.getValueType(); 1627 unsigned NumElts = VT.getVectorNumElements(); 1628 unsigned i; 1629 if (CIdx) { 1630 unsigned IdxVal = CIdx->getZExtValue(); 1631 for (i=0; i < NumElts; ++i) 1632 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1633 DAG.getConstant(IdxVal+i, IdxVT)); 1634 } else { 1635 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1636 for (i=1; i < NumElts; ++i) { 1637 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1638 DAG.getConstant(i, IdxVT)); 1639 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1640 } 1641 } 1642 1643 SDValue UndefVal = DAG.getUNDEF(EltVT); 1644 for (; i < WidenNumElts; ++i) 1645 Ops[i] = UndefVal; 1646 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1647} 1648 1649SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1650 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1651 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1652 InOp.getValueType(), InOp, 1653 N->getOperand(1), N->getOperand(2)); 1654} 1655 1656SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1657 LoadSDNode *LD = cast<LoadSDNode>(N); 1658 ISD::LoadExtType ExtType = LD->getExtensionType(); 1659 1660 SDValue Result; 1661 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1662 if (ExtType != ISD::NON_EXTLOAD) 1663 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 1664 else 1665 Result = GenWidenVectorLoads(LdChain, LD); 1666 1667 // If we generate a single load, we can use that for the chain. Otherwise, 1668 // build a factor node to remember the multiple loads are independent and 1669 // chain to that. 1670 SDValue NewChain; 1671 if (LdChain.size() == 1) 1672 NewChain = LdChain[0]; 1673 else 1674 NewChain = DAG.getNode(ISD::TokenFactor, LD->getDebugLoc(), MVT::Other, 1675 &LdChain[0], LdChain.size()); 1676 1677 // Modified the chain - switch anything that used the old chain to use 1678 // the new one. 1679 ReplaceValueWith(SDValue(N, 1), NewChain); 1680 1681 return Result; 1682} 1683 1684SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1685 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1686 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1687 WidenVT, N->getOperand(0)); 1688} 1689 1690SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1691 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1692 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1693 1694 SDValue Cond1 = N->getOperand(0); 1695 EVT CondVT = Cond1.getValueType(); 1696 if (CondVT.isVector()) { 1697 EVT CondEltVT = CondVT.getVectorElementType(); 1698 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts); 1699 if (getTypeAction(CondVT) == WidenVector) 1700 Cond1 = GetWidenedVector(Cond1); 1701 1702 if (Cond1.getValueType() != CondWidenVT) 1703 Cond1 = ModifyToType(Cond1, CondWidenVT); 1704 } 1705 1706 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1707 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1708 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1709 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1710 WidenVT, Cond1, InOp1, InOp2); 1711} 1712 1713SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1714 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1715 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1716 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1717 InOp1.getValueType(), N->getOperand(0), 1718 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1719} 1720 1721SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1722 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1723 return DAG.getUNDEF(WidenVT); 1724} 1725 1726SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1727 EVT VT = N->getValueType(0); 1728 DebugLoc dl = N->getDebugLoc(); 1729 1730 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1731 unsigned NumElts = VT.getVectorNumElements(); 1732 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1733 1734 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1735 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1736 1737 // Adjust mask based on new input vector length. 1738 SmallVector<int, 16> NewMask; 1739 for (unsigned i = 0; i != NumElts; ++i) { 1740 int Idx = N->getMaskElt(i); 1741 if (Idx < (int)NumElts) 1742 NewMask.push_back(Idx); 1743 else 1744 NewMask.push_back(Idx - NumElts + WidenNumElts); 1745 } 1746 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1747 NewMask.push_back(-1); 1748 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1749} 1750 1751SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1752 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1753 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1754 1755 SDValue InOp1 = N->getOperand(0); 1756 EVT InVT = InOp1.getValueType(); 1757 assert(InVT.isVector() && "can not widen non vector type"); 1758 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); 1759 InOp1 = GetWidenedVector(InOp1); 1760 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1761 1762 // Assume that the input and output will be widen appropriately. If not, 1763 // we will have to unroll it at some point. 1764 assert(InOp1.getValueType() == WidenInVT && 1765 InOp2.getValueType() == WidenInVT && 1766 "Input not widened to expected type!"); 1767 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1768 WidenVT, InOp1, InOp2, N->getOperand(2)); 1769} 1770 1771 1772//===----------------------------------------------------------------------===// 1773// Widen Vector Operand 1774//===----------------------------------------------------------------------===// 1775bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1776 DEBUG(dbgs() << "Widen node operand " << ResNo << ": "; 1777 N->dump(&DAG); 1778 dbgs() << "\n"); 1779 SDValue Res = SDValue(); 1780 1781 switch (N->getOpcode()) { 1782 default: 1783#ifndef NDEBUG 1784 dbgs() << "WidenVectorOperand op #" << ResNo << ": "; 1785 N->dump(&DAG); 1786 dbgs() << "\n"; 1787#endif 1788 llvm_unreachable("Do not know how to widen this operator's operand!"); 1789 1790 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1791 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1792 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 1793 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1794 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1795 1796 case ISD::FP_ROUND: 1797 case ISD::FP_TO_SINT: 1798 case ISD::FP_TO_UINT: 1799 case ISD::SINT_TO_FP: 1800 case ISD::UINT_TO_FP: 1801 case ISD::TRUNCATE: 1802 case ISD::SIGN_EXTEND: 1803 case ISD::ZERO_EXTEND: 1804 case ISD::ANY_EXTEND: 1805 Res = WidenVecOp_Convert(N); 1806 break; 1807 } 1808 1809 // If Res is null, the sub-method took care of registering the result. 1810 if (!Res.getNode()) return false; 1811 1812 // If the result is N, the sub-method updated N in place. Tell the legalizer 1813 // core about this. 1814 if (Res.getNode() == N) 1815 return true; 1816 1817 1818 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1819 "Invalid operand expansion"); 1820 1821 ReplaceValueWith(SDValue(N, 0), Res); 1822 return false; 1823} 1824 1825SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1826 // Since the result is legal and the input is illegal, it is unlikely 1827 // that we can fix the input to a legal type so unroll the convert 1828 // into some scalar code and create a nasty build vector. 1829 EVT VT = N->getValueType(0); 1830 EVT EltVT = VT.getVectorElementType(); 1831 DebugLoc dl = N->getDebugLoc(); 1832 unsigned NumElts = VT.getVectorNumElements(); 1833 SDValue InOp = N->getOperand(0); 1834 if (getTypeAction(InOp.getValueType()) == WidenVector) 1835 InOp = GetWidenedVector(InOp); 1836 EVT InVT = InOp.getValueType(); 1837 EVT InEltVT = InVT.getVectorElementType(); 1838 1839 unsigned Opcode = N->getOpcode(); 1840 SmallVector<SDValue, 16> Ops(NumElts); 1841 for (unsigned i=0; i < NumElts; ++i) 1842 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1843 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1844 DAG.getIntPtrConstant(i))); 1845 1846 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1847} 1848 1849SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1850 EVT VT = N->getValueType(0); 1851 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1852 EVT InWidenVT = InOp.getValueType(); 1853 DebugLoc dl = N->getDebugLoc(); 1854 1855 // Check if we can convert between two legal vector types and extract. 1856 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1857 unsigned Size = VT.getSizeInBits(); 1858 if (InWidenSize % Size == 0 && !VT.isVector()) { 1859 unsigned NewNumElts = InWidenSize / Size; 1860 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 1861 if (TLI.isTypeLegal(NewVT)) { 1862 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1863 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1864 DAG.getIntPtrConstant(0)); 1865 } 1866 } 1867 1868 return CreateStackStoreLoad(InOp, VT); 1869} 1870 1871SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1872 // If the input vector is not legal, it is likely that we will not find a 1873 // legal vector of the same size. Replace the concatenate vector with a 1874 // nasty build vector. 1875 EVT VT = N->getValueType(0); 1876 EVT EltVT = VT.getVectorElementType(); 1877 DebugLoc dl = N->getDebugLoc(); 1878 unsigned NumElts = VT.getVectorNumElements(); 1879 SmallVector<SDValue, 16> Ops(NumElts); 1880 1881 EVT InVT = N->getOperand(0).getValueType(); 1882 unsigned NumInElts = InVT.getVectorNumElements(); 1883 1884 unsigned Idx = 0; 1885 unsigned NumOperands = N->getNumOperands(); 1886 for (unsigned i=0; i < NumOperands; ++i) { 1887 SDValue InOp = N->getOperand(i); 1888 if (getTypeAction(InOp.getValueType()) == WidenVector) 1889 InOp = GetWidenedVector(InOp); 1890 for (unsigned j=0; j < NumInElts; ++j) 1891 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1892 DAG.getIntPtrConstant(j)); 1893 } 1894 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1895} 1896 1897SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1898 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1899 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), 1900 N->getValueType(0), InOp, N->getOperand(1)); 1901} 1902 1903SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1904 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1905 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1906 N->getValueType(0), InOp, N->getOperand(1)); 1907} 1908 1909SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1910 // We have to widen the value but we want only to store the original 1911 // vector type. 1912 StoreSDNode *ST = cast<StoreSDNode>(N); 1913 1914 SmallVector<SDValue, 16> StChain; 1915 if (ST->isTruncatingStore()) 1916 GenWidenVectorTruncStores(StChain, ST); 1917 else 1918 GenWidenVectorStores(StChain, ST); 1919 1920 if (StChain.size() == 1) 1921 return StChain[0]; 1922 else 1923 return DAG.getNode(ISD::TokenFactor, ST->getDebugLoc(), 1924 MVT::Other,&StChain[0],StChain.size()); 1925} 1926 1927//===----------------------------------------------------------------------===// 1928// Vector Widening Utilities 1929//===----------------------------------------------------------------------===// 1930 1931// Utility function to find the type to chop up a widen vector for load/store 1932// TLI: Target lowering used to determine legal types. 1933// Width: Width left need to load/store. 1934// WidenVT: The widen vector type to load to/store from 1935// Align: If 0, don't allow use of a wider type 1936// WidenEx: If Align is not 0, the amount additional we can load/store from. 1937 1938static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 1939 unsigned Width, EVT WidenVT, 1940 unsigned Align = 0, unsigned WidenEx = 0) { 1941 EVT WidenEltVT = WidenVT.getVectorElementType(); 1942 unsigned WidenWidth = WidenVT.getSizeInBits(); 1943 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 1944 unsigned AlignInBits = Align*8; 1945 1946 // If we have one element to load/store, return it. 1947 EVT RetVT = WidenEltVT; 1948 if (Width == WidenEltWidth) 1949 return RetVT; 1950 1951 // See if there is larger legal integer than the element type to load/store 1952 unsigned VT; 1953 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 1954 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 1955 EVT MemVT((MVT::SimpleValueType) VT); 1956 unsigned MemVTWidth = MemVT.getSizeInBits(); 1957 if (MemVT.getSizeInBits() <= WidenEltWidth) 1958 break; 1959 if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 1960 (MemVTWidth <= Width || 1961 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 1962 RetVT = MemVT; 1963 break; 1964 } 1965 } 1966 1967 // See if there is a larger vector type to load/store that has the same vector 1968 // element type and is evenly divisible with the WidenVT. 1969 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 1970 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 1971 EVT MemVT = (MVT::SimpleValueType) VT; 1972 unsigned MemVTWidth = MemVT.getSizeInBits(); 1973 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 1974 (WidenWidth % MemVTWidth) == 0 && 1975 (MemVTWidth <= Width || 1976 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 1977 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 1978 return MemVT; 1979 } 1980 } 1981 1982 return RetVT; 1983} 1984 1985// Builds a vector type from scalar loads 1986// VecTy: Resulting Vector type 1987// LDOps: Load operators to build a vector type 1988// [Start,End) the list of loads to use. 1989static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 1990 SmallVector<SDValue, 16>& LdOps, 1991 unsigned Start, unsigned End) { 1992 DebugLoc dl = LdOps[Start].getDebugLoc(); 1993 EVT LdTy = LdOps[Start].getValueType(); 1994 unsigned Width = VecTy.getSizeInBits(); 1995 unsigned NumElts = Width / LdTy.getSizeInBits(); 1996 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 1997 1998 unsigned Idx = 1; 1999 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2000 2001 for (unsigned i = Start + 1; i != End; ++i) { 2002 EVT NewLdTy = LdOps[i].getValueType(); 2003 if (NewLdTy != LdTy) { 2004 NumElts = Width / NewLdTy.getSizeInBits(); 2005 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2006 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2007 // Readjust position and vector position based on new load type 2008 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2009 LdTy = NewLdTy; 2010 } 2011 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2012 DAG.getIntPtrConstant(Idx++)); 2013 } 2014 return DAG.getNode(ISD::BIT_CONVERT, dl, VecTy, VecOp); 2015} 2016 2017SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2018 LoadSDNode * LD) { 2019 // The strategy assumes that we can efficiently load powers of two widths. 2020 // The routines chops the vector into the largest vector loads with the same 2021 // element type or scalar loads and then recombines it to the widen vector 2022 // type. 2023 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 2024 unsigned WidenWidth = WidenVT.getSizeInBits(); 2025 EVT LdVT = LD->getMemoryVT(); 2026 DebugLoc dl = LD->getDebugLoc(); 2027 assert(LdVT.isVector() && WidenVT.isVector()); 2028 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2029 2030 // Load information 2031 SDValue Chain = LD->getChain(); 2032 SDValue BasePtr = LD->getBasePtr(); 2033 int SVOffset = LD->getSrcValueOffset(); 2034 unsigned Align = LD->getAlignment(); 2035 bool isVolatile = LD->isVolatile(); 2036 const Value *SV = LD->getSrcValue(); 2037 2038 int LdWidth = LdVT.getSizeInBits(); 2039 int WidthDiff = WidenWidth - LdWidth; // Difference 2040 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 2041 2042 // Find the vector type that can load from. 2043 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2044 int NewVTWidth = NewVT.getSizeInBits(); 2045 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, SV, SVOffset, 2046 isVolatile, Align); 2047 LdChain.push_back(LdOp.getValue(1)); 2048 2049 // Check if we can load the element with one instruction 2050 if (LdWidth <= NewVTWidth) { 2051 if (NewVT.isVector()) { 2052 if (NewVT != WidenVT) { 2053 assert(WidenWidth % NewVTWidth == 0); 2054 unsigned NumConcat = WidenWidth / NewVTWidth; 2055 SmallVector<SDValue, 16> ConcatOps(NumConcat); 2056 SDValue UndefVal = DAG.getUNDEF(NewVT); 2057 ConcatOps[0] = LdOp; 2058 for (unsigned i = 1; i != NumConcat; ++i) 2059 ConcatOps[i] = UndefVal; 2060 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], 2061 NumConcat); 2062 } else 2063 return LdOp; 2064 } else { 2065 unsigned NumElts = WidenWidth / LdWidth; 2066 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2067 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2068 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, VecOp); 2069 } 2070 } 2071 2072 // Load vector by using multiple loads from largest vector to scalar 2073 SmallVector<SDValue, 16> LdOps; 2074 LdOps.push_back(LdOp); 2075 2076 LdWidth -= NewVTWidth; 2077 unsigned Offset = 0; 2078 2079 while (LdWidth > 0) { 2080 unsigned Increment = NewVTWidth / 8; 2081 Offset += Increment; 2082 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2083 DAG.getIntPtrConstant(Increment)); 2084 2085 if (LdWidth < NewVTWidth) { 2086 // Our current type we are using is too large, find a better size 2087 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2088 NewVTWidth = NewVT.getSizeInBits(); 2089 } 2090 2091 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, SV, 2092 SVOffset+Offset, isVolatile, 2093 MinAlign(Align, Increment)); 2094 LdChain.push_back(LdOp.getValue(1)); 2095 LdOps.push_back(LdOp); 2096 2097 LdWidth -= NewVTWidth; 2098 } 2099 2100 // Build the vector from the loads operations 2101 unsigned End = LdOps.size(); 2102 if (LdOps[0].getValueType().isVector()) { 2103 // If the load contains vectors, build the vector using concat vector. 2104 // All of the vectors used to loads are power of 2 and the scalars load 2105 // can be combined to make a power of 2 vector. 2106 SmallVector<SDValue, 16> ConcatOps(End); 2107 int i = End - 1; 2108 int Idx = End; 2109 EVT LdTy = LdOps[i].getValueType(); 2110 // First combine the scalar loads to a vector 2111 if (!LdTy.isVector()) { 2112 for (--i; i >= 0; --i) { 2113 LdTy = LdOps[i].getValueType(); 2114 if (LdTy.isVector()) 2115 break; 2116 } 2117 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 2118 } 2119 ConcatOps[--Idx] = LdOps[i]; 2120 for (--i; i >= 0; --i) { 2121 EVT NewLdTy = LdOps[i].getValueType(); 2122 if (NewLdTy != LdTy) { 2123 // Create a larger vector 2124 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 2125 &ConcatOps[Idx], End - Idx); 2126 Idx = End - 1; 2127 LdTy = NewLdTy; 2128 } 2129 ConcatOps[--Idx] = LdOps[i]; 2130 } 2131 2132 if (WidenWidth != LdTy.getSizeInBits()*(End - Idx)) { 2133 // We need to fill the rest with undefs to build the vector 2134 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 2135 SmallVector<SDValue, 16> WidenOps(NumOps); 2136 SDValue UndefVal = DAG.getUNDEF(LdTy); 2137 unsigned i = 0; 2138 for (; i != End-Idx; ++i) 2139 WidenOps[i] = ConcatOps[Idx+i]; 2140 for (; i != NumOps; ++i) 2141 WidenOps[i] = UndefVal; 2142 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &WidenOps[0],NumOps); 2143 } else 2144 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2145 &ConcatOps[Idx], End - Idx); 2146 } else // All the loads are scalar loads. 2147 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 2148} 2149 2150SDValue 2151DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVector<SDValue, 16>& LdChain, 2152 LoadSDNode * LD, 2153 ISD::LoadExtType ExtType) { 2154 // For extension loads, it may not be more efficient to chop up the vector 2155 // and then extended it. Instead, we unroll the load and build a new vector. 2156 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2157 EVT LdVT = LD->getMemoryVT(); 2158 DebugLoc dl = LD->getDebugLoc(); 2159 assert(LdVT.isVector() && WidenVT.isVector()); 2160 2161 // Load information 2162 SDValue Chain = LD->getChain(); 2163 SDValue BasePtr = LD->getBasePtr(); 2164 int SVOffset = LD->getSrcValueOffset(); 2165 unsigned Align = LD->getAlignment(); 2166 bool isVolatile = LD->isVolatile(); 2167 const Value *SV = LD->getSrcValue(); 2168 2169 EVT EltVT = WidenVT.getVectorElementType(); 2170 EVT LdEltVT = LdVT.getVectorElementType(); 2171 unsigned NumElts = LdVT.getVectorNumElements(); 2172 2173 // Load each element and widen 2174 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2175 SmallVector<SDValue, 16> Ops(WidenNumElts); 2176 unsigned Increment = LdEltVT.getSizeInBits() / 8; 2177 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 2178 LdEltVT, isVolatile, Align); 2179 LdChain.push_back(Ops[0].getValue(1)); 2180 unsigned i = 0, Offset = Increment; 2181 for (i=1; i < NumElts; ++i, Offset += Increment) { 2182 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2183 BasePtr, DAG.getIntPtrConstant(Offset)); 2184 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 2185 SVOffset + Offset, LdEltVT, isVolatile, Align); 2186 LdChain.push_back(Ops[i].getValue(1)); 2187 } 2188 2189 // Fill the rest with undefs 2190 SDValue UndefVal = DAG.getUNDEF(EltVT); 2191 for (; i != WidenNumElts; ++i) 2192 Ops[i] = UndefVal; 2193 2194 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 2195} 2196 2197 2198void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2199 StoreSDNode *ST) { 2200 // The strategy assumes that we can efficiently store powers of two widths. 2201 // The routines chops the vector into the largest vector stores with the same 2202 // element type or scalar stores. 2203 SDValue Chain = ST->getChain(); 2204 SDValue BasePtr = ST->getBasePtr(); 2205 const Value *SV = ST->getSrcValue(); 2206 int SVOffset = ST->getSrcValueOffset(); 2207 unsigned Align = ST->getAlignment(); 2208 bool isVolatile = ST->isVolatile(); 2209 SDValue ValOp = GetWidenedVector(ST->getValue()); 2210 DebugLoc dl = ST->getDebugLoc(); 2211 2212 EVT StVT = ST->getMemoryVT(); 2213 unsigned StWidth = StVT.getSizeInBits(); 2214 EVT ValVT = ValOp.getValueType(); 2215 unsigned ValWidth = ValVT.getSizeInBits(); 2216 EVT ValEltVT = ValVT.getVectorElementType(); 2217 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 2218 assert(StVT.getVectorElementType() == ValEltVT); 2219 2220 int Idx = 0; // current index to store 2221 unsigned Offset = 0; // offset from base to store 2222 while (StWidth != 0) { 2223 // Find the largest vector type we can store with 2224 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 2225 unsigned NewVTWidth = NewVT.getSizeInBits(); 2226 unsigned Increment = NewVTWidth / 8; 2227 if (NewVT.isVector()) { 2228 unsigned NumVTElts = NewVT.getVectorNumElements(); 2229 do { 2230 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 2231 DAG.getIntPtrConstant(Idx)); 2232 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2233 SVOffset + Offset, isVolatile, 2234 MinAlign(Align, Offset))); 2235 StWidth -= NewVTWidth; 2236 Offset += Increment; 2237 Idx += NumVTElts; 2238 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2239 DAG.getIntPtrConstant(Increment)); 2240 } while (StWidth != 0 && StWidth >= NewVTWidth); 2241 } else { 2242 // Cast the vector to the scalar type we can store 2243 unsigned NumElts = ValWidth / NewVTWidth; 2244 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2245 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2246 // Readjust index position based on new vector type 2247 Idx = Idx * ValEltWidth / NewVTWidth; 2248 do { 2249 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 2250 DAG.getIntPtrConstant(Idx++)); 2251 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2252 SVOffset + Offset, isVolatile, 2253 MinAlign(Align, Offset))); 2254 StWidth -= NewVTWidth; 2255 Offset += Increment; 2256 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2257 DAG.getIntPtrConstant(Increment)); 2258 } while (StWidth != 0 && StWidth >= NewVTWidth); 2259 // Restore index back to be relative to the original widen element type 2260 Idx = Idx * NewVTWidth / ValEltWidth; 2261 } 2262 } 2263} 2264 2265void 2266DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVector<SDValue, 16>& StChain, 2267 StoreSDNode *ST) { 2268 // For extension loads, it may not be more efficient to truncate the vector 2269 // and then store it. Instead, we extract each element and then store it. 2270 SDValue Chain = ST->getChain(); 2271 SDValue BasePtr = ST->getBasePtr(); 2272 const Value *SV = ST->getSrcValue(); 2273 int SVOffset = ST->getSrcValueOffset(); 2274 unsigned Align = ST->getAlignment(); 2275 bool isVolatile = ST->isVolatile(); 2276 SDValue ValOp = GetWidenedVector(ST->getValue()); 2277 DebugLoc dl = ST->getDebugLoc(); 2278 2279 EVT StVT = ST->getMemoryVT(); 2280 EVT ValVT = ValOp.getValueType(); 2281 2282 // It must be true that we the widen vector type is bigger than where 2283 // we need to store. 2284 assert(StVT.isVector() && ValOp.getValueType().isVector()); 2285 assert(StVT.bitsLT(ValOp.getValueType())); 2286 2287 // For truncating stores, we can not play the tricks of chopping legal 2288 // vector types and bit cast it to the right type. Instead, we unroll 2289 // the store. 2290 EVT StEltVT = StVT.getVectorElementType(); 2291 EVT ValEltVT = ValVT.getVectorElementType(); 2292 unsigned Increment = ValEltVT.getSizeInBits() / 8; 2293 unsigned NumElts = StVT.getVectorNumElements(); 2294 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2295 DAG.getIntPtrConstant(0)); 2296 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 2297 SVOffset, StEltVT, 2298 isVolatile, Align)); 2299 unsigned Offset = Increment; 2300 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 2301 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2302 BasePtr, DAG.getIntPtrConstant(Offset)); 2303 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2304 DAG.getIntPtrConstant(0)); 2305 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 2306 SVOffset + Offset, StEltVT, 2307 isVolatile, MinAlign(Align, Offset))); 2308 } 2309} 2310 2311/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2312/// input vector must have the same element type as NVT. 2313SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2314 // Note that InOp might have been widened so it might already have 2315 // the right width or it might need be narrowed. 2316 EVT InVT = InOp.getValueType(); 2317 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2318 "input and widen element type must match"); 2319 DebugLoc dl = InOp.getDebugLoc(); 2320 2321 // Check if InOp already has the right width. 2322 if (InVT == NVT) 2323 return InOp; 2324 2325 unsigned InNumElts = InVT.getVectorNumElements(); 2326 unsigned WidenNumElts = NVT.getVectorNumElements(); 2327 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2328 unsigned NumConcat = WidenNumElts / InNumElts; 2329 SmallVector<SDValue, 16> Ops(NumConcat); 2330 SDValue UndefVal = DAG.getUNDEF(InVT); 2331 Ops[0] = InOp; 2332 for (unsigned i = 1; i != NumConcat; ++i) 2333 Ops[i] = UndefVal; 2334 2335 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2336 } 2337 2338 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2339 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2340 DAG.getIntPtrConstant(0)); 2341 2342 // Fall back to extract and build. 2343 SmallVector<SDValue, 16> Ops(WidenNumElts); 2344 EVT EltVT = NVT.getVectorElementType(); 2345 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2346 unsigned Idx; 2347 for (Idx = 0; Idx < MinNumElts; ++Idx) 2348 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2349 DAG.getIntPtrConstant(Idx)); 2350 2351 SDValue UndefVal = DAG.getUNDEF(EltVT); 2352 for ( ; Idx < WidenNumElts; ++Idx) 2353 Ops[Idx] = UndefVal; 2354 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2355} 2356