LegalizeVectorTypes.cpp revision cd6e725f21852e2f8cdf5fd0e65eb42c224776f8
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in two vectors of half the size. For example, implementing 19// <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26#include "llvm/Support/ErrorHandling.h" 27#include "llvm/Support/raw_ostream.h" 28using namespace llvm; 29 30//===----------------------------------------------------------------------===// 31// Result Vector Scalarization: <1 x ty> -> ty. 32//===----------------------------------------------------------------------===// 33 34void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 35 DEBUG(errs() << "Scalarize node result " << ResNo << ": "; 36 N->dump(&DAG); 37 errs() << "\n"); 38 SDValue R = SDValue(); 39 40 switch (N->getOpcode()) { 41 default: 42#ifndef NDEBUG 43 errs() << "ScalarizeVectorResult #" << ResNo << ": "; 44 N->dump(&DAG); 45 errs() << "\n"; 46#endif 47 llvm_unreachable("Do not know how to scalarize the result of this operator!"); 48 49 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 50 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 51 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 52 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 53 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 54 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 55 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 56 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 57 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 58 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 59 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 60 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 61 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 62 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 63 64 case ISD::CTLZ: 65 case ISD::CTPOP: 66 case ISD::CTTZ: 67 case ISD::FABS: 68 case ISD::FCOS: 69 case ISD::FNEG: 70 case ISD::FP_TO_SINT: 71 case ISD::FP_TO_UINT: 72 case ISD::FSIN: 73 case ISD::FSQRT: 74 case ISD::FTRUNC: 75 case ISD::FFLOOR: 76 case ISD::FCEIL: 77 case ISD::FRINT: 78 case ISD::FNEARBYINT: 79 case ISD::UINT_TO_FP: 80 case ISD::SINT_TO_FP: 81 case ISD::TRUNCATE: 82 case ISD::SIGN_EXTEND: 83 case ISD::ZERO_EXTEND: 84 case ISD::ANY_EXTEND: 85 R = ScalarizeVecRes_UnaryOp(N); 86 break; 87 88 case ISD::ADD: 89 case ISD::AND: 90 case ISD::FADD: 91 case ISD::FDIV: 92 case ISD::FMUL: 93 case ISD::FPOW: 94 case ISD::FREM: 95 case ISD::FSUB: 96 case ISD::MUL: 97 case ISD::OR: 98 case ISD::SDIV: 99 case ISD::SREM: 100 case ISD::SUB: 101 case ISD::UDIV: 102 case ISD::UREM: 103 case ISD::XOR: 104 case ISD::SHL: 105 case ISD::SRA: 106 case ISD::SRL: 107 R = ScalarizeVecRes_BinOp(N); 108 break; 109 } 110 111 // If R is null, the sub-method took care of registering the result. 112 if (R.getNode()) 113 SetScalarizedVector(SDValue(N, ResNo), R); 114} 115 116SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 117 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 118 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 119 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 120 LHS.getValueType(), LHS, RHS); 121} 122 123SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 124 EVT NewVT = N->getValueType(0).getVectorElementType(); 125 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 126 NewVT, N->getOperand(0)); 127} 128 129SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 130 EVT NewVT = N->getValueType(0).getVectorElementType(); 131 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 132 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 133 Op0, DAG.getValueType(NewVT), 134 DAG.getValueType(Op0.getValueType()), 135 N->getOperand(3), 136 N->getOperand(4), 137 cast<CvtRndSatSDNode>(N)->getCvtCode()); 138} 139 140SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 141 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 142 N->getValueType(0).getVectorElementType(), 143 N->getOperand(0), N->getOperand(1)); 144} 145 146SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 147 SDValue Op = GetScalarizedVector(N->getOperand(0)); 148 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 149 Op.getValueType(), Op, N->getOperand(1)); 150} 151 152SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 153 // The value to insert may have a wider type than the vector element type, 154 // so be sure to truncate it to the element type if necessary. 155 SDValue Op = N->getOperand(1); 156 EVT EltVT = N->getValueType(0).getVectorElementType(); 157 if (Op.getValueType() != EltVT) 158 // FIXME: Can this happen for floating point types? 159 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 160 return Op; 161} 162 163SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 164 assert(N->isUnindexed() && "Indexed vector load?"); 165 166 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 167 N->getExtensionType(), 168 N->getValueType(0).getVectorElementType(), 169 N->getChain(), N->getBasePtr(), 170 DAG.getUNDEF(N->getBasePtr().getValueType()), 171 N->getSrcValue(), N->getSrcValueOffset(), 172 N->getMemoryVT().getVectorElementType(), 173 N->isVolatile(), N->getOriginalAlignment()); 174 175 // Legalized the chain result - switch anything that used the old chain to 176 // use the new one. 177 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 178 return Result; 179} 180 181SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 182 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 183 EVT DestVT = N->getValueType(0).getVectorElementType(); 184 SDValue Op = GetScalarizedVector(N->getOperand(0)); 185 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 186} 187 188SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 189 // If the operand is wider than the vector element type then it is implicitly 190 // truncated. Make that explicit here. 191 EVT EltVT = N->getValueType(0).getVectorElementType(); 192 SDValue InOp = N->getOperand(0); 193 if (InOp.getValueType() != EltVT) 194 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 195 return InOp; 196} 197 198SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 199 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 200 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 201 LHS.getValueType(), N->getOperand(0), LHS, 202 GetScalarizedVector(N->getOperand(2))); 203} 204 205SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 206 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 207 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 208 N->getOperand(0), N->getOperand(1), 209 LHS, GetScalarizedVector(N->getOperand(3)), 210 N->getOperand(4)); 211} 212 213SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 214 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 215 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 216 DebugLoc DL = N->getDebugLoc(); 217 218 // Turn it into a scalar SETCC. 219 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 220} 221 222SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 223 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 224} 225 226SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 227 // Figure out if the scalar is the LHS or RHS and return it. 228 SDValue Arg = N->getOperand(2).getOperand(0); 229 if (Arg.getOpcode() == ISD::UNDEF) 230 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 231 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 232 return GetScalarizedVector(N->getOperand(Op)); 233} 234 235SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 236 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 237 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 238 EVT NVT = N->getValueType(0).getVectorElementType(); 239 EVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 240 DebugLoc DL = N->getDebugLoc(); 241 242 // Turn it into a scalar SETCC. 243 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 244 245 // VSETCC always returns a sign-extended value, while SETCC may not. The 246 // SETCC result type may not match the vector element type. Correct these. 247 if (NVT.bitsLE(SVT)) { 248 // The SETCC result type is bigger than the vector element type. 249 // Ensure the SETCC result is sign-extended. 250 if (TLI.getBooleanContents() != 251 TargetLowering::ZeroOrNegativeOneBooleanContent) 252 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 253 DAG.getValueType(MVT::i1)); 254 // Truncate to the final type. 255 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 256 } 257 258 // The SETCC result type is smaller than the vector element type. 259 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 260 if (TLI.getBooleanContents() != 261 TargetLowering::ZeroOrNegativeOneBooleanContent) 262 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 263 // Sign extend to the final type. 264 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 265} 266 267 268//===----------------------------------------------------------------------===// 269// Operand Vector Scalarization <1 x ty> -> ty. 270//===----------------------------------------------------------------------===// 271 272bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 273 DEBUG(errs() << "Scalarize node operand " << OpNo << ": "; 274 N->dump(&DAG); 275 errs() << "\n"); 276 SDValue Res = SDValue(); 277 278 if (Res.getNode() == 0) { 279 switch (N->getOpcode()) { 280 default: 281#ifndef NDEBUG 282 errs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 283 N->dump(&DAG); 284 errs() << "\n"; 285#endif 286 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 287 case ISD::BIT_CONVERT: 288 Res = ScalarizeVecOp_BIT_CONVERT(N); 289 break; 290 case ISD::CONCAT_VECTORS: 291 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 292 break; 293 case ISD::EXTRACT_VECTOR_ELT: 294 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 295 break; 296 case ISD::STORE: 297 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 298 break; 299 } 300 } 301 302 // If the result is null, the sub-method took care of registering results etc. 303 if (!Res.getNode()) return false; 304 305 // If the result is N, the sub-method updated N in place. Tell the legalizer 306 // core about this. 307 if (Res.getNode() == N) 308 return true; 309 310 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 311 "Invalid operand expansion"); 312 313 ReplaceValueWith(SDValue(N, 0), Res); 314 return false; 315} 316 317/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 318/// to be scalarized, it must be <1 x ty>. Convert the element instead. 319SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 320 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 321 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 322 N->getValueType(0), Elt); 323} 324 325/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 326/// use a BUILD_VECTOR instead. 327SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 328 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 329 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 330 Ops[i] = GetScalarizedVector(N->getOperand(i)); 331 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 332 &Ops[0], Ops.size()); 333} 334 335/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 336/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 337/// index. 338SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 339 SDValue Res = GetScalarizedVector(N->getOperand(0)); 340 if (Res.getValueType() != N->getValueType(0)) 341 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 342 Res); 343 return Res; 344} 345 346/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 347/// scalarized, it must be <1 x ty>. Just store the element. 348SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 349 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 350 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 351 DebugLoc dl = N->getDebugLoc(); 352 353 if (N->isTruncatingStore()) 354 return DAG.getTruncStore(N->getChain(), dl, 355 GetScalarizedVector(N->getOperand(1)), 356 N->getBasePtr(), 357 N->getSrcValue(), N->getSrcValueOffset(), 358 N->getMemoryVT().getVectorElementType(), 359 N->isVolatile(), N->getAlignment()); 360 361 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 362 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 363 N->isVolatile(), N->getOriginalAlignment()); 364} 365 366 367//===----------------------------------------------------------------------===// 368// Result Vector Splitting 369//===----------------------------------------------------------------------===// 370 371/// SplitVectorResult - This method is called when the specified result of the 372/// specified node is found to need vector splitting. At this point, the node 373/// may also have invalid operands or may have other results that need 374/// legalization, we just know that (at least) one result needs vector 375/// splitting. 376void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 377 DEBUG(errs() << "Split node result: "; 378 N->dump(&DAG); 379 errs() << "\n"); 380 SDValue Lo, Hi; 381 382 switch (N->getOpcode()) { 383 default: 384#ifndef NDEBUG 385 errs() << "SplitVectorResult #" << ResNo << ": "; 386 N->dump(&DAG); 387 errs() << "\n"; 388#endif 389 llvm_unreachable("Do not know how to split the result of this operator!"); 390 391 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 392 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 393 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 394 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 395 396 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 397 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 398 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 399 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 400 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 401 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 402 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 403 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 404 case ISD::LOAD: 405 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 406 break; 407 case ISD::SETCC: 408 case ISD::VSETCC: 409 SplitVecRes_SETCC(N, Lo, Hi); 410 break; 411 case ISD::VECTOR_SHUFFLE: 412 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 413 break; 414 415 case ISD::CTTZ: 416 case ISD::CTLZ: 417 case ISD::CTPOP: 418 case ISD::FNEG: 419 case ISD::FABS: 420 case ISD::FSQRT: 421 case ISD::FSIN: 422 case ISD::FCOS: 423 case ISD::FTRUNC: 424 case ISD::FFLOOR: 425 case ISD::FCEIL: 426 case ISD::FRINT: 427 case ISD::FNEARBYINT: 428 case ISD::FP_TO_SINT: 429 case ISD::FP_TO_UINT: 430 case ISD::SINT_TO_FP: 431 case ISD::UINT_TO_FP: 432 case ISD::TRUNCATE: 433 case ISD::SIGN_EXTEND: 434 case ISD::ZERO_EXTEND: 435 case ISD::ANY_EXTEND: 436 SplitVecRes_UnaryOp(N, Lo, Hi); 437 break; 438 439 case ISD::ADD: 440 case ISD::SUB: 441 case ISD::MUL: 442 case ISD::FADD: 443 case ISD::FSUB: 444 case ISD::FMUL: 445 case ISD::SDIV: 446 case ISD::UDIV: 447 case ISD::FDIV: 448 case ISD::FPOW: 449 case ISD::AND: 450 case ISD::OR: 451 case ISD::XOR: 452 case ISD::SHL: 453 case ISD::SRA: 454 case ISD::SRL: 455 case ISD::UREM: 456 case ISD::SREM: 457 case ISD::FREM: 458 SplitVecRes_BinOp(N, Lo, Hi); 459 break; 460 } 461 462 // If Lo/Hi is null, the sub-method took care of registering results etc. 463 if (Lo.getNode()) 464 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 465} 466 467void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 468 SDValue &Hi) { 469 SDValue LHSLo, LHSHi; 470 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 471 SDValue RHSLo, RHSHi; 472 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 473 DebugLoc dl = N->getDebugLoc(); 474 475 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 476 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 477} 478 479void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 480 SDValue &Hi) { 481 // We know the result is a vector. The input may be either a vector or a 482 // scalar value. 483 EVT LoVT, HiVT; 484 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 485 DebugLoc dl = N->getDebugLoc(); 486 487 SDValue InOp = N->getOperand(0); 488 EVT InVT = InOp.getValueType(); 489 490 // Handle some special cases efficiently. 491 switch (getTypeAction(InVT)) { 492 default: 493 assert(false && "Unknown type action!"); 494 case Legal: 495 case PromoteInteger: 496 case SoftenFloat: 497 case ScalarizeVector: 498 break; 499 case ExpandInteger: 500 case ExpandFloat: 501 // A scalar to vector conversion, where the scalar needs expansion. 502 // If the vector is being split in two then we can just convert the 503 // expanded pieces. 504 if (LoVT == HiVT) { 505 GetExpandedOp(InOp, Lo, Hi); 506 if (TLI.isBigEndian()) 507 std::swap(Lo, Hi); 508 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 509 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 510 return; 511 } 512 break; 513 case SplitVector: 514 // If the input is a vector that needs to be split, convert each split 515 // piece of the input now. 516 GetSplitVector(InOp, Lo, Hi); 517 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 518 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 519 return; 520 } 521 522 // In the general case, convert the input to an integer and split it by hand. 523 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 524 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 525 if (TLI.isBigEndian()) 526 std::swap(LoIntVT, HiIntVT); 527 528 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 529 530 if (TLI.isBigEndian()) 531 std::swap(Lo, Hi); 532 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 533 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 534} 535 536void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 537 SDValue &Hi) { 538 EVT LoVT, HiVT; 539 DebugLoc dl = N->getDebugLoc(); 540 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 541 unsigned LoNumElts = LoVT.getVectorNumElements(); 542 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 543 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 544 545 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 546 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 547} 548 549void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 550 SDValue &Hi) { 551 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 552 DebugLoc dl = N->getDebugLoc(); 553 unsigned NumSubvectors = N->getNumOperands() / 2; 554 if (NumSubvectors == 1) { 555 Lo = N->getOperand(0); 556 Hi = N->getOperand(1); 557 return; 558 } 559 560 EVT LoVT, HiVT; 561 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 562 563 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 564 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 565 566 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 567 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 568} 569 570void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 571 SDValue &Hi) { 572 EVT LoVT, HiVT; 573 DebugLoc dl = N->getDebugLoc(); 574 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 575 576 SDValue DTyOpLo = DAG.getValueType(LoVT); 577 SDValue DTyOpHi = DAG.getValueType(HiVT); 578 579 SDValue RndOp = N->getOperand(3); 580 SDValue SatOp = N->getOperand(4); 581 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 582 583 // Split the input. 584 SDValue VLo, VHi; 585 EVT InVT = N->getOperand(0).getValueType(); 586 switch (getTypeAction(InVT)) { 587 default: llvm_unreachable("Unexpected type action!"); 588 case Legal: { 589 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 590 LoVT.getVectorNumElements()); 591 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 592 DAG.getIntPtrConstant(0)); 593 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 594 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 595 break; 596 } 597 case SplitVector: 598 GetSplitVector(N->getOperand(0), VLo, VHi); 599 break; 600 case WidenVector: { 601 // If the result needs to be split and the input needs to be widened, 602 // the two types must have different lengths. Use the widened result 603 // and extract from it to do the split. 604 SDValue InOp = GetWidenedVector(N->getOperand(0)); 605 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 606 LoVT.getVectorNumElements()); 607 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 608 DAG.getIntPtrConstant(0)); 609 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 610 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 611 break; 612 } 613 } 614 615 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 616 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 617 618 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 619 CvtCode); 620 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 621 CvtCode); 622} 623 624void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 625 SDValue &Hi) { 626 SDValue Vec = N->getOperand(0); 627 SDValue Idx = N->getOperand(1); 628 EVT IdxVT = Idx.getValueType(); 629 DebugLoc dl = N->getDebugLoc(); 630 631 EVT LoVT, HiVT; 632 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 633 634 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 635 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 636 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 637 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 638} 639 640void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 641 SDValue &Hi) { 642 DebugLoc dl = N->getDebugLoc(); 643 GetSplitVector(N->getOperand(0), Lo, Hi); 644 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 645 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 646} 647 648void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 649 SDValue &Hi) { 650 SDValue Vec = N->getOperand(0); 651 SDValue Elt = N->getOperand(1); 652 SDValue Idx = N->getOperand(2); 653 DebugLoc dl = N->getDebugLoc(); 654 GetSplitVector(Vec, Lo, Hi); 655 656 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 657 unsigned IdxVal = CIdx->getZExtValue(); 658 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 659 if (IdxVal < LoNumElts) 660 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 661 Lo.getValueType(), Lo, Elt, Idx); 662 else 663 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 664 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 665 return; 666 } 667 668 // Spill the vector to the stack. 669 EVT VecVT = Vec.getValueType(); 670 EVT EltVT = VecVT.getVectorElementType(); 671 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 672 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 673 674 // Store the new element. This may be larger than the vector element type, 675 // so use a truncating store. 676 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 677 unsigned Alignment = 678 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext())); 679 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 680 681 // Load the Lo part from the stack slot. 682 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 683 684 // Increment the pointer to the other part. 685 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 686 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 687 DAG.getIntPtrConstant(IncrementSize)); 688 689 // Load the Hi part from the stack slot. 690 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 691 MinAlign(Alignment, IncrementSize)); 692} 693 694void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 695 SDValue &Hi) { 696 EVT LoVT, HiVT; 697 DebugLoc dl = N->getDebugLoc(); 698 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 699 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 700 Hi = DAG.getUNDEF(HiVT); 701} 702 703void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 704 SDValue &Hi) { 705 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 706 EVT LoVT, HiVT; 707 DebugLoc dl = LD->getDebugLoc(); 708 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 709 710 ISD::LoadExtType ExtType = LD->getExtensionType(); 711 SDValue Ch = LD->getChain(); 712 SDValue Ptr = LD->getBasePtr(); 713 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 714 const Value *SV = LD->getSrcValue(); 715 int SVOffset = LD->getSrcValueOffset(); 716 EVT MemoryVT = LD->getMemoryVT(); 717 unsigned Alignment = LD->getOriginalAlignment(); 718 bool isVolatile = LD->isVolatile(); 719 720 EVT LoMemVT, HiMemVT; 721 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 722 723 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 724 SV, SVOffset, LoMemVT, isVolatile, Alignment); 725 726 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 727 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 728 DAG.getIntPtrConstant(IncrementSize)); 729 SVOffset += IncrementSize; 730 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 731 SV, SVOffset, HiMemVT, isVolatile, Alignment); 732 733 // Build a factor node to remember that this load is independent of the 734 // other one. 735 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 736 Hi.getValue(1)); 737 738 // Legalized the chain result - switch anything that used the old chain to 739 // use the new one. 740 ReplaceValueWith(SDValue(LD, 1), Ch); 741} 742 743void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 744 EVT LoVT, HiVT; 745 DebugLoc DL = N->getDebugLoc(); 746 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 747 748 // Split the input. 749 EVT InVT = N->getOperand(0).getValueType(); 750 SDValue LL, LH, RL, RH; 751 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 752 LoVT.getVectorNumElements()); 753 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 754 DAG.getIntPtrConstant(0)); 755 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 756 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 757 758 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 759 DAG.getIntPtrConstant(0)); 760 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 761 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 762 763 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 764 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 765} 766 767void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 768 SDValue &Hi) { 769 // Get the dest types - they may not match the input types, e.g. int_to_fp. 770 EVT LoVT, HiVT; 771 DebugLoc dl = N->getDebugLoc(); 772 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 773 774 // Split the input. 775 EVT InVT = N->getOperand(0).getValueType(); 776 switch (getTypeAction(InVT)) { 777 default: llvm_unreachable("Unexpected type action!"); 778 case Legal: { 779 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 780 LoVT.getVectorNumElements()); 781 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 782 DAG.getIntPtrConstant(0)); 783 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 784 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 785 break; 786 } 787 case SplitVector: 788 GetSplitVector(N->getOperand(0), Lo, Hi); 789 break; 790 case WidenVector: { 791 // If the result needs to be split and the input needs to be widened, 792 // the two types must have different lengths. Use the widened result 793 // and extract from it to do the split. 794 SDValue InOp = GetWidenedVector(N->getOperand(0)); 795 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 796 LoVT.getVectorNumElements()); 797 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 798 DAG.getIntPtrConstant(0)); 799 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 800 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 801 break; 802 } 803 } 804 805 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 806 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 807} 808 809void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 810 SDValue &Lo, SDValue &Hi) { 811 // The low and high parts of the original input give four input vectors. 812 SDValue Inputs[4]; 813 DebugLoc dl = N->getDebugLoc(); 814 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 815 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 816 EVT NewVT = Inputs[0].getValueType(); 817 unsigned NewElts = NewVT.getVectorNumElements(); 818 819 // If Lo or Hi uses elements from at most two of the four input vectors, then 820 // express it as a vector shuffle of those two inputs. Otherwise extract the 821 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 822 SmallVector<int, 16> Ops; 823 for (unsigned High = 0; High < 2; ++High) { 824 SDValue &Output = High ? Hi : Lo; 825 826 // Build a shuffle mask for the output, discovering on the fly which 827 // input vectors to use as shuffle operands (recorded in InputUsed). 828 // If building a suitable shuffle vector proves too hard, then bail 829 // out with useBuildVector set. 830 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 831 unsigned FirstMaskIdx = High * NewElts; 832 bool useBuildVector = false; 833 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 834 // The mask element. This indexes into the input. 835 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 836 837 // The input vector this mask element indexes into. 838 unsigned Input = (unsigned)Idx / NewElts; 839 840 if (Input >= array_lengthof(Inputs)) { 841 // The mask element does not index into any input vector. 842 Ops.push_back(-1); 843 continue; 844 } 845 846 // Turn the index into an offset from the start of the input vector. 847 Idx -= Input * NewElts; 848 849 // Find or create a shuffle vector operand to hold this input. 850 unsigned OpNo; 851 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 852 if (InputUsed[OpNo] == Input) { 853 // This input vector is already an operand. 854 break; 855 } else if (InputUsed[OpNo] == -1U) { 856 // Create a new operand for this input vector. 857 InputUsed[OpNo] = Input; 858 break; 859 } 860 } 861 862 if (OpNo >= array_lengthof(InputUsed)) { 863 // More than two input vectors used! Give up on trying to create a 864 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 865 useBuildVector = true; 866 break; 867 } 868 869 // Add the mask index for the new shuffle vector. 870 Ops.push_back(Idx + OpNo * NewElts); 871 } 872 873 if (useBuildVector) { 874 EVT EltVT = NewVT.getVectorElementType(); 875 SmallVector<SDValue, 16> SVOps; 876 877 // Extract the input elements by hand. 878 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 879 // The mask element. This indexes into the input. 880 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 881 882 // The input vector this mask element indexes into. 883 unsigned Input = (unsigned)Idx / NewElts; 884 885 if (Input >= array_lengthof(Inputs)) { 886 // The mask element is "undef" or indexes off the end of the input. 887 SVOps.push_back(DAG.getUNDEF(EltVT)); 888 continue; 889 } 890 891 // Turn the index into an offset from the start of the input vector. 892 Idx -= Input * NewElts; 893 894 // Extract the vector element by hand. 895 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 896 Inputs[Input], DAG.getIntPtrConstant(Idx))); 897 } 898 899 // Construct the Lo/Hi output using a BUILD_VECTOR. 900 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 901 } else if (InputUsed[0] == -1U) { 902 // No input vectors were used! The result is undefined. 903 Output = DAG.getUNDEF(NewVT); 904 } else { 905 SDValue Op0 = Inputs[InputUsed[0]]; 906 // If only one input was used, use an undefined vector for the other. 907 SDValue Op1 = InputUsed[1] == -1U ? 908 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 909 // At least one input vector was used. Create a new shuffle vector. 910 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 911 } 912 913 Ops.clear(); 914 } 915} 916 917 918//===----------------------------------------------------------------------===// 919// Operand Vector Splitting 920//===----------------------------------------------------------------------===// 921 922/// SplitVectorOperand - This method is called when the specified operand of the 923/// specified node is found to need vector splitting. At this point, all of the 924/// result types of the node are known to be legal, but other operands of the 925/// node may need legalization as well as the specified one. 926bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 927 DEBUG(errs() << "Split node operand: "; 928 N->dump(&DAG); 929 errs() << "\n"); 930 SDValue Res = SDValue(); 931 932 if (Res.getNode() == 0) { 933 switch (N->getOpcode()) { 934 default: 935#ifndef NDEBUG 936 errs() << "SplitVectorOperand Op #" << OpNo << ": "; 937 N->dump(&DAG); 938 errs() << "\n"; 939#endif 940 llvm_unreachable("Do not know how to split this operator's operand!"); 941 942 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 943 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 944 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 945 case ISD::STORE: 946 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 947 break; 948 949 case ISD::CTTZ: 950 case ISD::CTLZ: 951 case ISD::CTPOP: 952 case ISD::FP_TO_SINT: 953 case ISD::FP_TO_UINT: 954 case ISD::SINT_TO_FP: 955 case ISD::UINT_TO_FP: 956 case ISD::TRUNCATE: 957 case ISD::SIGN_EXTEND: 958 case ISD::ZERO_EXTEND: 959 case ISD::ANY_EXTEND: 960 Res = SplitVecOp_UnaryOp(N); 961 break; 962 } 963 } 964 965 // If the result is null, the sub-method took care of registering results etc. 966 if (!Res.getNode()) return false; 967 968 // If the result is N, the sub-method updated N in place. Tell the legalizer 969 // core about this. 970 if (Res.getNode() == N) 971 return true; 972 973 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 974 "Invalid operand expansion"); 975 976 ReplaceValueWith(SDValue(N, 0), Res); 977 return false; 978} 979 980SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 981 // The result has a legal vector type, but the input needs splitting. 982 EVT ResVT = N->getValueType(0); 983 SDValue Lo, Hi; 984 DebugLoc dl = N->getDebugLoc(); 985 GetSplitVector(N->getOperand(0), Lo, Hi); 986 EVT InVT = Lo.getValueType(); 987 988 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 989 InVT.getVectorNumElements()); 990 991 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 992 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 993 994 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 995} 996 997SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 998 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 999 // end up being split all the way down to individual components. Convert the 1000 // split pieces into integers and reassemble. 1001 SDValue Lo, Hi; 1002 GetSplitVector(N->getOperand(0), Lo, Hi); 1003 Lo = BitConvertToInteger(Lo); 1004 Hi = BitConvertToInteger(Hi); 1005 1006 if (TLI.isBigEndian()) 1007 std::swap(Lo, Hi); 1008 1009 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 1010 JoinIntegers(Lo, Hi)); 1011} 1012 1013SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1014 // We know that the extracted result type is legal. For now, assume the index 1015 // is a constant. 1016 EVT SubVT = N->getValueType(0); 1017 SDValue Idx = N->getOperand(1); 1018 DebugLoc dl = N->getDebugLoc(); 1019 SDValue Lo, Hi; 1020 GetSplitVector(N->getOperand(0), Lo, Hi); 1021 1022 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1023 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1024 1025 if (IdxVal < LoElts) { 1026 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1027 "Extracted subvector crosses vector split!"); 1028 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1029 } else { 1030 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1031 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1032 } 1033} 1034 1035SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1036 SDValue Vec = N->getOperand(0); 1037 SDValue Idx = N->getOperand(1); 1038 EVT VecVT = Vec.getValueType(); 1039 1040 if (isa<ConstantSDNode>(Idx)) { 1041 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1042 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1043 1044 SDValue Lo, Hi; 1045 GetSplitVector(Vec, Lo, Hi); 1046 1047 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1048 1049 if (IdxVal < LoElts) 1050 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1051 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1052 DAG.getConstant(IdxVal - LoElts, 1053 Idx.getValueType())); 1054 } 1055 1056 // Store the vector to the stack. 1057 EVT EltVT = VecVT.getVectorElementType(); 1058 DebugLoc dl = N->getDebugLoc(); 1059 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1060 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1061 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1062 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1063 1064 // Load back the required element. 1065 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1066 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1067 SV, 0, EltVT); 1068} 1069 1070SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1071 assert(N->isUnindexed() && "Indexed store of vector?"); 1072 assert(OpNo == 1 && "Can only split the stored value"); 1073 DebugLoc dl = N->getDebugLoc(); 1074 1075 bool isTruncating = N->isTruncatingStore(); 1076 SDValue Ch = N->getChain(); 1077 SDValue Ptr = N->getBasePtr(); 1078 int SVOffset = N->getSrcValueOffset(); 1079 EVT MemoryVT = N->getMemoryVT(); 1080 unsigned Alignment = N->getOriginalAlignment(); 1081 bool isVol = N->isVolatile(); 1082 SDValue Lo, Hi; 1083 GetSplitVector(N->getOperand(1), Lo, Hi); 1084 1085 EVT LoMemVT, HiMemVT; 1086 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1087 1088 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1089 1090 if (isTruncating) 1091 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1092 LoMemVT, isVol, Alignment); 1093 else 1094 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1095 isVol, Alignment); 1096 1097 // Increment the pointer to the other half. 1098 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1099 DAG.getIntPtrConstant(IncrementSize)); 1100 SVOffset += IncrementSize; 1101 1102 if (isTruncating) 1103 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1104 HiMemVT, isVol, Alignment); 1105 else 1106 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1107 isVol, Alignment); 1108 1109 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1110} 1111 1112 1113//===----------------------------------------------------------------------===// 1114// Result Vector Widening 1115//===----------------------------------------------------------------------===// 1116 1117void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1118 DEBUG(errs() << "Widen node result " << ResNo << ": "; 1119 N->dump(&DAG); 1120 errs() << "\n"); 1121 1122 // See if the target wants to custom widen this node. 1123 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1124 return; 1125 1126 SDValue Res = SDValue(); 1127 switch (N->getOpcode()) { 1128 default: 1129#ifndef NDEBUG 1130 errs() << "WidenVectorResult #" << ResNo << ": "; 1131 N->dump(&DAG); 1132 errs() << "\n"; 1133#endif 1134 llvm_unreachable("Do not know how to widen the result of this operator!"); 1135 1136 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1137 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1138 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1139 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1140 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1141 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1142 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1143 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1144 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1145 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1146 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1147 case ISD::VECTOR_SHUFFLE: 1148 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1149 break; 1150 case ISD::VSETCC: 1151 Res = WidenVecRes_VSETCC(N); 1152 break; 1153 1154 case ISD::ADD: 1155 case ISD::AND: 1156 case ISD::BSWAP: 1157 case ISD::FADD: 1158 case ISD::FCOPYSIGN: 1159 case ISD::FDIV: 1160 case ISD::FMUL: 1161 case ISD::FPOW: 1162 case ISD::FPOWI: 1163 case ISD::FREM: 1164 case ISD::FSUB: 1165 case ISD::MUL: 1166 case ISD::MULHS: 1167 case ISD::MULHU: 1168 case ISD::OR: 1169 case ISD::SDIV: 1170 case ISD::SREM: 1171 case ISD::UDIV: 1172 case ISD::UREM: 1173 case ISD::SUB: 1174 case ISD::XOR: 1175 Res = WidenVecRes_Binary(N); 1176 break; 1177 1178 case ISD::SHL: 1179 case ISD::SRA: 1180 case ISD::SRL: 1181 Res = WidenVecRes_Shift(N); 1182 break; 1183 1184 case ISD::FP_ROUND: 1185 case ISD::FP_TO_SINT: 1186 case ISD::FP_TO_UINT: 1187 case ISD::SINT_TO_FP: 1188 case ISD::UINT_TO_FP: 1189 case ISD::TRUNCATE: 1190 case ISD::SIGN_EXTEND: 1191 case ISD::ZERO_EXTEND: 1192 case ISD::ANY_EXTEND: 1193 Res = WidenVecRes_Convert(N); 1194 break; 1195 1196 case ISD::CTLZ: 1197 case ISD::CTPOP: 1198 case ISD::CTTZ: 1199 case ISD::FABS: 1200 case ISD::FCOS: 1201 case ISD::FNEG: 1202 case ISD::FSIN: 1203 case ISD::FSQRT: 1204 Res = WidenVecRes_Unary(N); 1205 break; 1206 } 1207 1208 // If Res is null, the sub-method took care of registering the result. 1209 if (Res.getNode()) 1210 SetWidenedVector(SDValue(N, ResNo), Res); 1211} 1212 1213SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1214 // Binary op widening. 1215 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1216 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1217 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1218 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1219} 1220 1221SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1222 SDValue InOp = N->getOperand(0); 1223 DebugLoc dl = N->getDebugLoc(); 1224 1225 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1226 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1227 1228 EVT InVT = InOp.getValueType(); 1229 EVT InEltVT = InVT.getVectorElementType(); 1230 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1231 1232 unsigned Opcode = N->getOpcode(); 1233 unsigned InVTNumElts = InVT.getVectorNumElements(); 1234 1235 if (getTypeAction(InVT) == WidenVector) { 1236 InOp = GetWidenedVector(N->getOperand(0)); 1237 InVT = InOp.getValueType(); 1238 InVTNumElts = InVT.getVectorNumElements(); 1239 if (InVTNumElts == WidenNumElts) 1240 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1241 } 1242 1243 if (TLI.isTypeLegal(InWidenVT)) { 1244 // Because the result and the input are different vector types, widening 1245 // the result could create a legal type but widening the input might make 1246 // it an illegal type that might lead to repeatedly splitting the input 1247 // and then widening it. To avoid this, we widen the input only if 1248 // it results in a legal type. 1249 if (WidenNumElts % InVTNumElts == 0) { 1250 // Widen the input and call convert on the widened input vector. 1251 unsigned NumConcat = WidenNumElts/InVTNumElts; 1252 SmallVector<SDValue, 16> Ops(NumConcat); 1253 Ops[0] = InOp; 1254 SDValue UndefVal = DAG.getUNDEF(InVT); 1255 for (unsigned i = 1; i != NumConcat; ++i) 1256 Ops[i] = UndefVal; 1257 return DAG.getNode(Opcode, dl, WidenVT, 1258 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1259 &Ops[0], NumConcat)); 1260 } 1261 1262 if (InVTNumElts % WidenNumElts == 0) { 1263 // Extract the input and convert the shorten input vector. 1264 return DAG.getNode(Opcode, dl, WidenVT, 1265 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1266 InOp, DAG.getIntPtrConstant(0))); 1267 } 1268 } 1269 1270 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1271 SmallVector<SDValue, 16> Ops(WidenNumElts); 1272 EVT EltVT = WidenVT.getVectorElementType(); 1273 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1274 unsigned i; 1275 for (i=0; i < MinElts; ++i) 1276 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1277 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1278 DAG.getIntPtrConstant(i))); 1279 1280 SDValue UndefVal = DAG.getUNDEF(EltVT); 1281 for (; i < WidenNumElts; ++i) 1282 Ops[i] = UndefVal; 1283 1284 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1285} 1286 1287SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1288 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1289 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1290 SDValue ShOp = N->getOperand(1); 1291 1292 EVT ShVT = ShOp.getValueType(); 1293 if (getTypeAction(ShVT) == WidenVector) { 1294 ShOp = GetWidenedVector(ShOp); 1295 ShVT = ShOp.getValueType(); 1296 } 1297 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(), 1298 WidenVT.getVectorNumElements()); 1299 if (ShVT != ShWidenVT) 1300 ShOp = ModifyToType(ShOp, ShWidenVT); 1301 1302 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1303} 1304 1305SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1306 // Unary op widening. 1307 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1308 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1309 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1310} 1311 1312SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1313 SDValue InOp = N->getOperand(0); 1314 EVT InVT = InOp.getValueType(); 1315 EVT VT = N->getValueType(0); 1316 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1317 DebugLoc dl = N->getDebugLoc(); 1318 1319 switch (getTypeAction(InVT)) { 1320 default: 1321 assert(false && "Unknown type action!"); 1322 break; 1323 case Legal: 1324 break; 1325 case PromoteInteger: 1326 // If the InOp is promoted to the same size, convert it. Otherwise, 1327 // fall out of the switch and widen the promoted input. 1328 InOp = GetPromotedInteger(InOp); 1329 InVT = InOp.getValueType(); 1330 if (WidenVT.bitsEq(InVT)) 1331 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1332 break; 1333 case SoftenFloat: 1334 case ExpandInteger: 1335 case ExpandFloat: 1336 case ScalarizeVector: 1337 case SplitVector: 1338 break; 1339 case WidenVector: 1340 // If the InOp is widened to the same size, convert it. Otherwise, fall 1341 // out of the switch and widen the widened input. 1342 InOp = GetWidenedVector(InOp); 1343 InVT = InOp.getValueType(); 1344 if (WidenVT.bitsEq(InVT)) 1345 // The input widens to the same size. Convert to the widen value. 1346 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1347 break; 1348 } 1349 1350 unsigned WidenSize = WidenVT.getSizeInBits(); 1351 unsigned InSize = InVT.getSizeInBits(); 1352 if (WidenSize % InSize == 0) { 1353 // Determine new input vector type. The new input vector type will use 1354 // the same element type (if its a vector) or use the input type as a 1355 // vector. It is the same size as the type to widen to. 1356 EVT NewInVT; 1357 unsigned NewNumElts = WidenSize / InSize; 1358 if (InVT.isVector()) { 1359 EVT InEltVT = InVT.getVectorElementType(); 1360 NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits()); 1361 } else { 1362 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1363 } 1364 1365 if (TLI.isTypeLegal(NewInVT)) { 1366 // Because the result and the input are different vector types, widening 1367 // the result could create a legal type but widening the input might make 1368 // it an illegal type that might lead to repeatedly splitting the input 1369 // and then widening it. To avoid this, we widen the input only if 1370 // it results in a legal type. 1371 SmallVector<SDValue, 16> Ops(NewNumElts); 1372 SDValue UndefVal = DAG.getUNDEF(InVT); 1373 Ops[0] = InOp; 1374 for (unsigned i = 1; i < NewNumElts; ++i) 1375 Ops[i] = UndefVal; 1376 1377 SDValue NewVec; 1378 if (InVT.isVector()) 1379 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1380 NewInVT, &Ops[0], NewNumElts); 1381 else 1382 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1383 NewInVT, &Ops[0], NewNumElts); 1384 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1385 } 1386 } 1387 1388 return CreateStackStoreLoad(InOp, WidenVT); 1389} 1390 1391SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1392 DebugLoc dl = N->getDebugLoc(); 1393 // Build a vector with undefined for the new nodes. 1394 EVT VT = N->getValueType(0); 1395 EVT EltVT = VT.getVectorElementType(); 1396 unsigned NumElts = VT.getVectorNumElements(); 1397 1398 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1399 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1400 1401 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1402 NewOps.reserve(WidenNumElts); 1403 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1404 NewOps.push_back(DAG.getUNDEF(EltVT)); 1405 1406 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1407} 1408 1409SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1410 EVT InVT = N->getOperand(0).getValueType(); 1411 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1412 DebugLoc dl = N->getDebugLoc(); 1413 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1414 unsigned NumOperands = N->getNumOperands(); 1415 1416 bool InputWidened = false; // Indicates we need to widen the input. 1417 if (getTypeAction(InVT) != WidenVector) { 1418 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1419 // Add undef vectors to widen to correct length. 1420 unsigned NumConcat = WidenVT.getVectorNumElements() / 1421 InVT.getVectorNumElements(); 1422 SDValue UndefVal = DAG.getUNDEF(InVT); 1423 SmallVector<SDValue, 16> Ops(NumConcat); 1424 for (unsigned i=0; i < NumOperands; ++i) 1425 Ops[i] = N->getOperand(i); 1426 for (unsigned i = NumOperands; i != NumConcat; ++i) 1427 Ops[i] = UndefVal; 1428 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1429 } 1430 } else { 1431 InputWidened = true; 1432 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1433 // The inputs and the result are widen to the same value. 1434 unsigned i; 1435 for (i=1; i < NumOperands; ++i) 1436 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1437 break; 1438 1439 if (i > NumOperands) 1440 // Everything but the first operand is an UNDEF so just return the 1441 // widened first operand. 1442 return GetWidenedVector(N->getOperand(0)); 1443 1444 if (NumOperands == 2) { 1445 // Replace concat of two operands with a shuffle. 1446 SmallVector<int, 16> MaskOps(WidenNumElts); 1447 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1448 MaskOps[i] = i; 1449 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1450 } 1451 return DAG.getVectorShuffle(WidenVT, dl, 1452 GetWidenedVector(N->getOperand(0)), 1453 GetWidenedVector(N->getOperand(1)), 1454 &MaskOps[0]); 1455 } 1456 } 1457 } 1458 1459 // Fall back to use extracts and build vector. 1460 EVT EltVT = WidenVT.getVectorElementType(); 1461 unsigned NumInElts = InVT.getVectorNumElements(); 1462 SmallVector<SDValue, 16> Ops(WidenNumElts); 1463 unsigned Idx = 0; 1464 for (unsigned i=0; i < NumOperands; ++i) { 1465 SDValue InOp = N->getOperand(i); 1466 if (InputWidened) 1467 InOp = GetWidenedVector(InOp); 1468 for (unsigned j=0; j < NumInElts; ++j) 1469 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1470 DAG.getIntPtrConstant(j)); 1471 } 1472 SDValue UndefVal = DAG.getUNDEF(EltVT); 1473 for (; Idx < WidenNumElts; ++Idx) 1474 Ops[Idx] = UndefVal; 1475 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1476} 1477 1478SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1479 DebugLoc dl = N->getDebugLoc(); 1480 SDValue InOp = N->getOperand(0); 1481 SDValue RndOp = N->getOperand(3); 1482 SDValue SatOp = N->getOperand(4); 1483 1484 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1485 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1486 1487 EVT InVT = InOp.getValueType(); 1488 EVT InEltVT = InVT.getVectorElementType(); 1489 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1490 1491 SDValue DTyOp = DAG.getValueType(WidenVT); 1492 SDValue STyOp = DAG.getValueType(InWidenVT); 1493 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1494 1495 unsigned InVTNumElts = InVT.getVectorNumElements(); 1496 if (getTypeAction(InVT) == WidenVector) { 1497 InOp = GetWidenedVector(InOp); 1498 InVT = InOp.getValueType(); 1499 InVTNumElts = InVT.getVectorNumElements(); 1500 if (InVTNumElts == WidenNumElts) 1501 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1502 SatOp, CvtCode); 1503 } 1504 1505 if (TLI.isTypeLegal(InWidenVT)) { 1506 // Because the result and the input are different vector types, widening 1507 // the result could create a legal type but widening the input might make 1508 // it an illegal type that might lead to repeatedly splitting the input 1509 // and then widening it. To avoid this, we widen the input only if 1510 // it results in a legal type. 1511 if (WidenNumElts % InVTNumElts == 0) { 1512 // Widen the input and call convert on the widened input vector. 1513 unsigned NumConcat = WidenNumElts/InVTNumElts; 1514 SmallVector<SDValue, 16> Ops(NumConcat); 1515 Ops[0] = InOp; 1516 SDValue UndefVal = DAG.getUNDEF(InVT); 1517 for (unsigned i = 1; i != NumConcat; ++i) { 1518 Ops[i] = UndefVal; 1519 } 1520 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1521 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1522 SatOp, CvtCode); 1523 } 1524 1525 if (InVTNumElts % WidenNumElts == 0) { 1526 // Extract the input and convert the shorten input vector. 1527 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1528 DAG.getIntPtrConstant(0)); 1529 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1530 SatOp, CvtCode); 1531 } 1532 } 1533 1534 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1535 SmallVector<SDValue, 16> Ops(WidenNumElts); 1536 EVT EltVT = WidenVT.getVectorElementType(); 1537 DTyOp = DAG.getValueType(EltVT); 1538 STyOp = DAG.getValueType(InEltVT); 1539 1540 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1541 unsigned i; 1542 for (i=0; i < MinElts; ++i) { 1543 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1544 DAG.getIntPtrConstant(i)); 1545 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1546 SatOp, CvtCode); 1547 } 1548 1549 SDValue UndefVal = DAG.getUNDEF(EltVT); 1550 for (; i < WidenNumElts; ++i) 1551 Ops[i] = UndefVal; 1552 1553 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1554} 1555 1556SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1557 EVT VT = N->getValueType(0); 1558 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1559 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1560 SDValue InOp = N->getOperand(0); 1561 SDValue Idx = N->getOperand(1); 1562 DebugLoc dl = N->getDebugLoc(); 1563 1564 if (getTypeAction(InOp.getValueType()) == WidenVector) 1565 InOp = GetWidenedVector(InOp); 1566 1567 EVT InVT = InOp.getValueType(); 1568 1569 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1570 if (CIdx) { 1571 unsigned IdxVal = CIdx->getZExtValue(); 1572 // Check if we can just return the input vector after widening. 1573 if (IdxVal == 0 && InVT == WidenVT) 1574 return InOp; 1575 1576 // Check if we can extract from the vector. 1577 unsigned InNumElts = InVT.getVectorNumElements(); 1578 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1579 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1580 } 1581 1582 // We could try widening the input to the right length but for now, extract 1583 // the original elements, fill the rest with undefs and build a vector. 1584 SmallVector<SDValue, 16> Ops(WidenNumElts); 1585 EVT EltVT = VT.getVectorElementType(); 1586 EVT IdxVT = Idx.getValueType(); 1587 unsigned NumElts = VT.getVectorNumElements(); 1588 unsigned i; 1589 if (CIdx) { 1590 unsigned IdxVal = CIdx->getZExtValue(); 1591 for (i=0; i < NumElts; ++i) 1592 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1593 DAG.getConstant(IdxVal+i, IdxVT)); 1594 } else { 1595 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1596 for (i=1; i < NumElts; ++i) { 1597 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1598 DAG.getConstant(i, IdxVT)); 1599 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1600 } 1601 } 1602 1603 SDValue UndefVal = DAG.getUNDEF(EltVT); 1604 for (; i < WidenNumElts; ++i) 1605 Ops[i] = UndefVal; 1606 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1607} 1608 1609SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1610 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1611 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1612 InOp.getValueType(), InOp, 1613 N->getOperand(1), N->getOperand(2)); 1614} 1615 1616SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1617 LoadSDNode *LD = cast<LoadSDNode>(N); 1618 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1619 EVT LdVT = LD->getMemoryVT(); 1620 DebugLoc dl = N->getDebugLoc(); 1621 assert(LdVT.isVector() && WidenVT.isVector()); 1622 1623 // Load information 1624 SDValue Chain = LD->getChain(); 1625 SDValue BasePtr = LD->getBasePtr(); 1626 int SVOffset = LD->getSrcValueOffset(); 1627 unsigned Align = LD->getAlignment(); 1628 bool isVolatile = LD->isVolatile(); 1629 const Value *SV = LD->getSrcValue(); 1630 ISD::LoadExtType ExtType = LD->getExtensionType(); 1631 1632 SDValue Result; 1633 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1634 if (ExtType != ISD::NON_EXTLOAD) { 1635 // For extension loads, we can not play the tricks of chopping legal 1636 // vector types and bit cast it to the right type. Instead, we unroll 1637 // the load and build a vector. 1638 EVT EltVT = WidenVT.getVectorElementType(); 1639 EVT LdEltVT = LdVT.getVectorElementType(); 1640 unsigned NumElts = LdVT.getVectorNumElements(); 1641 1642 // Load each element and widen 1643 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1644 SmallVector<SDValue, 16> Ops(WidenNumElts); 1645 unsigned Increment = LdEltVT.getSizeInBits() / 8; 1646 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 1647 LdEltVT, isVolatile, Align); 1648 LdChain.push_back(Ops[0].getValue(1)); 1649 unsigned i = 0, Offset = Increment; 1650 for (i=1; i < NumElts; ++i, Offset += Increment) { 1651 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1652 BasePtr, DAG.getIntPtrConstant(Offset)); 1653 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 1654 SVOffset + Offset, LdEltVT, isVolatile, Align); 1655 LdChain.push_back(Ops[i].getValue(1)); 1656 } 1657 1658 // Fill the rest with undefs 1659 SDValue UndefVal = DAG.getUNDEF(EltVT); 1660 for (; i != WidenNumElts; ++i) 1661 Ops[i] = UndefVal; 1662 1663 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 1664 } else { 1665 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 1666 unsigned int LdWidth = LdVT.getSizeInBits(); 1667 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 1668 Align, isVolatile, LdWidth, WidenVT, dl); 1669 } 1670 1671 // If we generate a single load, we can use that for the chain. Otherwise, 1672 // build a factor node to remember the multiple loads are independent and 1673 // chain to that. 1674 SDValue NewChain; 1675 if (LdChain.size() == 1) 1676 NewChain = LdChain[0]; 1677 else 1678 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 1679 LdChain.size()); 1680 1681 // Modified the chain - switch anything that used the old chain to use 1682 // the new one. 1683 ReplaceValueWith(SDValue(N, 1), NewChain); 1684 1685 return Result; 1686} 1687 1688SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1689 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1690 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1691 WidenVT, N->getOperand(0)); 1692} 1693 1694SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1695 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1696 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1697 1698 SDValue Cond1 = N->getOperand(0); 1699 EVT CondVT = Cond1.getValueType(); 1700 if (CondVT.isVector()) { 1701 EVT CondEltVT = CondVT.getVectorElementType(); 1702 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts); 1703 if (getTypeAction(CondVT) == WidenVector) 1704 Cond1 = GetWidenedVector(Cond1); 1705 1706 if (Cond1.getValueType() != CondWidenVT) 1707 Cond1 = ModifyToType(Cond1, CondWidenVT); 1708 } 1709 1710 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1711 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1712 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1713 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1714 WidenVT, Cond1, InOp1, InOp2); 1715} 1716 1717SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1718 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1719 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1720 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1721 InOp1.getValueType(), N->getOperand(0), 1722 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1723} 1724 1725SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1726 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1727 return DAG.getUNDEF(WidenVT); 1728} 1729 1730SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1731 EVT VT = N->getValueType(0); 1732 DebugLoc dl = N->getDebugLoc(); 1733 1734 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1735 unsigned NumElts = VT.getVectorNumElements(); 1736 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1737 1738 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1739 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1740 1741 // Adjust mask based on new input vector length. 1742 SmallVector<int, 16> NewMask; 1743 for (unsigned i = 0; i != NumElts; ++i) { 1744 int Idx = N->getMaskElt(i); 1745 if (Idx < (int)NumElts) 1746 NewMask.push_back(Idx); 1747 else 1748 NewMask.push_back(Idx - NumElts + WidenNumElts); 1749 } 1750 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1751 NewMask.push_back(-1); 1752 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1753} 1754 1755SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1756 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1757 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1758 1759 SDValue InOp1 = N->getOperand(0); 1760 EVT InVT = InOp1.getValueType(); 1761 assert(InVT.isVector() && "can not widen non vector type"); 1762 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); 1763 InOp1 = GetWidenedVector(InOp1); 1764 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1765 1766 // Assume that the input and output will be widen appropriately. If not, 1767 // we will have to unroll it at some point. 1768 assert(InOp1.getValueType() == WidenInVT && 1769 InOp2.getValueType() == WidenInVT && 1770 "Input not widened to expected type!"); 1771 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1772 WidenVT, InOp1, InOp2, N->getOperand(2)); 1773} 1774 1775 1776//===----------------------------------------------------------------------===// 1777// Widen Vector Operand 1778//===----------------------------------------------------------------------===// 1779bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1780 DEBUG(errs() << "Widen node operand " << ResNo << ": "; 1781 N->dump(&DAG); 1782 errs() << "\n"); 1783 SDValue Res = SDValue(); 1784 1785 switch (N->getOpcode()) { 1786 default: 1787#ifndef NDEBUG 1788 errs() << "WidenVectorOperand op #" << ResNo << ": "; 1789 N->dump(&DAG); 1790 errs() << "\n"; 1791#endif 1792 llvm_unreachable("Do not know how to widen this operator's operand!"); 1793 1794 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1795 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1796 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 1797 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1798 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1799 1800 case ISD::FP_ROUND: 1801 case ISD::FP_TO_SINT: 1802 case ISD::FP_TO_UINT: 1803 case ISD::SINT_TO_FP: 1804 case ISD::UINT_TO_FP: 1805 case ISD::TRUNCATE: 1806 case ISD::SIGN_EXTEND: 1807 case ISD::ZERO_EXTEND: 1808 case ISD::ANY_EXTEND: 1809 Res = WidenVecOp_Convert(N); 1810 break; 1811 } 1812 1813 // If Res is null, the sub-method took care of registering the result. 1814 if (!Res.getNode()) return false; 1815 1816 // If the result is N, the sub-method updated N in place. Tell the legalizer 1817 // core about this. 1818 if (Res.getNode() == N) 1819 return true; 1820 1821 1822 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1823 "Invalid operand expansion"); 1824 1825 ReplaceValueWith(SDValue(N, 0), Res); 1826 return false; 1827} 1828 1829SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1830 // Since the result is legal and the input is illegal, it is unlikely 1831 // that we can fix the input to a legal type so unroll the convert 1832 // into some scalar code and create a nasty build vector. 1833 EVT VT = N->getValueType(0); 1834 EVT EltVT = VT.getVectorElementType(); 1835 DebugLoc dl = N->getDebugLoc(); 1836 unsigned NumElts = VT.getVectorNumElements(); 1837 SDValue InOp = N->getOperand(0); 1838 if (getTypeAction(InOp.getValueType()) == WidenVector) 1839 InOp = GetWidenedVector(InOp); 1840 EVT InVT = InOp.getValueType(); 1841 EVT InEltVT = InVT.getVectorElementType(); 1842 1843 unsigned Opcode = N->getOpcode(); 1844 SmallVector<SDValue, 16> Ops(NumElts); 1845 for (unsigned i=0; i < NumElts; ++i) 1846 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1847 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1848 DAG.getIntPtrConstant(i))); 1849 1850 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1851} 1852 1853SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1854 EVT VT = N->getValueType(0); 1855 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1856 EVT InWidenVT = InOp.getValueType(); 1857 DebugLoc dl = N->getDebugLoc(); 1858 1859 // Check if we can convert between two legal vector types and extract. 1860 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1861 unsigned Size = VT.getSizeInBits(); 1862 if (InWidenSize % Size == 0 && !VT.isVector()) { 1863 unsigned NewNumElts = InWidenSize / Size; 1864 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 1865 if (TLI.isTypeLegal(NewVT)) { 1866 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1867 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1868 DAG.getIntPtrConstant(0)); 1869 } 1870 } 1871 1872 return CreateStackStoreLoad(InOp, VT); 1873} 1874 1875SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1876 // If the input vector is not legal, it is likely that we will not find a 1877 // legal vector of the same size. Replace the concatenate vector with a 1878 // nasty build vector. 1879 EVT VT = N->getValueType(0); 1880 EVT EltVT = VT.getVectorElementType(); 1881 DebugLoc dl = N->getDebugLoc(); 1882 unsigned NumElts = VT.getVectorNumElements(); 1883 SmallVector<SDValue, 16> Ops(NumElts); 1884 1885 EVT InVT = N->getOperand(0).getValueType(); 1886 unsigned NumInElts = InVT.getVectorNumElements(); 1887 1888 unsigned Idx = 0; 1889 unsigned NumOperands = N->getNumOperands(); 1890 for (unsigned i=0; i < NumOperands; ++i) { 1891 SDValue InOp = N->getOperand(i); 1892 if (getTypeAction(InOp.getValueType()) == WidenVector) 1893 InOp = GetWidenedVector(InOp); 1894 for (unsigned j=0; j < NumInElts; ++j) 1895 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1896 DAG.getIntPtrConstant(j)); 1897 } 1898 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1899} 1900 1901SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1902 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1903 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), 1904 N->getValueType(0), InOp, N->getOperand(1)); 1905} 1906 1907SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1908 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1909 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1910 N->getValueType(0), InOp, N->getOperand(1)); 1911} 1912 1913SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1914 // We have to widen the value but we want only to store the original 1915 // vector type. 1916 StoreSDNode *ST = cast<StoreSDNode>(N); 1917 SDValue Chain = ST->getChain(); 1918 SDValue BasePtr = ST->getBasePtr(); 1919 const Value *SV = ST->getSrcValue(); 1920 int SVOffset = ST->getSrcValueOffset(); 1921 unsigned Align = ST->getAlignment(); 1922 bool isVolatile = ST->isVolatile(); 1923 SDValue ValOp = GetWidenedVector(ST->getValue()); 1924 DebugLoc dl = N->getDebugLoc(); 1925 1926 EVT StVT = ST->getMemoryVT(); 1927 EVT ValVT = ValOp.getValueType(); 1928 // It must be true that we the widen vector type is bigger than where 1929 // we need to store. 1930 assert(StVT.isVector() && ValOp.getValueType().isVector()); 1931 assert(StVT.bitsLT(ValOp.getValueType())); 1932 1933 SmallVector<SDValue, 16> StChain; 1934 if (ST->isTruncatingStore()) { 1935 // For truncating stores, we can not play the tricks of chopping legal 1936 // vector types and bit cast it to the right type. Instead, we unroll 1937 // the store. 1938 EVT StEltVT = StVT.getVectorElementType(); 1939 EVT ValEltVT = ValVT.getVectorElementType(); 1940 unsigned Increment = ValEltVT.getSizeInBits() / 8; 1941 unsigned NumElts = StVT.getVectorNumElements(); 1942 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1943 DAG.getIntPtrConstant(0)); 1944 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 1945 SVOffset, StEltVT, 1946 isVolatile, Align)); 1947 unsigned Offset = Increment; 1948 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 1949 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1950 BasePtr, DAG.getIntPtrConstant(Offset)); 1951 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1952 DAG.getIntPtrConstant(0)); 1953 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 1954 SVOffset + Offset, StEltVT, 1955 isVolatile, MinAlign(Align, Offset))); 1956 } 1957 } 1958 else { 1959 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 1960 // Store value 1961 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 1962 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 1963 } 1964 if (StChain.size() == 1) 1965 return StChain[0]; 1966 else 1967 return DAG.getNode(ISD::TokenFactor, dl, 1968 MVT::Other,&StChain[0],StChain.size()); 1969} 1970 1971//===----------------------------------------------------------------------===// 1972// Vector Widening Utilities 1973//===----------------------------------------------------------------------===// 1974 1975 1976// Utility function to find a vector type and its associated element 1977// type from a preferred width and whose vector type must be the same size 1978// as the VecVT. 1979// TLI: Target lowering used to determine legal types. 1980// Width: Preferred width to store. 1981// VecVT: Vector value type whose size we must match. 1982// Returns NewVecVT and NewEltVT - the vector type and its associated 1983// element type. 1984static void FindAssocWidenVecType(SelectionDAG& DAG, 1985 const TargetLowering &TLI, unsigned Width, 1986 EVT VecVT, 1987 EVT& NewEltVT, EVT& NewVecVT) { 1988 unsigned EltWidth = Width + 1; 1989 if (TLI.isTypeLegal(VecVT)) { 1990 // We start with the preferred with, making it a power of 2 and find a 1991 // legal vector type of that width. If not, we reduce it by another of 2. 1992 // For incoming type is legal, this process will end as a vector of the 1993 // smallest loadable type should always be legal. 1994 do { 1995 assert(EltWidth > 0); 1996 EltWidth = 1 << Log2_32(EltWidth - 1); 1997 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 1998 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 1999 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 2000 } while (!TLI.isTypeLegal(NewVecVT) || 2001 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2002 } else { 2003 // The incoming vector type is illegal and is the result of widening 2004 // a vector to a power of 2. In this case, we will use the preferred 2005 // with as long as it is a multiple of the incoming vector length. 2006 // The legalization process will eventually make this into a legal type 2007 // and remove the illegal bit converts (which would turn to stack converts 2008 // if they are allow to exist). 2009 do { 2010 assert(EltWidth > 0); 2011 EltWidth = 1 << Log2_32(EltWidth - 1); 2012 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 2013 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2014 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 2015 } while (!TLI.isTypeLegal(NewEltVT) || 2016 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2017 } 2018} 2019 2020SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2021 SDValue Chain, 2022 SDValue BasePtr, 2023 const Value *SV, 2024 int SVOffset, 2025 unsigned Alignment, 2026 bool isVolatile, 2027 unsigned LdWidth, 2028 EVT ResType, 2029 DebugLoc dl) { 2030 // The strategy assumes that we can efficiently load powers of two widths. 2031 // The routines chops the vector into the largest power of 2 load and 2032 // can be inserted into a legal vector and then cast the result into the 2033 // vector type we want. This avoids unnecessary stack converts. 2034 2035 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 2036 // the load is nonvolatile, we an use a wider load for the value. 2037 2038 // Find the vector type that can load from. 2039 EVT NewEltVT, NewVecVT; 2040 unsigned NewEltVTWidth; 2041 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2042 NewEltVTWidth = NewEltVT.getSizeInBits(); 2043 2044 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 2045 isVolatile, Alignment); 2046 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2047 LdChain.push_back(LdOp.getValue(1)); 2048 2049 // Check if we can load the element with one instruction 2050 if (LdWidth == NewEltVTWidth) { 2051 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2052 } 2053 2054 unsigned Idx = 1; 2055 LdWidth -= NewEltVTWidth; 2056 unsigned Offset = 0; 2057 2058 while (LdWidth > 0) { 2059 unsigned Increment = NewEltVTWidth / 8; 2060 Offset += Increment; 2061 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2062 DAG.getIntPtrConstant(Increment)); 2063 2064 if (LdWidth < NewEltVTWidth) { 2065 // Our current type we are using is too large, use a smaller size by 2066 // using a smaller power of 2 2067 unsigned oNewEltVTWidth = NewEltVTWidth; 2068 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2069 NewEltVTWidth = NewEltVT.getSizeInBits(); 2070 // Readjust position and vector position based on new load type 2071 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2072 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2073 } 2074 2075 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 2076 SVOffset+Offset, isVolatile, 2077 MinAlign(Alignment, Offset)); 2078 LdChain.push_back(LdOp.getValue(1)); 2079 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 2080 DAG.getIntPtrConstant(Idx++)); 2081 2082 LdWidth -= NewEltVTWidth; 2083 } 2084 2085 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2086} 2087 2088void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2089 SDValue Chain, 2090 SDValue BasePtr, 2091 const Value *SV, 2092 int SVOffset, 2093 unsigned Alignment, 2094 bool isVolatile, 2095 SDValue ValOp, 2096 unsigned StWidth, 2097 DebugLoc dl) { 2098 // Breaks the stores into a series of power of 2 width stores. For any 2099 // width, we convert the vector to the vector of element size that we 2100 // want to store. This avoids requiring a stack convert. 2101 2102 // Find a width of the element type we can store with 2103 EVT WidenVT = ValOp.getValueType(); 2104 EVT NewEltVT, NewVecVT; 2105 2106 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2107 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 2108 2109 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2110 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2111 DAG.getIntPtrConstant(0)); 2112 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 2113 isVolatile, Alignment); 2114 StChain.push_back(StOp); 2115 2116 // Check if we are done 2117 if (StWidth == NewEltVTWidth) { 2118 return; 2119 } 2120 2121 unsigned Idx = 1; 2122 StWidth -= NewEltVTWidth; 2123 unsigned Offset = 0; 2124 2125 while (StWidth > 0) { 2126 unsigned Increment = NewEltVTWidth / 8; 2127 Offset += Increment; 2128 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2129 DAG.getIntPtrConstant(Increment)); 2130 2131 if (StWidth < NewEltVTWidth) { 2132 // Our current type we are using is too large, use a smaller size by 2133 // using a smaller power of 2 2134 unsigned oNewEltVTWidth = NewEltVTWidth; 2135 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2136 NewEltVTWidth = NewEltVT.getSizeInBits(); 2137 // Readjust position and vector position based on new load type 2138 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2139 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2140 } 2141 2142 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2143 DAG.getIntPtrConstant(Idx++)); 2144 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2145 SVOffset + Offset, isVolatile, 2146 MinAlign(Alignment, Offset))); 2147 StWidth -= NewEltVTWidth; 2148 } 2149} 2150 2151/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2152/// input vector must have the same element type as NVT. 2153SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2154 // Note that InOp might have been widened so it might already have 2155 // the right width or it might need be narrowed. 2156 EVT InVT = InOp.getValueType(); 2157 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2158 "input and widen element type must match"); 2159 DebugLoc dl = InOp.getDebugLoc(); 2160 2161 // Check if InOp already has the right width. 2162 if (InVT == NVT) 2163 return InOp; 2164 2165 unsigned InNumElts = InVT.getVectorNumElements(); 2166 unsigned WidenNumElts = NVT.getVectorNumElements(); 2167 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2168 unsigned NumConcat = WidenNumElts / InNumElts; 2169 SmallVector<SDValue, 16> Ops(NumConcat); 2170 SDValue UndefVal = DAG.getUNDEF(InVT); 2171 Ops[0] = InOp; 2172 for (unsigned i = 1; i != NumConcat; ++i) 2173 Ops[i] = UndefVal; 2174 2175 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2176 } 2177 2178 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2179 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2180 DAG.getIntPtrConstant(0)); 2181 2182 // Fall back to extract and build. 2183 SmallVector<SDValue, 16> Ops(WidenNumElts); 2184 EVT EltVT = NVT.getVectorElementType(); 2185 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2186 unsigned Idx; 2187 for (Idx = 0; Idx < MinNumElts; ++Idx) 2188 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2189 DAG.getIntPtrConstant(Idx)); 2190 2191 SDValue UndefVal = DAG.getUNDEF(EltVT); 2192 for ( ; Idx < WidenNumElts; ++Idx) 2193 Ops[Idx] = UndefVal; 2194 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2195} 2196