LegalizeVectorTypes.cpp revision da58ebcc2659fad293f11e4cd48f9d92b7fc594b
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in two vectors of half the size. For example, implementing 19// <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26#include "llvm/Support/ErrorHandling.h" 27#include "llvm/Support/raw_ostream.h" 28using namespace llvm; 29 30//===----------------------------------------------------------------------===// 31// Result Vector Scalarization: <1 x ty> -> ty. 32//===----------------------------------------------------------------------===// 33 34void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 35 DEBUG(errs() << "Scalarize node result " << ResNo << ": "; 36 N->dump(&DAG); 37 errs() << "\n"); 38 SDValue R = SDValue(); 39 40 switch (N->getOpcode()) { 41 default: 42#ifndef NDEBUG 43 errs() << "ScalarizeVectorResult #" << ResNo << ": "; 44 N->dump(&DAG); 45 errs() << "\n"; 46#endif 47 llvm_unreachable("Do not know how to scalarize the result of this operator!"); 48 49 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 50 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 51 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 52 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 53 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 54 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 55 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 56 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 57 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 58 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 59 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 60 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 61 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 62 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 63 64 case ISD::CTLZ: 65 case ISD::CTPOP: 66 case ISD::CTTZ: 67 case ISD::FABS: 68 case ISD::FCOS: 69 case ISD::FNEG: 70 case ISD::FP_TO_SINT: 71 case ISD::FP_TO_UINT: 72 case ISD::FSIN: 73 case ISD::FSQRT: 74 case ISD::FTRUNC: 75 case ISD::FFLOOR: 76 case ISD::FCEIL: 77 case ISD::FRINT: 78 case ISD::FNEARBYINT: 79 case ISD::UINT_TO_FP: 80 case ISD::SINT_TO_FP: 81 case ISD::TRUNCATE: 82 case ISD::SIGN_EXTEND: 83 case ISD::ZERO_EXTEND: 84 case ISD::ANY_EXTEND: 85 R = ScalarizeVecRes_UnaryOp(N); 86 break; 87 88 case ISD::ADD: 89 case ISD::AND: 90 case ISD::FADD: 91 case ISD::FDIV: 92 case ISD::FMUL: 93 case ISD::FPOW: 94 case ISD::FREM: 95 case ISD::FSUB: 96 case ISD::MUL: 97 case ISD::OR: 98 case ISD::SDIV: 99 case ISD::SREM: 100 case ISD::SUB: 101 case ISD::UDIV: 102 case ISD::UREM: 103 case ISD::XOR: 104 case ISD::SHL: 105 case ISD::SRA: 106 case ISD::SRL: 107 R = ScalarizeVecRes_BinOp(N); 108 break; 109 } 110 111 // If R is null, the sub-method took care of registering the result. 112 if (R.getNode()) 113 SetScalarizedVector(SDValue(N, ResNo), R); 114} 115 116SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 117 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 118 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 119 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 120 LHS.getValueType(), LHS, RHS); 121} 122 123SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 124 EVT NewVT = N->getValueType(0).getVectorElementType(); 125 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 126 NewVT, N->getOperand(0)); 127} 128 129SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 130 EVT NewVT = N->getValueType(0).getVectorElementType(); 131 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 132 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 133 Op0, DAG.getValueType(NewVT), 134 DAG.getValueType(Op0.getValueType()), 135 N->getOperand(3), 136 N->getOperand(4), 137 cast<CvtRndSatSDNode>(N)->getCvtCode()); 138} 139 140SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 141 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 142 N->getValueType(0).getVectorElementType(), 143 N->getOperand(0), N->getOperand(1)); 144} 145 146SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 147 SDValue Op = GetScalarizedVector(N->getOperand(0)); 148 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 149 Op.getValueType(), Op, N->getOperand(1)); 150} 151 152SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 153 // The value to insert may have a wider type than the vector element type, 154 // so be sure to truncate it to the element type if necessary. 155 SDValue Op = N->getOperand(1); 156 EVT EltVT = N->getValueType(0).getVectorElementType(); 157 if (Op.getValueType() != EltVT) 158 // FIXME: Can this happen for floating point types? 159 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 160 return Op; 161} 162 163SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 164 assert(N->isUnindexed() && "Indexed vector load?"); 165 166 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 167 N->getExtensionType(), 168 N->getValueType(0).getVectorElementType(), 169 N->getChain(), N->getBasePtr(), 170 DAG.getUNDEF(N->getBasePtr().getValueType()), 171 N->getSrcValue(), N->getSrcValueOffset(), 172 N->getMemoryVT().getVectorElementType(), 173 N->isVolatile(), N->getAlignment()); 174 175 // Legalized the chain result - switch anything that used the old chain to 176 // use the new one. 177 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 178 return Result; 179} 180 181SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 182 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 183 EVT DestVT = N->getValueType(0).getVectorElementType(); 184 SDValue Op = GetScalarizedVector(N->getOperand(0)); 185 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 186} 187 188SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 189 // If the operand is wider than the vector element type then it is implicitly 190 // truncated. Make that explicit here. 191 EVT EltVT = N->getValueType(0).getVectorElementType(); 192 SDValue InOp = N->getOperand(0); 193 if (InOp.getValueType() != EltVT) 194 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 195 return InOp; 196} 197 198SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 199 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 200 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 201 LHS.getValueType(), N->getOperand(0), LHS, 202 GetScalarizedVector(N->getOperand(2))); 203} 204 205SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 206 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 207 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 208 N->getOperand(0), N->getOperand(1), 209 LHS, GetScalarizedVector(N->getOperand(3)), 210 N->getOperand(4)); 211} 212 213SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 214 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 215 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 216 DebugLoc DL = N->getDebugLoc(); 217 218 // Turn it into a scalar SETCC. 219 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 220} 221 222SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 223 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 224} 225 226SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 227 // Figure out if the scalar is the LHS or RHS and return it. 228 SDValue Arg = N->getOperand(2).getOperand(0); 229 if (Arg.getOpcode() == ISD::UNDEF) 230 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 231 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 232 return GetScalarizedVector(N->getOperand(Op)); 233} 234 235SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 236 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 237 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 238 EVT NVT = N->getValueType(0).getVectorElementType(); 239 EVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 240 DebugLoc DL = N->getDebugLoc(); 241 242 // Turn it into a scalar SETCC. 243 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 244 245 // VSETCC always returns a sign-extended value, while SETCC may not. The 246 // SETCC result type may not match the vector element type. Correct these. 247 if (NVT.bitsLE(SVT)) { 248 // The SETCC result type is bigger than the vector element type. 249 // Ensure the SETCC result is sign-extended. 250 if (TLI.getBooleanContents() != 251 TargetLowering::ZeroOrNegativeOneBooleanContent) 252 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 253 DAG.getValueType(MVT::i1)); 254 // Truncate to the final type. 255 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 256 } 257 258 // The SETCC result type is smaller than the vector element type. 259 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 260 if (TLI.getBooleanContents() != 261 TargetLowering::ZeroOrNegativeOneBooleanContent) 262 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 263 // Sign extend to the final type. 264 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 265} 266 267 268//===----------------------------------------------------------------------===// 269// Operand Vector Scalarization <1 x ty> -> ty. 270//===----------------------------------------------------------------------===// 271 272bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 273 DEBUG(errs() << "Scalarize node operand " << OpNo << ": "; 274 N->dump(&DAG); 275 errs() << "\n"); 276 SDValue Res = SDValue(); 277 278 if (Res.getNode() == 0) { 279 switch (N->getOpcode()) { 280 default: 281#ifndef NDEBUG 282 errs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 283 N->dump(&DAG); 284 errs() << "\n"; 285#endif 286 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 287 case ISD::BIT_CONVERT: 288 Res = ScalarizeVecOp_BIT_CONVERT(N); 289 break; 290 case ISD::CONCAT_VECTORS: 291 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 292 break; 293 case ISD::EXTRACT_VECTOR_ELT: 294 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 295 break; 296 case ISD::STORE: 297 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 298 break; 299 } 300 } 301 302 // If the result is null, the sub-method took care of registering results etc. 303 if (!Res.getNode()) return false; 304 305 // If the result is N, the sub-method updated N in place. Tell the legalizer 306 // core about this. 307 if (Res.getNode() == N) 308 return true; 309 310 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 311 "Invalid operand expansion"); 312 313 ReplaceValueWith(SDValue(N, 0), Res); 314 return false; 315} 316 317/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 318/// to be scalarized, it must be <1 x ty>. Convert the element instead. 319SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 320 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 321 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 322 N->getValueType(0), Elt); 323} 324 325/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 326/// use a BUILD_VECTOR instead. 327SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 328 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 329 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 330 Ops[i] = GetScalarizedVector(N->getOperand(i)); 331 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 332 &Ops[0], Ops.size()); 333} 334 335/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 336/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 337/// index. 338SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 339 SDValue Res = GetScalarizedVector(N->getOperand(0)); 340 if (Res.getValueType() != N->getValueType(0)) 341 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 342 Res); 343 return Res; 344} 345 346/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 347/// scalarized, it must be <1 x ty>. Just store the element. 348SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 349 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 350 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 351 DebugLoc dl = N->getDebugLoc(); 352 353 if (N->isTruncatingStore()) 354 return DAG.getTruncStore(N->getChain(), dl, 355 GetScalarizedVector(N->getOperand(1)), 356 N->getBasePtr(), 357 N->getSrcValue(), N->getSrcValueOffset(), 358 N->getMemoryVT().getVectorElementType(), 359 N->isVolatile(), N->getAlignment()); 360 361 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 362 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 363 N->isVolatile(), N->getAlignment()); 364} 365 366 367//===----------------------------------------------------------------------===// 368// Result Vector Splitting 369//===----------------------------------------------------------------------===// 370 371/// SplitVectorResult - This method is called when the specified result of the 372/// specified node is found to need vector splitting. At this point, the node 373/// may also have invalid operands or may have other results that need 374/// legalization, we just know that (at least) one result needs vector 375/// splitting. 376void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 377 DEBUG(errs() << "Split node result: "; 378 N->dump(&DAG); 379 errs() << "\n"); 380 SDValue Lo, Hi; 381 382 switch (N->getOpcode()) { 383 default: 384#ifndef NDEBUG 385 errs() << "SplitVectorResult #" << ResNo << ": "; 386 N->dump(&DAG); 387 errs() << "\n"; 388#endif 389 llvm_unreachable("Do not know how to split the result of this operator!"); 390 391 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 392 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 393 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 394 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 395 396 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 397 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 398 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 399 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 400 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 401 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 402 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 403 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 404 case ISD::LOAD: 405 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 406 break; 407 case ISD::SETCC: 408 case ISD::VSETCC: 409 SplitVecRes_SETCC(N, Lo, Hi); 410 break; 411 case ISD::VECTOR_SHUFFLE: 412 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 413 break; 414 415 case ISD::CTTZ: 416 case ISD::CTLZ: 417 case ISD::CTPOP: 418 case ISD::FNEG: 419 case ISD::FABS: 420 case ISD::FSQRT: 421 case ISD::FSIN: 422 case ISD::FCOS: 423 case ISD::FTRUNC: 424 case ISD::FFLOOR: 425 case ISD::FCEIL: 426 case ISD::FRINT: 427 case ISD::FNEARBYINT: 428 case ISD::FP_TO_SINT: 429 case ISD::FP_TO_UINT: 430 case ISD::SINT_TO_FP: 431 case ISD::UINT_TO_FP: 432 case ISD::TRUNCATE: 433 case ISD::SIGN_EXTEND: 434 case ISD::ZERO_EXTEND: 435 case ISD::ANY_EXTEND: 436 SplitVecRes_UnaryOp(N, Lo, Hi); 437 break; 438 439 case ISD::ADD: 440 case ISD::SUB: 441 case ISD::MUL: 442 case ISD::FADD: 443 case ISD::FSUB: 444 case ISD::FMUL: 445 case ISD::SDIV: 446 case ISD::UDIV: 447 case ISD::FDIV: 448 case ISD::FPOW: 449 case ISD::AND: 450 case ISD::OR: 451 case ISD::XOR: 452 case ISD::SHL: 453 case ISD::SRA: 454 case ISD::SRL: 455 case ISD::UREM: 456 case ISD::SREM: 457 case ISD::FREM: 458 SplitVecRes_BinOp(N, Lo, Hi); 459 break; 460 } 461 462 // If Lo/Hi is null, the sub-method took care of registering results etc. 463 if (Lo.getNode()) 464 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 465} 466 467void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 468 SDValue &Hi) { 469 SDValue LHSLo, LHSHi; 470 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 471 SDValue RHSLo, RHSHi; 472 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 473 DebugLoc dl = N->getDebugLoc(); 474 475 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 476 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 477} 478 479void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 480 SDValue &Hi) { 481 // We know the result is a vector. The input may be either a vector or a 482 // scalar value. 483 EVT LoVT, HiVT; 484 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 485 DebugLoc dl = N->getDebugLoc(); 486 487 SDValue InOp = N->getOperand(0); 488 EVT InVT = InOp.getValueType(); 489 490 // Handle some special cases efficiently. 491 switch (getTypeAction(InVT)) { 492 default: 493 assert(false && "Unknown type action!"); 494 case Legal: 495 case PromoteInteger: 496 case SoftenFloat: 497 case ScalarizeVector: 498 break; 499 case ExpandInteger: 500 case ExpandFloat: 501 // A scalar to vector conversion, where the scalar needs expansion. 502 // If the vector is being split in two then we can just convert the 503 // expanded pieces. 504 if (LoVT == HiVT) { 505 GetExpandedOp(InOp, Lo, Hi); 506 if (TLI.isBigEndian()) 507 std::swap(Lo, Hi); 508 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 509 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 510 return; 511 } 512 break; 513 case SplitVector: 514 // If the input is a vector that needs to be split, convert each split 515 // piece of the input now. 516 GetSplitVector(InOp, Lo, Hi); 517 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 518 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 519 return; 520 } 521 522 // In the general case, convert the input to an integer and split it by hand. 523 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 524 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 525 if (TLI.isBigEndian()) 526 std::swap(LoIntVT, HiIntVT); 527 528 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 529 530 if (TLI.isBigEndian()) 531 std::swap(Lo, Hi); 532 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 533 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 534} 535 536void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 537 SDValue &Hi) { 538 EVT LoVT, HiVT; 539 DebugLoc dl = N->getDebugLoc(); 540 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 541 unsigned LoNumElts = LoVT.getVectorNumElements(); 542 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 543 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 544 545 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 546 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 547} 548 549void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 550 SDValue &Hi) { 551 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 552 DebugLoc dl = N->getDebugLoc(); 553 unsigned NumSubvectors = N->getNumOperands() / 2; 554 if (NumSubvectors == 1) { 555 Lo = N->getOperand(0); 556 Hi = N->getOperand(1); 557 return; 558 } 559 560 EVT LoVT, HiVT; 561 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 562 563 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 564 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 565 566 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 567 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 568} 569 570void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 571 SDValue &Hi) { 572 EVT LoVT, HiVT; 573 DebugLoc dl = N->getDebugLoc(); 574 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 575 576 SDValue DTyOpLo = DAG.getValueType(LoVT); 577 SDValue DTyOpHi = DAG.getValueType(HiVT); 578 579 SDValue RndOp = N->getOperand(3); 580 SDValue SatOp = N->getOperand(4); 581 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 582 583 // Split the input. 584 SDValue VLo, VHi; 585 EVT InVT = N->getOperand(0).getValueType(); 586 switch (getTypeAction(InVT)) { 587 default: llvm_unreachable("Unexpected type action!"); 588 case Legal: { 589 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 590 LoVT.getVectorNumElements()); 591 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 592 DAG.getIntPtrConstant(0)); 593 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 594 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 595 break; 596 } 597 case SplitVector: 598 GetSplitVector(N->getOperand(0), VLo, VHi); 599 break; 600 case WidenVector: { 601 // If the result needs to be split and the input needs to be widened, 602 // the two types must have different lengths. Use the widened result 603 // and extract from it to do the split. 604 SDValue InOp = GetWidenedVector(N->getOperand(0)); 605 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 606 LoVT.getVectorNumElements()); 607 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 608 DAG.getIntPtrConstant(0)); 609 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 610 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 611 break; 612 } 613 } 614 615 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 616 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 617 618 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 619 CvtCode); 620 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 621 CvtCode); 622} 623 624void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 625 SDValue &Hi) { 626 SDValue Vec = N->getOperand(0); 627 SDValue Idx = N->getOperand(1); 628 EVT IdxVT = Idx.getValueType(); 629 DebugLoc dl = N->getDebugLoc(); 630 631 EVT LoVT, HiVT; 632 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 633 634 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 635 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 636 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 637 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 638} 639 640void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 641 SDValue &Hi) { 642 DebugLoc dl = N->getDebugLoc(); 643 GetSplitVector(N->getOperand(0), Lo, Hi); 644 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 645 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 646} 647 648void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 649 SDValue &Hi) { 650 SDValue Vec = N->getOperand(0); 651 SDValue Elt = N->getOperand(1); 652 SDValue Idx = N->getOperand(2); 653 DebugLoc dl = N->getDebugLoc(); 654 GetSplitVector(Vec, Lo, Hi); 655 656 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 657 unsigned IdxVal = CIdx->getZExtValue(); 658 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 659 if (IdxVal < LoNumElts) 660 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 661 Lo.getValueType(), Lo, Elt, Idx); 662 else 663 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 664 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 665 return; 666 } 667 668 // Spill the vector to the stack. 669 EVT VecVT = Vec.getValueType(); 670 EVT EltVT = VecVT.getVectorElementType(); 671 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 672 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 673 674 // Store the new element. This may be larger than the vector element type, 675 // so use a truncating store. 676 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 677 unsigned Alignment = 678 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext())); 679 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 680 681 // Load the Lo part from the stack slot. 682 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 683 684 // Increment the pointer to the other part. 685 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 686 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 687 DAG.getIntPtrConstant(IncrementSize)); 688 689 // Load the Hi part from the stack slot. 690 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 691 MinAlign(Alignment, IncrementSize)); 692} 693 694void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 695 SDValue &Hi) { 696 EVT LoVT, HiVT; 697 DebugLoc dl = N->getDebugLoc(); 698 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 699 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 700 Hi = DAG.getUNDEF(HiVT); 701} 702 703void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 704 SDValue &Hi) { 705 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 706 EVT LoVT, HiVT; 707 DebugLoc dl = LD->getDebugLoc(); 708 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 709 710 ISD::LoadExtType ExtType = LD->getExtensionType(); 711 SDValue Ch = LD->getChain(); 712 SDValue Ptr = LD->getBasePtr(); 713 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 714 const Value *SV = LD->getSrcValue(); 715 int SVOffset = LD->getSrcValueOffset(); 716 EVT MemoryVT = LD->getMemoryVT(); 717 unsigned Alignment = LD->getAlignment(); 718 bool isVolatile = LD->isVolatile(); 719 720 EVT LoMemVT, HiMemVT; 721 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 722 723 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 724 SV, SVOffset, LoMemVT, isVolatile, Alignment); 725 726 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 727 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 728 DAG.getIntPtrConstant(IncrementSize)); 729 SVOffset += IncrementSize; 730 Alignment = MinAlign(Alignment, IncrementSize); 731 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 732 SV, SVOffset, HiMemVT, isVolatile, Alignment); 733 734 // Build a factor node to remember that this load is independent of the 735 // other one. 736 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 737 Hi.getValue(1)); 738 739 // Legalized the chain result - switch anything that used the old chain to 740 // use the new one. 741 ReplaceValueWith(SDValue(LD, 1), Ch); 742} 743 744void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 745 EVT LoVT, HiVT; 746 DebugLoc DL = N->getDebugLoc(); 747 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 748 749 // Split the input. 750 EVT InVT = N->getOperand(0).getValueType(); 751 SDValue LL, LH, RL, RH; 752 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 753 LoVT.getVectorNumElements()); 754 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 755 DAG.getIntPtrConstant(0)); 756 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 757 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 758 759 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 760 DAG.getIntPtrConstant(0)); 761 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 762 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 763 764 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 765 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 766} 767 768void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 769 SDValue &Hi) { 770 // Get the dest types - they may not match the input types, e.g. int_to_fp. 771 EVT LoVT, HiVT; 772 DebugLoc dl = N->getDebugLoc(); 773 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 774 775 // Split the input. 776 EVT InVT = N->getOperand(0).getValueType(); 777 switch (getTypeAction(InVT)) { 778 default: llvm_unreachable("Unexpected type action!"); 779 case Legal: { 780 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 781 LoVT.getVectorNumElements()); 782 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 783 DAG.getIntPtrConstant(0)); 784 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 785 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 786 break; 787 } 788 case SplitVector: 789 GetSplitVector(N->getOperand(0), Lo, Hi); 790 break; 791 case WidenVector: { 792 // If the result needs to be split and the input needs to be widened, 793 // the two types must have different lengths. Use the widened result 794 // and extract from it to do the split. 795 SDValue InOp = GetWidenedVector(N->getOperand(0)); 796 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 797 LoVT.getVectorNumElements()); 798 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 799 DAG.getIntPtrConstant(0)); 800 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 801 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 802 break; 803 } 804 } 805 806 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 807 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 808} 809 810void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 811 SDValue &Lo, SDValue &Hi) { 812 // The low and high parts of the original input give four input vectors. 813 SDValue Inputs[4]; 814 DebugLoc dl = N->getDebugLoc(); 815 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 816 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 817 EVT NewVT = Inputs[0].getValueType(); 818 unsigned NewElts = NewVT.getVectorNumElements(); 819 820 // If Lo or Hi uses elements from at most two of the four input vectors, then 821 // express it as a vector shuffle of those two inputs. Otherwise extract the 822 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 823 SmallVector<int, 16> Ops; 824 for (unsigned High = 0; High < 2; ++High) { 825 SDValue &Output = High ? Hi : Lo; 826 827 // Build a shuffle mask for the output, discovering on the fly which 828 // input vectors to use as shuffle operands (recorded in InputUsed). 829 // If building a suitable shuffle vector proves too hard, then bail 830 // out with useBuildVector set. 831 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 832 unsigned FirstMaskIdx = High * NewElts; 833 bool useBuildVector = false; 834 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 835 // The mask element. This indexes into the input. 836 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 837 838 // The input vector this mask element indexes into. 839 unsigned Input = (unsigned)Idx / NewElts; 840 841 if (Input >= array_lengthof(Inputs)) { 842 // The mask element does not index into any input vector. 843 Ops.push_back(-1); 844 continue; 845 } 846 847 // Turn the index into an offset from the start of the input vector. 848 Idx -= Input * NewElts; 849 850 // Find or create a shuffle vector operand to hold this input. 851 unsigned OpNo; 852 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 853 if (InputUsed[OpNo] == Input) { 854 // This input vector is already an operand. 855 break; 856 } else if (InputUsed[OpNo] == -1U) { 857 // Create a new operand for this input vector. 858 InputUsed[OpNo] = Input; 859 break; 860 } 861 } 862 863 if (OpNo >= array_lengthof(InputUsed)) { 864 // More than two input vectors used! Give up on trying to create a 865 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 866 useBuildVector = true; 867 break; 868 } 869 870 // Add the mask index for the new shuffle vector. 871 Ops.push_back(Idx + OpNo * NewElts); 872 } 873 874 if (useBuildVector) { 875 EVT EltVT = NewVT.getVectorElementType(); 876 SmallVector<SDValue, 16> SVOps; 877 878 // Extract the input elements by hand. 879 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 880 // The mask element. This indexes into the input. 881 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 882 883 // The input vector this mask element indexes into. 884 unsigned Input = (unsigned)Idx / NewElts; 885 886 if (Input >= array_lengthof(Inputs)) { 887 // The mask element is "undef" or indexes off the end of the input. 888 SVOps.push_back(DAG.getUNDEF(EltVT)); 889 continue; 890 } 891 892 // Turn the index into an offset from the start of the input vector. 893 Idx -= Input * NewElts; 894 895 // Extract the vector element by hand. 896 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 897 Inputs[Input], DAG.getIntPtrConstant(Idx))); 898 } 899 900 // Construct the Lo/Hi output using a BUILD_VECTOR. 901 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 902 } else if (InputUsed[0] == -1U) { 903 // No input vectors were used! The result is undefined. 904 Output = DAG.getUNDEF(NewVT); 905 } else { 906 SDValue Op0 = Inputs[InputUsed[0]]; 907 // If only one input was used, use an undefined vector for the other. 908 SDValue Op1 = InputUsed[1] == -1U ? 909 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 910 // At least one input vector was used. Create a new shuffle vector. 911 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 912 } 913 914 Ops.clear(); 915 } 916} 917 918 919//===----------------------------------------------------------------------===// 920// Operand Vector Splitting 921//===----------------------------------------------------------------------===// 922 923/// SplitVectorOperand - This method is called when the specified operand of the 924/// specified node is found to need vector splitting. At this point, all of the 925/// result types of the node are known to be legal, but other operands of the 926/// node may need legalization as well as the specified one. 927bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 928 DEBUG(errs() << "Split node operand: "; 929 N->dump(&DAG); 930 errs() << "\n"); 931 SDValue Res = SDValue(); 932 933 if (Res.getNode() == 0) { 934 switch (N->getOpcode()) { 935 default: 936#ifndef NDEBUG 937 errs() << "SplitVectorOperand Op #" << OpNo << ": "; 938 N->dump(&DAG); 939 errs() << "\n"; 940#endif 941 llvm_unreachable("Do not know how to split this operator's operand!"); 942 943 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 944 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 945 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 946 case ISD::STORE: 947 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 948 break; 949 950 case ISD::CTTZ: 951 case ISD::CTLZ: 952 case ISD::CTPOP: 953 case ISD::FP_TO_SINT: 954 case ISD::FP_TO_UINT: 955 case ISD::SINT_TO_FP: 956 case ISD::UINT_TO_FP: 957 case ISD::TRUNCATE: 958 case ISD::SIGN_EXTEND: 959 case ISD::ZERO_EXTEND: 960 case ISD::ANY_EXTEND: 961 Res = SplitVecOp_UnaryOp(N); 962 break; 963 } 964 } 965 966 // If the result is null, the sub-method took care of registering results etc. 967 if (!Res.getNode()) return false; 968 969 // If the result is N, the sub-method updated N in place. Tell the legalizer 970 // core about this. 971 if (Res.getNode() == N) 972 return true; 973 974 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 975 "Invalid operand expansion"); 976 977 ReplaceValueWith(SDValue(N, 0), Res); 978 return false; 979} 980 981SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 982 // The result has a legal vector type, but the input needs splitting. 983 EVT ResVT = N->getValueType(0); 984 SDValue Lo, Hi; 985 DebugLoc dl = N->getDebugLoc(); 986 GetSplitVector(N->getOperand(0), Lo, Hi); 987 EVT InVT = Lo.getValueType(); 988 989 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 990 InVT.getVectorNumElements()); 991 992 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 993 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 994 995 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 996} 997 998SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 999 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 1000 // end up being split all the way down to individual components. Convert the 1001 // split pieces into integers and reassemble. 1002 SDValue Lo, Hi; 1003 GetSplitVector(N->getOperand(0), Lo, Hi); 1004 Lo = BitConvertToInteger(Lo); 1005 Hi = BitConvertToInteger(Hi); 1006 1007 if (TLI.isBigEndian()) 1008 std::swap(Lo, Hi); 1009 1010 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 1011 JoinIntegers(Lo, Hi)); 1012} 1013 1014SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1015 // We know that the extracted result type is legal. For now, assume the index 1016 // is a constant. 1017 EVT SubVT = N->getValueType(0); 1018 SDValue Idx = N->getOperand(1); 1019 DebugLoc dl = N->getDebugLoc(); 1020 SDValue Lo, Hi; 1021 GetSplitVector(N->getOperand(0), Lo, Hi); 1022 1023 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1024 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1025 1026 if (IdxVal < LoElts) { 1027 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1028 "Extracted subvector crosses vector split!"); 1029 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1030 } else { 1031 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1032 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1033 } 1034} 1035 1036SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1037 SDValue Vec = N->getOperand(0); 1038 SDValue Idx = N->getOperand(1); 1039 EVT VecVT = Vec.getValueType(); 1040 1041 if (isa<ConstantSDNode>(Idx)) { 1042 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1043 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1044 1045 SDValue Lo, Hi; 1046 GetSplitVector(Vec, Lo, Hi); 1047 1048 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1049 1050 if (IdxVal < LoElts) 1051 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1052 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1053 DAG.getConstant(IdxVal - LoElts, 1054 Idx.getValueType())); 1055 } 1056 1057 // Store the vector to the stack. 1058 EVT EltVT = VecVT.getVectorElementType(); 1059 DebugLoc dl = N->getDebugLoc(); 1060 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1061 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1062 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1063 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1064 1065 // Load back the required element. 1066 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1067 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1068 SV, 0, EltVT); 1069} 1070 1071SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1072 assert(N->isUnindexed() && "Indexed store of vector?"); 1073 assert(OpNo == 1 && "Can only split the stored value"); 1074 DebugLoc dl = N->getDebugLoc(); 1075 1076 bool isTruncating = N->isTruncatingStore(); 1077 SDValue Ch = N->getChain(); 1078 SDValue Ptr = N->getBasePtr(); 1079 int SVOffset = N->getSrcValueOffset(); 1080 EVT MemoryVT = N->getMemoryVT(); 1081 unsigned Alignment = N->getAlignment(); 1082 bool isVol = N->isVolatile(); 1083 SDValue Lo, Hi; 1084 GetSplitVector(N->getOperand(1), Lo, Hi); 1085 1086 EVT LoMemVT, HiMemVT; 1087 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1088 1089 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1090 1091 if (isTruncating) 1092 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1093 LoMemVT, isVol, Alignment); 1094 else 1095 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1096 isVol, Alignment); 1097 1098 // Increment the pointer to the other half. 1099 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1100 DAG.getIntPtrConstant(IncrementSize)); 1101 1102 if (isTruncating) 1103 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, 1104 N->getSrcValue(), SVOffset+IncrementSize, 1105 HiMemVT, 1106 isVol, MinAlign(Alignment, IncrementSize)); 1107 else 1108 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 1109 isVol, MinAlign(Alignment, IncrementSize)); 1110 1111 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1112} 1113 1114 1115//===----------------------------------------------------------------------===// 1116// Result Vector Widening 1117//===----------------------------------------------------------------------===// 1118 1119void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1120 DEBUG(errs() << "Widen node result " << ResNo << ": "; 1121 N->dump(&DAG); 1122 errs() << "\n"); 1123 SDValue Res = SDValue(); 1124 1125 switch (N->getOpcode()) { 1126 default: 1127#ifndef NDEBUG 1128 errs() << "WidenVectorResult #" << ResNo << ": "; 1129 N->dump(&DAG); 1130 errs() << "\n"; 1131#endif 1132 llvm_unreachable("Do not know how to widen the result of this operator!"); 1133 1134 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1135 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1136 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1137 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1138 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1139 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1140 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1141 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1142 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1143 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1144 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1145 case ISD::VECTOR_SHUFFLE: 1146 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1147 break; 1148 case ISD::VSETCC: 1149 Res = WidenVecRes_VSETCC(N); 1150 break; 1151 1152 case ISD::ADD: 1153 case ISD::AND: 1154 case ISD::BSWAP: 1155 case ISD::FADD: 1156 case ISD::FCOPYSIGN: 1157 case ISD::FDIV: 1158 case ISD::FMUL: 1159 case ISD::FPOW: 1160 case ISD::FPOWI: 1161 case ISD::FREM: 1162 case ISD::FSUB: 1163 case ISD::MUL: 1164 case ISD::MULHS: 1165 case ISD::MULHU: 1166 case ISD::OR: 1167 case ISD::SDIV: 1168 case ISD::SREM: 1169 case ISD::UDIV: 1170 case ISD::UREM: 1171 case ISD::SUB: 1172 case ISD::XOR: 1173 Res = WidenVecRes_Binary(N); 1174 break; 1175 1176 case ISD::SHL: 1177 case ISD::SRA: 1178 case ISD::SRL: 1179 Res = WidenVecRes_Shift(N); 1180 break; 1181 1182 case ISD::FP_ROUND: 1183 case ISD::FP_TO_SINT: 1184 case ISD::FP_TO_UINT: 1185 case ISD::SINT_TO_FP: 1186 case ISD::UINT_TO_FP: 1187 case ISD::TRUNCATE: 1188 case ISD::SIGN_EXTEND: 1189 case ISD::ZERO_EXTEND: 1190 case ISD::ANY_EXTEND: 1191 Res = WidenVecRes_Convert(N); 1192 break; 1193 1194 case ISD::CTLZ: 1195 case ISD::CTPOP: 1196 case ISD::CTTZ: 1197 case ISD::FABS: 1198 case ISD::FCOS: 1199 case ISD::FNEG: 1200 case ISD::FSIN: 1201 case ISD::FSQRT: 1202 Res = WidenVecRes_Unary(N); 1203 break; 1204 } 1205 1206 // If Res is null, the sub-method took care of registering the result. 1207 if (Res.getNode()) 1208 SetWidenedVector(SDValue(N, ResNo), Res); 1209} 1210 1211SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1212 // Binary op widening. 1213 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1214 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1215 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1216 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1217} 1218 1219SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1220 SDValue InOp = N->getOperand(0); 1221 DebugLoc dl = N->getDebugLoc(); 1222 1223 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1224 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1225 1226 EVT InVT = InOp.getValueType(); 1227 EVT InEltVT = InVT.getVectorElementType(); 1228 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1229 1230 unsigned Opcode = N->getOpcode(); 1231 unsigned InVTNumElts = InVT.getVectorNumElements(); 1232 1233 if (getTypeAction(InVT) == WidenVector) { 1234 InOp = GetWidenedVector(N->getOperand(0)); 1235 InVT = InOp.getValueType(); 1236 InVTNumElts = InVT.getVectorNumElements(); 1237 if (InVTNumElts == WidenNumElts) 1238 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1239 } 1240 1241 if (TLI.isTypeLegal(InWidenVT)) { 1242 // Because the result and the input are different vector types, widening 1243 // the result could create a legal type but widening the input might make 1244 // it an illegal type that might lead to repeatedly splitting the input 1245 // and then widening it. To avoid this, we widen the input only if 1246 // it results in a legal type. 1247 if (WidenNumElts % InVTNumElts == 0) { 1248 // Widen the input and call convert on the widened input vector. 1249 unsigned NumConcat = WidenNumElts/InVTNumElts; 1250 SmallVector<SDValue, 16> Ops(NumConcat); 1251 Ops[0] = InOp; 1252 SDValue UndefVal = DAG.getUNDEF(InVT); 1253 for (unsigned i = 1; i != NumConcat; ++i) 1254 Ops[i] = UndefVal; 1255 return DAG.getNode(Opcode, dl, WidenVT, 1256 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1257 &Ops[0], NumConcat)); 1258 } 1259 1260 if (InVTNumElts % WidenNumElts == 0) { 1261 // Extract the input and convert the shorten input vector. 1262 return DAG.getNode(Opcode, dl, WidenVT, 1263 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1264 InOp, DAG.getIntPtrConstant(0))); 1265 } 1266 } 1267 1268 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1269 SmallVector<SDValue, 16> Ops(WidenNumElts); 1270 EVT EltVT = WidenVT.getVectorElementType(); 1271 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1272 unsigned i; 1273 for (i=0; i < MinElts; ++i) 1274 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1275 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1276 DAG.getIntPtrConstant(i))); 1277 1278 SDValue UndefVal = DAG.getUNDEF(EltVT); 1279 for (; i < WidenNumElts; ++i) 1280 Ops[i] = UndefVal; 1281 1282 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1283} 1284 1285SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1286 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1287 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1288 SDValue ShOp = N->getOperand(1); 1289 1290 EVT ShVT = ShOp.getValueType(); 1291 if (getTypeAction(ShVT) == WidenVector) { 1292 ShOp = GetWidenedVector(ShOp); 1293 ShVT = ShOp.getValueType(); 1294 } 1295 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(), 1296 WidenVT.getVectorNumElements()); 1297 if (ShVT != ShWidenVT) 1298 ShOp = ModifyToType(ShOp, ShWidenVT); 1299 1300 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1301} 1302 1303SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1304 // Unary op widening. 1305 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1306 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1307 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1308} 1309 1310SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1311 SDValue InOp = N->getOperand(0); 1312 EVT InVT = InOp.getValueType(); 1313 EVT VT = N->getValueType(0); 1314 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1315 DebugLoc dl = N->getDebugLoc(); 1316 1317 switch (getTypeAction(InVT)) { 1318 default: 1319 assert(false && "Unknown type action!"); 1320 break; 1321 case Legal: 1322 break; 1323 case PromoteInteger: 1324 // If the InOp is promoted to the same size, convert it. Otherwise, 1325 // fall out of the switch and widen the promoted input. 1326 InOp = GetPromotedInteger(InOp); 1327 InVT = InOp.getValueType(); 1328 if (WidenVT.bitsEq(InVT)) 1329 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1330 break; 1331 case SoftenFloat: 1332 case ExpandInteger: 1333 case ExpandFloat: 1334 case ScalarizeVector: 1335 case SplitVector: 1336 break; 1337 case WidenVector: 1338 // If the InOp is widened to the same size, convert it. Otherwise, fall 1339 // out of the switch and widen the widened input. 1340 InOp = GetWidenedVector(InOp); 1341 InVT = InOp.getValueType(); 1342 if (WidenVT.bitsEq(InVT)) 1343 // The input widens to the same size. Convert to the widen value. 1344 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1345 break; 1346 } 1347 1348 unsigned WidenSize = WidenVT.getSizeInBits(); 1349 unsigned InSize = InVT.getSizeInBits(); 1350 if (WidenSize % InSize == 0) { 1351 // Determine new input vector type. The new input vector type will use 1352 // the same element type (if its a vector) or use the input type as a 1353 // vector. It is the same size as the type to widen to. 1354 EVT NewInVT; 1355 unsigned NewNumElts = WidenSize / InSize; 1356 if (InVT.isVector()) { 1357 EVT InEltVT = InVT.getVectorElementType(); 1358 NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits()); 1359 } else { 1360 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1361 } 1362 1363 if (TLI.isTypeLegal(NewInVT)) { 1364 // Because the result and the input are different vector types, widening 1365 // the result could create a legal type but widening the input might make 1366 // it an illegal type that might lead to repeatedly splitting the input 1367 // and then widening it. To avoid this, we widen the input only if 1368 // it results in a legal type. 1369 SmallVector<SDValue, 16> Ops(NewNumElts); 1370 SDValue UndefVal = DAG.getUNDEF(InVT); 1371 Ops[0] = InOp; 1372 for (unsigned i = 1; i < NewNumElts; ++i) 1373 Ops[i] = UndefVal; 1374 1375 SDValue NewVec; 1376 if (InVT.isVector()) 1377 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1378 NewInVT, &Ops[0], NewNumElts); 1379 else 1380 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1381 NewInVT, &Ops[0], NewNumElts); 1382 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1383 } 1384 } 1385 1386 return CreateStackStoreLoad(InOp, WidenVT); 1387} 1388 1389SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1390 DebugLoc dl = N->getDebugLoc(); 1391 // Build a vector with undefined for the new nodes. 1392 EVT VT = N->getValueType(0); 1393 EVT EltVT = VT.getVectorElementType(); 1394 unsigned NumElts = VT.getVectorNumElements(); 1395 1396 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1397 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1398 1399 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1400 NewOps.reserve(WidenNumElts); 1401 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1402 NewOps.push_back(DAG.getUNDEF(EltVT)); 1403 1404 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1405} 1406 1407SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1408 EVT InVT = N->getOperand(0).getValueType(); 1409 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1410 DebugLoc dl = N->getDebugLoc(); 1411 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1412 unsigned NumOperands = N->getNumOperands(); 1413 1414 bool InputWidened = false; // Indicates we need to widen the input. 1415 if (getTypeAction(InVT) != WidenVector) { 1416 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1417 // Add undef vectors to widen to correct length. 1418 unsigned NumConcat = WidenVT.getVectorNumElements() / 1419 InVT.getVectorNumElements(); 1420 SDValue UndefVal = DAG.getUNDEF(InVT); 1421 SmallVector<SDValue, 16> Ops(NumConcat); 1422 for (unsigned i=0; i < NumOperands; ++i) 1423 Ops[i] = N->getOperand(i); 1424 for (unsigned i = NumOperands; i != NumConcat; ++i) 1425 Ops[i] = UndefVal; 1426 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1427 } 1428 } else { 1429 InputWidened = true; 1430 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1431 // The inputs and the result are widen to the same value. 1432 unsigned i; 1433 for (i=1; i < NumOperands; ++i) 1434 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1435 break; 1436 1437 if (i > NumOperands) 1438 // Everything but the first operand is an UNDEF so just return the 1439 // widened first operand. 1440 return GetWidenedVector(N->getOperand(0)); 1441 1442 if (NumOperands == 2) { 1443 // Replace concat of two operands with a shuffle. 1444 SmallVector<int, 16> MaskOps(WidenNumElts); 1445 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1446 MaskOps[i] = i; 1447 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1448 } 1449 return DAG.getVectorShuffle(WidenVT, dl, 1450 GetWidenedVector(N->getOperand(0)), 1451 GetWidenedVector(N->getOperand(1)), 1452 &MaskOps[0]); 1453 } 1454 } 1455 } 1456 1457 // Fall back to use extracts and build vector. 1458 EVT EltVT = WidenVT.getVectorElementType(); 1459 unsigned NumInElts = InVT.getVectorNumElements(); 1460 SmallVector<SDValue, 16> Ops(WidenNumElts); 1461 unsigned Idx = 0; 1462 for (unsigned i=0; i < NumOperands; ++i) { 1463 SDValue InOp = N->getOperand(i); 1464 if (InputWidened) 1465 InOp = GetWidenedVector(InOp); 1466 for (unsigned j=0; j < NumInElts; ++j) 1467 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1468 DAG.getIntPtrConstant(j)); 1469 } 1470 SDValue UndefVal = DAG.getUNDEF(EltVT); 1471 for (; Idx < WidenNumElts; ++Idx) 1472 Ops[Idx] = UndefVal; 1473 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1474} 1475 1476SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1477 DebugLoc dl = N->getDebugLoc(); 1478 SDValue InOp = N->getOperand(0); 1479 SDValue RndOp = N->getOperand(3); 1480 SDValue SatOp = N->getOperand(4); 1481 1482 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1483 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1484 1485 EVT InVT = InOp.getValueType(); 1486 EVT InEltVT = InVT.getVectorElementType(); 1487 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1488 1489 SDValue DTyOp = DAG.getValueType(WidenVT); 1490 SDValue STyOp = DAG.getValueType(InWidenVT); 1491 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1492 1493 unsigned InVTNumElts = InVT.getVectorNumElements(); 1494 if (getTypeAction(InVT) == WidenVector) { 1495 InOp = GetWidenedVector(InOp); 1496 InVT = InOp.getValueType(); 1497 InVTNumElts = InVT.getVectorNumElements(); 1498 if (InVTNumElts == WidenNumElts) 1499 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1500 SatOp, CvtCode); 1501 } 1502 1503 if (TLI.isTypeLegal(InWidenVT)) { 1504 // Because the result and the input are different vector types, widening 1505 // the result could create a legal type but widening the input might make 1506 // it an illegal type that might lead to repeatedly splitting the input 1507 // and then widening it. To avoid this, we widen the input only if 1508 // it results in a legal type. 1509 if (WidenNumElts % InVTNumElts == 0) { 1510 // Widen the input and call convert on the widened input vector. 1511 unsigned NumConcat = WidenNumElts/InVTNumElts; 1512 SmallVector<SDValue, 16> Ops(NumConcat); 1513 Ops[0] = InOp; 1514 SDValue UndefVal = DAG.getUNDEF(InVT); 1515 for (unsigned i = 1; i != NumConcat; ++i) { 1516 Ops[i] = UndefVal; 1517 } 1518 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1519 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1520 SatOp, CvtCode); 1521 } 1522 1523 if (InVTNumElts % WidenNumElts == 0) { 1524 // Extract the input and convert the shorten input vector. 1525 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1526 DAG.getIntPtrConstant(0)); 1527 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1528 SatOp, CvtCode); 1529 } 1530 } 1531 1532 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1533 SmallVector<SDValue, 16> Ops(WidenNumElts); 1534 EVT EltVT = WidenVT.getVectorElementType(); 1535 DTyOp = DAG.getValueType(EltVT); 1536 STyOp = DAG.getValueType(InEltVT); 1537 1538 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1539 unsigned i; 1540 for (i=0; i < MinElts; ++i) { 1541 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1542 DAG.getIntPtrConstant(i)); 1543 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1544 SatOp, CvtCode); 1545 } 1546 1547 SDValue UndefVal = DAG.getUNDEF(EltVT); 1548 for (; i < WidenNumElts; ++i) 1549 Ops[i] = UndefVal; 1550 1551 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1552} 1553 1554SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1555 EVT VT = N->getValueType(0); 1556 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1557 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1558 SDValue InOp = N->getOperand(0); 1559 SDValue Idx = N->getOperand(1); 1560 DebugLoc dl = N->getDebugLoc(); 1561 1562 if (getTypeAction(InOp.getValueType()) == WidenVector) 1563 InOp = GetWidenedVector(InOp); 1564 1565 EVT InVT = InOp.getValueType(); 1566 1567 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1568 if (CIdx) { 1569 unsigned IdxVal = CIdx->getZExtValue(); 1570 // Check if we can just return the input vector after widening. 1571 if (IdxVal == 0 && InVT == WidenVT) 1572 return InOp; 1573 1574 // Check if we can extract from the vector. 1575 unsigned InNumElts = InVT.getVectorNumElements(); 1576 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1577 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1578 } 1579 1580 // We could try widening the input to the right length but for now, extract 1581 // the original elements, fill the rest with undefs and build a vector. 1582 SmallVector<SDValue, 16> Ops(WidenNumElts); 1583 EVT EltVT = VT.getVectorElementType(); 1584 EVT IdxVT = Idx.getValueType(); 1585 unsigned NumElts = VT.getVectorNumElements(); 1586 unsigned i; 1587 if (CIdx) { 1588 unsigned IdxVal = CIdx->getZExtValue(); 1589 for (i=0; i < NumElts; ++i) 1590 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1591 DAG.getConstant(IdxVal+i, IdxVT)); 1592 } else { 1593 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1594 for (i=1; i < NumElts; ++i) { 1595 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1596 DAG.getConstant(i, IdxVT)); 1597 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1598 } 1599 } 1600 1601 SDValue UndefVal = DAG.getUNDEF(EltVT); 1602 for (; i < WidenNumElts; ++i) 1603 Ops[i] = UndefVal; 1604 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1605} 1606 1607SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1608 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1609 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1610 InOp.getValueType(), InOp, 1611 N->getOperand(1), N->getOperand(2)); 1612} 1613 1614SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1615 LoadSDNode *LD = cast<LoadSDNode>(N); 1616 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1617 EVT LdVT = LD->getMemoryVT(); 1618 DebugLoc dl = N->getDebugLoc(); 1619 assert(LdVT.isVector() && WidenVT.isVector()); 1620 1621 // Load information 1622 SDValue Chain = LD->getChain(); 1623 SDValue BasePtr = LD->getBasePtr(); 1624 int SVOffset = LD->getSrcValueOffset(); 1625 unsigned Align = LD->getAlignment(); 1626 bool isVolatile = LD->isVolatile(); 1627 const Value *SV = LD->getSrcValue(); 1628 ISD::LoadExtType ExtType = LD->getExtensionType(); 1629 1630 SDValue Result; 1631 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1632 if (ExtType != ISD::NON_EXTLOAD) { 1633 // For extension loads, we can not play the tricks of chopping legal 1634 // vector types and bit cast it to the right type. Instead, we unroll 1635 // the load and build a vector. 1636 EVT EltVT = WidenVT.getVectorElementType(); 1637 EVT LdEltVT = LdVT.getVectorElementType(); 1638 unsigned NumElts = LdVT.getVectorNumElements(); 1639 1640 // Load each element and widen 1641 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1642 SmallVector<SDValue, 16> Ops(WidenNumElts); 1643 unsigned Increment = LdEltVT.getSizeInBits() / 8; 1644 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 1645 LdEltVT, isVolatile, Align); 1646 LdChain.push_back(Ops[0].getValue(1)); 1647 unsigned i = 0, Offset = Increment; 1648 for (i=1; i < NumElts; ++i, Offset += Increment) { 1649 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1650 BasePtr, DAG.getIntPtrConstant(Offset)); 1651 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 1652 SVOffset + Offset, LdEltVT, isVolatile, Align); 1653 LdChain.push_back(Ops[i].getValue(1)); 1654 } 1655 1656 // Fill the rest with undefs 1657 SDValue UndefVal = DAG.getUNDEF(EltVT); 1658 for (; i != WidenNumElts; ++i) 1659 Ops[i] = UndefVal; 1660 1661 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 1662 } else { 1663 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 1664 unsigned int LdWidth = LdVT.getSizeInBits(); 1665 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 1666 Align, isVolatile, LdWidth, WidenVT, dl); 1667 } 1668 1669 // If we generate a single load, we can use that for the chain. Otherwise, 1670 // build a factor node to remember the multiple loads are independent and 1671 // chain to that. 1672 SDValue NewChain; 1673 if (LdChain.size() == 1) 1674 NewChain = LdChain[0]; 1675 else 1676 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 1677 LdChain.size()); 1678 1679 // Modified the chain - switch anything that used the old chain to use 1680 // the new one. 1681 ReplaceValueWith(SDValue(N, 1), NewChain); 1682 1683 return Result; 1684} 1685 1686SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1687 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1688 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1689 WidenVT, N->getOperand(0)); 1690} 1691 1692SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1693 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1694 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1695 1696 SDValue Cond1 = N->getOperand(0); 1697 EVT CondVT = Cond1.getValueType(); 1698 if (CondVT.isVector()) { 1699 EVT CondEltVT = CondVT.getVectorElementType(); 1700 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts); 1701 if (getTypeAction(CondVT) == WidenVector) 1702 Cond1 = GetWidenedVector(Cond1); 1703 1704 if (Cond1.getValueType() != CondWidenVT) 1705 Cond1 = ModifyToType(Cond1, CondWidenVT); 1706 } 1707 1708 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1709 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1710 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1711 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1712 WidenVT, Cond1, InOp1, InOp2); 1713} 1714 1715SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1716 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1717 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1718 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1719 InOp1.getValueType(), N->getOperand(0), 1720 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1721} 1722 1723SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1724 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1725 return DAG.getUNDEF(WidenVT); 1726} 1727 1728SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1729 EVT VT = N->getValueType(0); 1730 DebugLoc dl = N->getDebugLoc(); 1731 1732 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1733 unsigned NumElts = VT.getVectorNumElements(); 1734 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1735 1736 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1737 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1738 1739 // Adjust mask based on new input vector length. 1740 SmallVector<int, 16> NewMask; 1741 for (unsigned i = 0; i != NumElts; ++i) { 1742 int Idx = N->getMaskElt(i); 1743 if (Idx < (int)NumElts) 1744 NewMask.push_back(Idx); 1745 else 1746 NewMask.push_back(Idx - NumElts + WidenNumElts); 1747 } 1748 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1749 NewMask.push_back(-1); 1750 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1751} 1752 1753SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1754 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1755 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1756 1757 SDValue InOp1 = N->getOperand(0); 1758 EVT InVT = InOp1.getValueType(); 1759 assert(InVT.isVector() && "can not widen non vector type"); 1760 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); 1761 InOp1 = GetWidenedVector(InOp1); 1762 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1763 1764 // Assume that the input and output will be widen appropriately. If not, 1765 // we will have to unroll it at some point. 1766 assert(InOp1.getValueType() == WidenInVT && 1767 InOp2.getValueType() == WidenInVT && 1768 "Input not widened to expected type!"); 1769 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1770 WidenVT, InOp1, InOp2, N->getOperand(2)); 1771} 1772 1773 1774//===----------------------------------------------------------------------===// 1775// Widen Vector Operand 1776//===----------------------------------------------------------------------===// 1777bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1778 DEBUG(errs() << "Widen node operand " << ResNo << ": "; 1779 N->dump(&DAG); 1780 errs() << "\n"); 1781 SDValue Res = SDValue(); 1782 1783 switch (N->getOpcode()) { 1784 default: 1785#ifndef NDEBUG 1786 errs() << "WidenVectorOperand op #" << ResNo << ": "; 1787 N->dump(&DAG); 1788 errs() << "\n"; 1789#endif 1790 llvm_unreachable("Do not know how to widen this operator's operand!"); 1791 1792 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1793 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1794 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1795 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1796 1797 case ISD::FP_ROUND: 1798 case ISD::FP_TO_SINT: 1799 case ISD::FP_TO_UINT: 1800 case ISD::SINT_TO_FP: 1801 case ISD::UINT_TO_FP: 1802 case ISD::TRUNCATE: 1803 case ISD::SIGN_EXTEND: 1804 case ISD::ZERO_EXTEND: 1805 case ISD::ANY_EXTEND: 1806 Res = WidenVecOp_Convert(N); 1807 break; 1808 } 1809 1810 // If Res is null, the sub-method took care of registering the result. 1811 if (!Res.getNode()) return false; 1812 1813 // If the result is N, the sub-method updated N in place. Tell the legalizer 1814 // core about this. 1815 if (Res.getNode() == N) 1816 return true; 1817 1818 1819 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1820 "Invalid operand expansion"); 1821 1822 ReplaceValueWith(SDValue(N, 0), Res); 1823 return false; 1824} 1825 1826SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1827 // Since the result is legal and the input is illegal, it is unlikely 1828 // that we can fix the input to a legal type so unroll the convert 1829 // into some scalar code and create a nasty build vector. 1830 EVT VT = N->getValueType(0); 1831 EVT EltVT = VT.getVectorElementType(); 1832 DebugLoc dl = N->getDebugLoc(); 1833 unsigned NumElts = VT.getVectorNumElements(); 1834 SDValue InOp = N->getOperand(0); 1835 if (getTypeAction(InOp.getValueType()) == WidenVector) 1836 InOp = GetWidenedVector(InOp); 1837 EVT InVT = InOp.getValueType(); 1838 EVT InEltVT = InVT.getVectorElementType(); 1839 1840 unsigned Opcode = N->getOpcode(); 1841 SmallVector<SDValue, 16> Ops(NumElts); 1842 for (unsigned i=0; i < NumElts; ++i) 1843 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1844 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1845 DAG.getIntPtrConstant(i))); 1846 1847 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1848} 1849 1850SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1851 EVT VT = N->getValueType(0); 1852 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1853 EVT InWidenVT = InOp.getValueType(); 1854 DebugLoc dl = N->getDebugLoc(); 1855 1856 // Check if we can convert between two legal vector types and extract. 1857 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1858 unsigned Size = VT.getSizeInBits(); 1859 if (InWidenSize % Size == 0 && !VT.isVector()) { 1860 unsigned NewNumElts = InWidenSize / Size; 1861 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 1862 if (TLI.isTypeLegal(NewVT)) { 1863 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1864 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1865 DAG.getIntPtrConstant(0)); 1866 } 1867 } 1868 1869 return CreateStackStoreLoad(InOp, VT); 1870} 1871 1872SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1873 // If the input vector is not legal, it is likely that we will not find a 1874 // legal vector of the same size. Replace the concatenate vector with a 1875 // nasty build vector. 1876 EVT VT = N->getValueType(0); 1877 EVT EltVT = VT.getVectorElementType(); 1878 DebugLoc dl = N->getDebugLoc(); 1879 unsigned NumElts = VT.getVectorNumElements(); 1880 SmallVector<SDValue, 16> Ops(NumElts); 1881 1882 EVT InVT = N->getOperand(0).getValueType(); 1883 unsigned NumInElts = InVT.getVectorNumElements(); 1884 1885 unsigned Idx = 0; 1886 unsigned NumOperands = N->getNumOperands(); 1887 for (unsigned i=0; i < NumOperands; ++i) { 1888 SDValue InOp = N->getOperand(i); 1889 if (getTypeAction(InOp.getValueType()) == WidenVector) 1890 InOp = GetWidenedVector(InOp); 1891 for (unsigned j=0; j < NumInElts; ++j) 1892 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1893 DAG.getIntPtrConstant(j)); 1894 } 1895 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1896} 1897 1898SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1899 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1900 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1901 N->getValueType(0), InOp, N->getOperand(1)); 1902} 1903 1904SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1905 // We have to widen the value but we want only to store the original 1906 // vector type. 1907 StoreSDNode *ST = cast<StoreSDNode>(N); 1908 SDValue Chain = ST->getChain(); 1909 SDValue BasePtr = ST->getBasePtr(); 1910 const Value *SV = ST->getSrcValue(); 1911 int SVOffset = ST->getSrcValueOffset(); 1912 unsigned Align = ST->getAlignment(); 1913 bool isVolatile = ST->isVolatile(); 1914 SDValue ValOp = GetWidenedVector(ST->getValue()); 1915 DebugLoc dl = N->getDebugLoc(); 1916 1917 EVT StVT = ST->getMemoryVT(); 1918 EVT ValVT = ValOp.getValueType(); 1919 // It must be true that we the widen vector type is bigger than where 1920 // we need to store. 1921 assert(StVT.isVector() && ValOp.getValueType().isVector()); 1922 assert(StVT.bitsLT(ValOp.getValueType())); 1923 1924 SmallVector<SDValue, 16> StChain; 1925 if (ST->isTruncatingStore()) { 1926 // For truncating stores, we can not play the tricks of chopping legal 1927 // vector types and bit cast it to the right type. Instead, we unroll 1928 // the store. 1929 EVT StEltVT = StVT.getVectorElementType(); 1930 EVT ValEltVT = ValVT.getVectorElementType(); 1931 unsigned Increment = ValEltVT.getSizeInBits() / 8; 1932 unsigned NumElts = StVT.getVectorNumElements(); 1933 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1934 DAG.getIntPtrConstant(0)); 1935 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 1936 SVOffset, StEltVT, 1937 isVolatile, Align)); 1938 unsigned Offset = Increment; 1939 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 1940 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1941 BasePtr, DAG.getIntPtrConstant(Offset)); 1942 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1943 DAG.getIntPtrConstant(0)); 1944 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 1945 SVOffset + Offset, StEltVT, 1946 isVolatile, MinAlign(Align, Offset))); 1947 } 1948 } 1949 else { 1950 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 1951 // Store value 1952 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 1953 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 1954 } 1955 if (StChain.size() == 1) 1956 return StChain[0]; 1957 else 1958 return DAG.getNode(ISD::TokenFactor, dl, 1959 MVT::Other,&StChain[0],StChain.size()); 1960} 1961 1962//===----------------------------------------------------------------------===// 1963// Vector Widening Utilities 1964//===----------------------------------------------------------------------===// 1965 1966 1967// Utility function to find a vector type and its associated element 1968// type from a preferred width and whose vector type must be the same size 1969// as the VecVT. 1970// TLI: Target lowering used to determine legal types. 1971// Width: Preferred width to store. 1972// VecVT: Vector value type whose size we must match. 1973// Returns NewVecVT and NewEltVT - the vector type and its associated 1974// element type. 1975static void FindAssocWidenVecType(SelectionDAG& DAG, 1976 const TargetLowering &TLI, unsigned Width, 1977 EVT VecVT, 1978 EVT& NewEltVT, EVT& NewVecVT) { 1979 unsigned EltWidth = Width + 1; 1980 if (TLI.isTypeLegal(VecVT)) { 1981 // We start with the preferred with, making it a power of 2 and find a 1982 // legal vector type of that width. If not, we reduce it by another of 2. 1983 // For incoming type is legal, this process will end as a vector of the 1984 // smallest loadable type should always be legal. 1985 do { 1986 assert(EltWidth > 0); 1987 EltWidth = 1 << Log2_32(EltWidth - 1); 1988 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 1989 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 1990 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 1991 } while (!TLI.isTypeLegal(NewVecVT) || 1992 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 1993 } else { 1994 // The incoming vector type is illegal and is the result of widening 1995 // a vector to a power of 2. In this case, we will use the preferred 1996 // with as long as it is a multiple of the incoming vector length. 1997 // The legalization process will eventually make this into a legal type 1998 // and remove the illegal bit converts (which would turn to stack converts 1999 // if they are allow to exist). 2000 do { 2001 assert(EltWidth > 0); 2002 EltWidth = 1 << Log2_32(EltWidth - 1); 2003 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 2004 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2005 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 2006 } while (!TLI.isTypeLegal(NewEltVT) || 2007 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2008 } 2009} 2010 2011SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2012 SDValue Chain, 2013 SDValue BasePtr, 2014 const Value *SV, 2015 int SVOffset, 2016 unsigned Alignment, 2017 bool isVolatile, 2018 unsigned LdWidth, 2019 EVT ResType, 2020 DebugLoc dl) { 2021 // The strategy assumes that we can efficiently load powers of two widths. 2022 // The routines chops the vector into the largest power of 2 load and 2023 // can be inserted into a legal vector and then cast the result into the 2024 // vector type we want. This avoids unnecessary stack converts. 2025 2026 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 2027 // the load is nonvolatile, we an use a wider load for the value. 2028 2029 // Find the vector type that can load from. 2030 EVT NewEltVT, NewVecVT; 2031 unsigned NewEltVTWidth; 2032 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2033 NewEltVTWidth = NewEltVT.getSizeInBits(); 2034 2035 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 2036 isVolatile, Alignment); 2037 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2038 LdChain.push_back(LdOp.getValue(1)); 2039 2040 // Check if we can load the element with one instruction 2041 if (LdWidth == NewEltVTWidth) { 2042 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2043 } 2044 2045 unsigned Idx = 1; 2046 LdWidth -= NewEltVTWidth; 2047 unsigned Offset = 0; 2048 2049 while (LdWidth > 0) { 2050 unsigned Increment = NewEltVTWidth / 8; 2051 Offset += Increment; 2052 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2053 DAG.getIntPtrConstant(Increment)); 2054 2055 if (LdWidth < NewEltVTWidth) { 2056 // Our current type we are using is too large, use a smaller size by 2057 // using a smaller power of 2 2058 unsigned oNewEltVTWidth = NewEltVTWidth; 2059 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2060 NewEltVTWidth = NewEltVT.getSizeInBits(); 2061 // Readjust position and vector position based on new load type 2062 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2063 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2064 } 2065 2066 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 2067 SVOffset+Offset, isVolatile, 2068 MinAlign(Alignment, Offset)); 2069 LdChain.push_back(LdOp.getValue(1)); 2070 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 2071 DAG.getIntPtrConstant(Idx++)); 2072 2073 LdWidth -= NewEltVTWidth; 2074 } 2075 2076 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2077} 2078 2079void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2080 SDValue Chain, 2081 SDValue BasePtr, 2082 const Value *SV, 2083 int SVOffset, 2084 unsigned Alignment, 2085 bool isVolatile, 2086 SDValue ValOp, 2087 unsigned StWidth, 2088 DebugLoc dl) { 2089 // Breaks the stores into a series of power of 2 width stores. For any 2090 // width, we convert the vector to the vector of element size that we 2091 // want to store. This avoids requiring a stack convert. 2092 2093 // Find a width of the element type we can store with 2094 EVT WidenVT = ValOp.getValueType(); 2095 EVT NewEltVT, NewVecVT; 2096 2097 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2098 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 2099 2100 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2101 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2102 DAG.getIntPtrConstant(0)); 2103 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 2104 isVolatile, Alignment); 2105 StChain.push_back(StOp); 2106 2107 // Check if we are done 2108 if (StWidth == NewEltVTWidth) { 2109 return; 2110 } 2111 2112 unsigned Idx = 1; 2113 StWidth -= NewEltVTWidth; 2114 unsigned Offset = 0; 2115 2116 while (StWidth > 0) { 2117 unsigned Increment = NewEltVTWidth / 8; 2118 Offset += Increment; 2119 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2120 DAG.getIntPtrConstant(Increment)); 2121 2122 if (StWidth < NewEltVTWidth) { 2123 // Our current type we are using is too large, use a smaller size by 2124 // using a smaller power of 2 2125 unsigned oNewEltVTWidth = NewEltVTWidth; 2126 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2127 NewEltVTWidth = NewEltVT.getSizeInBits(); 2128 // Readjust position and vector position based on new load type 2129 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2130 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2131 } 2132 2133 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2134 DAG.getIntPtrConstant(Idx++)); 2135 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2136 SVOffset + Offset, isVolatile, 2137 MinAlign(Alignment, Offset))); 2138 StWidth -= NewEltVTWidth; 2139 } 2140} 2141 2142/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2143/// input vector must have the same element type as NVT. 2144SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2145 // Note that InOp might have been widened so it might already have 2146 // the right width or it might need be narrowed. 2147 EVT InVT = InOp.getValueType(); 2148 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2149 "input and widen element type must match"); 2150 DebugLoc dl = InOp.getDebugLoc(); 2151 2152 // Check if InOp already has the right width. 2153 if (InVT == NVT) 2154 return InOp; 2155 2156 unsigned InNumElts = InVT.getVectorNumElements(); 2157 unsigned WidenNumElts = NVT.getVectorNumElements(); 2158 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2159 unsigned NumConcat = WidenNumElts / InNumElts; 2160 SmallVector<SDValue, 16> Ops(NumConcat); 2161 SDValue UndefVal = DAG.getUNDEF(InVT); 2162 Ops[0] = InOp; 2163 for (unsigned i = 1; i != NumConcat; ++i) 2164 Ops[i] = UndefVal; 2165 2166 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2167 } 2168 2169 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2170 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2171 DAG.getIntPtrConstant(0)); 2172 2173 // Fall back to extract and build. 2174 SmallVector<SDValue, 16> Ops(WidenNumElts); 2175 EVT EltVT = NVT.getVectorElementType(); 2176 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2177 unsigned Idx; 2178 for (Idx = 0; Idx < MinNumElts; ++Idx) 2179 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2180 DAG.getIntPtrConstant(Idx)); 2181 2182 SDValue UndefVal = DAG.getUNDEF(EltVT); 2183 for ( ; Idx < WidenNumElts; ++Idx) 2184 Ops[Idx] = UndefVal; 2185 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2186} 2187