LegalizeVectorTypes.cpp revision d1996360399ad6dbe75ee185b661b16c83146373
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in two vectors of half the size. For example, implementing 19// <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/CodeGen/PseudoSourceValue.h" 25#include "llvm/Target/TargetData.h" 26#include "llvm/Support/ErrorHandling.h" 27#include "llvm/Support/raw_ostream.h" 28using namespace llvm; 29 30//===----------------------------------------------------------------------===// 31// Result Vector Scalarization: <1 x ty> -> ty. 32//===----------------------------------------------------------------------===// 33 34void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 35 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 36 N->dump(&DAG); 37 dbgs() << "\n"); 38 SDValue R = SDValue(); 39 40 switch (N->getOpcode()) { 41 default: 42#ifndef NDEBUG 43 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 44 N->dump(&DAG); 45 dbgs() << "\n"; 46#endif 47 llvm_unreachable("Do not know how to scalarize the result of this operator!"); 48 49 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 50 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 51 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 52 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 53 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 54 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 55 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 56 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 57 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 58 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 59 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 60 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 61 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 62 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 63 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 64 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 65 66 case ISD::CTLZ: 67 case ISD::CTPOP: 68 case ISD::CTTZ: 69 case ISD::FABS: 70 case ISD::FCOS: 71 case ISD::FNEG: 72 case ISD::FP_TO_SINT: 73 case ISD::FP_TO_UINT: 74 case ISD::FSIN: 75 case ISD::FSQRT: 76 case ISD::FTRUNC: 77 case ISD::FFLOOR: 78 case ISD::FCEIL: 79 case ISD::FRINT: 80 case ISD::FNEARBYINT: 81 case ISD::UINT_TO_FP: 82 case ISD::SINT_TO_FP: 83 case ISD::TRUNCATE: 84 case ISD::SIGN_EXTEND: 85 case ISD::ZERO_EXTEND: 86 case ISD::ANY_EXTEND: 87 R = ScalarizeVecRes_UnaryOp(N); 88 break; 89 90 case ISD::ADD: 91 case ISD::AND: 92 case ISD::FADD: 93 case ISD::FDIV: 94 case ISD::FMUL: 95 case ISD::FPOW: 96 case ISD::FREM: 97 case ISD::FSUB: 98 case ISD::MUL: 99 case ISD::OR: 100 case ISD::SDIV: 101 case ISD::SREM: 102 case ISD::SUB: 103 case ISD::UDIV: 104 case ISD::UREM: 105 case ISD::XOR: 106 case ISD::SHL: 107 case ISD::SRA: 108 case ISD::SRL: 109 R = ScalarizeVecRes_BinOp(N); 110 break; 111 } 112 113 // If R is null, the sub-method took care of registering the result. 114 if (R.getNode()) 115 SetScalarizedVector(SDValue(N, ResNo), R); 116} 117 118SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 119 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 120 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 121 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 122 LHS.getValueType(), LHS, RHS); 123} 124 125SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 126 EVT NewVT = N->getValueType(0).getVectorElementType(); 127 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 128 NewVT, N->getOperand(0)); 129} 130 131SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 132 EVT NewVT = N->getValueType(0).getVectorElementType(); 133 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 134 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 135 Op0, DAG.getValueType(NewVT), 136 DAG.getValueType(Op0.getValueType()), 137 N->getOperand(3), 138 N->getOperand(4), 139 cast<CvtRndSatSDNode>(N)->getCvtCode()); 140} 141 142SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 143 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 144 N->getValueType(0).getVectorElementType(), 145 N->getOperand(0), N->getOperand(1)); 146} 147 148SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 149 SDValue Op = GetScalarizedVector(N->getOperand(0)); 150 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 151 Op.getValueType(), Op, N->getOperand(1)); 152} 153 154SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 155 // The value to insert may have a wider type than the vector element type, 156 // so be sure to truncate it to the element type if necessary. 157 SDValue Op = N->getOperand(1); 158 EVT EltVT = N->getValueType(0).getVectorElementType(); 159 if (Op.getValueType() != EltVT) 160 // FIXME: Can this happen for floating point types? 161 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 162 return Op; 163} 164 165SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 166 assert(N->isUnindexed() && "Indexed vector load?"); 167 168 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(), 169 N->getExtensionType(), 170 N->getValueType(0).getVectorElementType(), 171 N->getChain(), N->getBasePtr(), 172 DAG.getUNDEF(N->getBasePtr().getValueType()), 173 N->getSrcValue(), N->getSrcValueOffset(), 174 N->getMemoryVT().getVectorElementType(), 175 N->isVolatile(), N->getOriginalAlignment()); 176 177 // Legalized the chain result - switch anything that used the old chain to 178 // use the new one. 179 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 180 return Result; 181} 182 183SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 184 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 185 EVT DestVT = N->getValueType(0).getVectorElementType(); 186 SDValue Op = GetScalarizedVector(N->getOperand(0)); 187 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 188} 189 190SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 191 EVT EltVT = N->getValueType(0).getVectorElementType(); 192 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 193 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 194 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), EltVT, 195 LHS, DAG.getValueType(ExtVT)); 196} 197 198SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 199 // If the operand is wider than the vector element type then it is implicitly 200 // truncated. Make that explicit here. 201 EVT EltVT = N->getValueType(0).getVectorElementType(); 202 SDValue InOp = N->getOperand(0); 203 if (InOp.getValueType() != EltVT) 204 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 205 return InOp; 206} 207 208SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 209 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 210 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 211 LHS.getValueType(), N->getOperand(0), LHS, 212 GetScalarizedVector(N->getOperand(2))); 213} 214 215SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 216 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 217 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 218 N->getOperand(0), N->getOperand(1), 219 LHS, GetScalarizedVector(N->getOperand(3)), 220 N->getOperand(4)); 221} 222 223SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 224 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 225 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 226 DebugLoc DL = N->getDebugLoc(); 227 228 // Turn it into a scalar SETCC. 229 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 230} 231 232SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 233 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 234} 235 236SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 237 // Figure out if the scalar is the LHS or RHS and return it. 238 SDValue Arg = N->getOperand(2).getOperand(0); 239 if (Arg.getOpcode() == ISD::UNDEF) 240 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 241 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 242 return GetScalarizedVector(N->getOperand(Op)); 243} 244 245SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 246 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 247 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 248 EVT NVT = N->getValueType(0).getVectorElementType(); 249 EVT SVT = TLI.getSetCCResultType(LHS.getValueType()); 250 DebugLoc DL = N->getDebugLoc(); 251 252 // Turn it into a scalar SETCC. 253 SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); 254 255 // VSETCC always returns a sign-extended value, while SETCC may not. The 256 // SETCC result type may not match the vector element type. Correct these. 257 if (NVT.bitsLE(SVT)) { 258 // The SETCC result type is bigger than the vector element type. 259 // Ensure the SETCC result is sign-extended. 260 if (TLI.getBooleanContents() != 261 TargetLowering::ZeroOrNegativeOneBooleanContent) 262 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, 263 DAG.getValueType(MVT::i1)); 264 // Truncate to the final type. 265 return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); 266 } 267 268 // The SETCC result type is smaller than the vector element type. 269 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 270 if (TLI.getBooleanContents() != 271 TargetLowering::ZeroOrNegativeOneBooleanContent) 272 Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); 273 // Sign extend to the final type. 274 return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); 275} 276 277 278//===----------------------------------------------------------------------===// 279// Operand Vector Scalarization <1 x ty> -> ty. 280//===----------------------------------------------------------------------===// 281 282bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 283 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 284 N->dump(&DAG); 285 dbgs() << "\n"); 286 SDValue Res = SDValue(); 287 288 if (Res.getNode() == 0) { 289 switch (N->getOpcode()) { 290 default: 291#ifndef NDEBUG 292 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 293 N->dump(&DAG); 294 dbgs() << "\n"; 295#endif 296 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 297 case ISD::BIT_CONVERT: 298 Res = ScalarizeVecOp_BIT_CONVERT(N); 299 break; 300 case ISD::CONCAT_VECTORS: 301 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 302 break; 303 case ISD::EXTRACT_VECTOR_ELT: 304 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 305 break; 306 case ISD::STORE: 307 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 308 break; 309 } 310 } 311 312 // If the result is null, the sub-method took care of registering results etc. 313 if (!Res.getNode()) return false; 314 315 // If the result is N, the sub-method updated N in place. Tell the legalizer 316 // core about this. 317 if (Res.getNode() == N) 318 return true; 319 320 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 321 "Invalid operand expansion"); 322 323 ReplaceValueWith(SDValue(N, 0), Res); 324 return false; 325} 326 327/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 328/// to be scalarized, it must be <1 x ty>. Convert the element instead. 329SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 330 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 331 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), 332 N->getValueType(0), Elt); 333} 334 335/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 336/// use a BUILD_VECTOR instead. 337SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 338 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 339 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 340 Ops[i] = GetScalarizedVector(N->getOperand(i)); 341 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 342 &Ops[0], Ops.size()); 343} 344 345/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 346/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 347/// index. 348SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 349 SDValue Res = GetScalarizedVector(N->getOperand(0)); 350 if (Res.getValueType() != N->getValueType(0)) 351 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 352 Res); 353 return Res; 354} 355 356/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 357/// scalarized, it must be <1 x ty>. Just store the element. 358SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 359 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 360 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 361 DebugLoc dl = N->getDebugLoc(); 362 363 if (N->isTruncatingStore()) 364 return DAG.getTruncStore(N->getChain(), dl, 365 GetScalarizedVector(N->getOperand(1)), 366 N->getBasePtr(), 367 N->getSrcValue(), N->getSrcValueOffset(), 368 N->getMemoryVT().getVectorElementType(), 369 N->isVolatile(), N->getAlignment()); 370 371 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 372 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 373 N->isVolatile(), N->getOriginalAlignment()); 374} 375 376 377//===----------------------------------------------------------------------===// 378// Result Vector Splitting 379//===----------------------------------------------------------------------===// 380 381/// SplitVectorResult - This method is called when the specified result of the 382/// specified node is found to need vector splitting. At this point, the node 383/// may also have invalid operands or may have other results that need 384/// legalization, we just know that (at least) one result needs vector 385/// splitting. 386void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 387 DEBUG(dbgs() << "Split node result: "; 388 N->dump(&DAG); 389 dbgs() << "\n"); 390 SDValue Lo, Hi; 391 392 switch (N->getOpcode()) { 393 default: 394#ifndef NDEBUG 395 dbgs() << "SplitVectorResult #" << ResNo << ": "; 396 N->dump(&DAG); 397 dbgs() << "\n"; 398#endif 399 llvm_unreachable("Do not know how to split the result of this operator!"); 400 401 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 402 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 403 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 404 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 405 406 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 407 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 408 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 409 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 410 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 411 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 412 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 413 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 414 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 415 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 416 case ISD::LOAD: 417 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 418 break; 419 case ISD::SETCC: 420 case ISD::VSETCC: 421 SplitVecRes_SETCC(N, Lo, Hi); 422 break; 423 case ISD::VECTOR_SHUFFLE: 424 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 425 break; 426 427 case ISD::CTTZ: 428 case ISD::CTLZ: 429 case ISD::CTPOP: 430 case ISD::FNEG: 431 case ISD::FABS: 432 case ISD::FSQRT: 433 case ISD::FSIN: 434 case ISD::FCOS: 435 case ISD::FTRUNC: 436 case ISD::FFLOOR: 437 case ISD::FCEIL: 438 case ISD::FRINT: 439 case ISD::FNEARBYINT: 440 case ISD::FP_TO_SINT: 441 case ISD::FP_TO_UINT: 442 case ISD::SINT_TO_FP: 443 case ISD::UINT_TO_FP: 444 case ISD::TRUNCATE: 445 case ISD::SIGN_EXTEND: 446 case ISD::ZERO_EXTEND: 447 case ISD::ANY_EXTEND: 448 SplitVecRes_UnaryOp(N, Lo, Hi); 449 break; 450 451 case ISD::ADD: 452 case ISD::SUB: 453 case ISD::MUL: 454 case ISD::FADD: 455 case ISD::FSUB: 456 case ISD::FMUL: 457 case ISD::SDIV: 458 case ISD::UDIV: 459 case ISD::FDIV: 460 case ISD::FPOW: 461 case ISD::AND: 462 case ISD::OR: 463 case ISD::XOR: 464 case ISD::SHL: 465 case ISD::SRA: 466 case ISD::SRL: 467 case ISD::UREM: 468 case ISD::SREM: 469 case ISD::FREM: 470 SplitVecRes_BinOp(N, Lo, Hi); 471 break; 472 } 473 474 // If Lo/Hi is null, the sub-method took care of registering results etc. 475 if (Lo.getNode()) 476 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 477} 478 479void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 480 SDValue &Hi) { 481 SDValue LHSLo, LHSHi; 482 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 483 SDValue RHSLo, RHSHi; 484 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 485 DebugLoc dl = N->getDebugLoc(); 486 487 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 488 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 489} 490 491void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 492 SDValue &Hi) { 493 // We know the result is a vector. The input may be either a vector or a 494 // scalar value. 495 EVT LoVT, HiVT; 496 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 497 DebugLoc dl = N->getDebugLoc(); 498 499 SDValue InOp = N->getOperand(0); 500 EVT InVT = InOp.getValueType(); 501 502 // Handle some special cases efficiently. 503 switch (getTypeAction(InVT)) { 504 default: 505 assert(false && "Unknown type action!"); 506 case Legal: 507 case PromoteInteger: 508 case SoftenFloat: 509 case ScalarizeVector: 510 break; 511 case ExpandInteger: 512 case ExpandFloat: 513 // A scalar to vector conversion, where the scalar needs expansion. 514 // If the vector is being split in two then we can just convert the 515 // expanded pieces. 516 if (LoVT == HiVT) { 517 GetExpandedOp(InOp, Lo, Hi); 518 if (TLI.isBigEndian()) 519 std::swap(Lo, Hi); 520 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 521 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 522 return; 523 } 524 break; 525 case SplitVector: 526 // If the input is a vector that needs to be split, convert each split 527 // piece of the input now. 528 GetSplitVector(InOp, Lo, Hi); 529 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 530 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 531 return; 532 } 533 534 // In the general case, convert the input to an integer and split it by hand. 535 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 536 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 537 if (TLI.isBigEndian()) 538 std::swap(LoIntVT, HiIntVT); 539 540 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 541 542 if (TLI.isBigEndian()) 543 std::swap(Lo, Hi); 544 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo); 545 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi); 546} 547 548void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 549 SDValue &Hi) { 550 EVT LoVT, HiVT; 551 DebugLoc dl = N->getDebugLoc(); 552 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 553 unsigned LoNumElts = LoVT.getVectorNumElements(); 554 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 555 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 556 557 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 558 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 559} 560 561void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 562 SDValue &Hi) { 563 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 564 DebugLoc dl = N->getDebugLoc(); 565 unsigned NumSubvectors = N->getNumOperands() / 2; 566 if (NumSubvectors == 1) { 567 Lo = N->getOperand(0); 568 Hi = N->getOperand(1); 569 return; 570 } 571 572 EVT LoVT, HiVT; 573 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 574 575 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 576 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 577 578 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 579 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 580} 581 582void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 583 SDValue &Hi) { 584 EVT LoVT, HiVT; 585 DebugLoc dl = N->getDebugLoc(); 586 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 587 588 SDValue DTyOpLo = DAG.getValueType(LoVT); 589 SDValue DTyOpHi = DAG.getValueType(HiVT); 590 591 SDValue RndOp = N->getOperand(3); 592 SDValue SatOp = N->getOperand(4); 593 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 594 595 // Split the input. 596 SDValue VLo, VHi; 597 EVT InVT = N->getOperand(0).getValueType(); 598 switch (getTypeAction(InVT)) { 599 default: llvm_unreachable("Unexpected type action!"); 600 case Legal: { 601 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 602 LoVT.getVectorNumElements()); 603 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 604 DAG.getIntPtrConstant(0)); 605 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 606 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 607 break; 608 } 609 case SplitVector: 610 GetSplitVector(N->getOperand(0), VLo, VHi); 611 break; 612 case WidenVector: { 613 // If the result needs to be split and the input needs to be widened, 614 // the two types must have different lengths. Use the widened result 615 // and extract from it to do the split. 616 SDValue InOp = GetWidenedVector(N->getOperand(0)); 617 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 618 LoVT.getVectorNumElements()); 619 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 620 DAG.getIntPtrConstant(0)); 621 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 622 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 623 break; 624 } 625 } 626 627 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 628 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 629 630 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, 631 CvtCode); 632 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, 633 CvtCode); 634} 635 636void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 637 SDValue &Hi) { 638 SDValue Vec = N->getOperand(0); 639 SDValue Idx = N->getOperand(1); 640 EVT IdxVT = Idx.getValueType(); 641 DebugLoc dl = N->getDebugLoc(); 642 643 EVT LoVT, HiVT; 644 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 645 646 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 647 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, 648 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT)); 649 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx); 650} 651 652void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 653 SDValue &Hi) { 654 DebugLoc dl = N->getDebugLoc(); 655 GetSplitVector(N->getOperand(0), Lo, Hi); 656 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 657 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 658} 659 660void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 661 SDValue &Hi) { 662 SDValue LHSLo, LHSHi; 663 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 664 DebugLoc dl = N->getDebugLoc(); 665 666 EVT LoVT, HiVT; 667 GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT(), LoVT, HiVT); 668 669 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 670 DAG.getValueType(LoVT)); 671 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 672 DAG.getValueType(HiVT)); 673} 674 675void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 676 SDValue &Hi) { 677 SDValue Vec = N->getOperand(0); 678 SDValue Elt = N->getOperand(1); 679 SDValue Idx = N->getOperand(2); 680 DebugLoc dl = N->getDebugLoc(); 681 GetSplitVector(Vec, Lo, Hi); 682 683 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 684 unsigned IdxVal = CIdx->getZExtValue(); 685 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 686 if (IdxVal < LoNumElts) 687 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 688 Lo.getValueType(), Lo, Elt, Idx); 689 else 690 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 691 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 692 return; 693 } 694 695 // Spill the vector to the stack. 696 EVT VecVT = Vec.getValueType(); 697 EVT EltVT = VecVT.getVectorElementType(); 698 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 699 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0); 700 701 // Store the new element. This may be larger than the vector element type, 702 // so use a truncating store. 703 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 704 unsigned Alignment = 705 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext())); 706 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT); 707 708 // Load the Lo part from the stack slot. 709 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0); 710 711 // Increment the pointer to the other part. 712 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 713 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 714 DAG.getIntPtrConstant(IncrementSize)); 715 716 // Load the Hi part from the stack slot. 717 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false, 718 MinAlign(Alignment, IncrementSize)); 719} 720 721void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 722 SDValue &Hi) { 723 EVT LoVT, HiVT; 724 DebugLoc dl = N->getDebugLoc(); 725 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 726 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 727 Hi = DAG.getUNDEF(HiVT); 728} 729 730void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 731 SDValue &Hi) { 732 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 733 EVT LoVT, HiVT; 734 DebugLoc dl = LD->getDebugLoc(); 735 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 736 737 ISD::LoadExtType ExtType = LD->getExtensionType(); 738 SDValue Ch = LD->getChain(); 739 SDValue Ptr = LD->getBasePtr(); 740 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 741 const Value *SV = LD->getSrcValue(); 742 int SVOffset = LD->getSrcValueOffset(); 743 EVT MemoryVT = LD->getMemoryVT(); 744 unsigned Alignment = LD->getOriginalAlignment(); 745 bool isVolatile = LD->isVolatile(); 746 747 EVT LoMemVT, HiMemVT; 748 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 749 750 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, 751 SV, SVOffset, LoMemVT, isVolatile, Alignment); 752 753 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 754 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 755 DAG.getIntPtrConstant(IncrementSize)); 756 SVOffset += IncrementSize; 757 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, 758 SV, SVOffset, HiMemVT, isVolatile, Alignment); 759 760 // Build a factor node to remember that this load is independent of the 761 // other one. 762 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 763 Hi.getValue(1)); 764 765 // Legalized the chain result - switch anything that used the old chain to 766 // use the new one. 767 ReplaceValueWith(SDValue(LD, 1), Ch); 768} 769 770void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 771 EVT LoVT, HiVT; 772 DebugLoc DL = N->getDebugLoc(); 773 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 774 775 // Split the input. 776 EVT InVT = N->getOperand(0).getValueType(); 777 SDValue LL, LH, RL, RH; 778 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 779 LoVT.getVectorNumElements()); 780 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 781 DAG.getIntPtrConstant(0)); 782 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 783 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 784 785 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 786 DAG.getIntPtrConstant(0)); 787 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 788 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 789 790 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 791 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 792} 793 794void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 795 SDValue &Hi) { 796 // Get the dest types - they may not match the input types, e.g. int_to_fp. 797 EVT LoVT, HiVT; 798 DebugLoc dl = N->getDebugLoc(); 799 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 800 801 // Split the input. 802 EVT InVT = N->getOperand(0).getValueType(); 803 switch (getTypeAction(InVT)) { 804 default: llvm_unreachable("Unexpected type action!"); 805 case Legal: { 806 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 807 LoVT.getVectorNumElements()); 808 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 809 DAG.getIntPtrConstant(0)); 810 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 811 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 812 break; 813 } 814 case SplitVector: 815 GetSplitVector(N->getOperand(0), Lo, Hi); 816 break; 817 case WidenVector: { 818 // If the result needs to be split and the input needs to be widened, 819 // the two types must have different lengths. Use the widened result 820 // and extract from it to do the split. 821 SDValue InOp = GetWidenedVector(N->getOperand(0)); 822 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 823 LoVT.getVectorNumElements()); 824 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 825 DAG.getIntPtrConstant(0)); 826 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp, 827 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 828 break; 829 } 830 } 831 832 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 833 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 834} 835 836void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 837 SDValue &Lo, SDValue &Hi) { 838 // The low and high parts of the original input give four input vectors. 839 SDValue Inputs[4]; 840 DebugLoc dl = N->getDebugLoc(); 841 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 842 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 843 EVT NewVT = Inputs[0].getValueType(); 844 unsigned NewElts = NewVT.getVectorNumElements(); 845 846 // If Lo or Hi uses elements from at most two of the four input vectors, then 847 // express it as a vector shuffle of those two inputs. Otherwise extract the 848 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 849 SmallVector<int, 16> Ops; 850 for (unsigned High = 0; High < 2; ++High) { 851 SDValue &Output = High ? Hi : Lo; 852 853 // Build a shuffle mask for the output, discovering on the fly which 854 // input vectors to use as shuffle operands (recorded in InputUsed). 855 // If building a suitable shuffle vector proves too hard, then bail 856 // out with useBuildVector set. 857 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 858 unsigned FirstMaskIdx = High * NewElts; 859 bool useBuildVector = false; 860 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 861 // The mask element. This indexes into the input. 862 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 863 864 // The input vector this mask element indexes into. 865 unsigned Input = (unsigned)Idx / NewElts; 866 867 if (Input >= array_lengthof(Inputs)) { 868 // The mask element does not index into any input vector. 869 Ops.push_back(-1); 870 continue; 871 } 872 873 // Turn the index into an offset from the start of the input vector. 874 Idx -= Input * NewElts; 875 876 // Find or create a shuffle vector operand to hold this input. 877 unsigned OpNo; 878 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 879 if (InputUsed[OpNo] == Input) { 880 // This input vector is already an operand. 881 break; 882 } else if (InputUsed[OpNo] == -1U) { 883 // Create a new operand for this input vector. 884 InputUsed[OpNo] = Input; 885 break; 886 } 887 } 888 889 if (OpNo >= array_lengthof(InputUsed)) { 890 // More than two input vectors used! Give up on trying to create a 891 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 892 useBuildVector = true; 893 break; 894 } 895 896 // Add the mask index for the new shuffle vector. 897 Ops.push_back(Idx + OpNo * NewElts); 898 } 899 900 if (useBuildVector) { 901 EVT EltVT = NewVT.getVectorElementType(); 902 SmallVector<SDValue, 16> SVOps; 903 904 // Extract the input elements by hand. 905 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 906 // The mask element. This indexes into the input. 907 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 908 909 // The input vector this mask element indexes into. 910 unsigned Input = (unsigned)Idx / NewElts; 911 912 if (Input >= array_lengthof(Inputs)) { 913 // The mask element is "undef" or indexes off the end of the input. 914 SVOps.push_back(DAG.getUNDEF(EltVT)); 915 continue; 916 } 917 918 // Turn the index into an offset from the start of the input vector. 919 Idx -= Input * NewElts; 920 921 // Extract the vector element by hand. 922 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 923 Inputs[Input], DAG.getIntPtrConstant(Idx))); 924 } 925 926 // Construct the Lo/Hi output using a BUILD_VECTOR. 927 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 928 } else if (InputUsed[0] == -1U) { 929 // No input vectors were used! The result is undefined. 930 Output = DAG.getUNDEF(NewVT); 931 } else { 932 SDValue Op0 = Inputs[InputUsed[0]]; 933 // If only one input was used, use an undefined vector for the other. 934 SDValue Op1 = InputUsed[1] == -1U ? 935 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 936 // At least one input vector was used. Create a new shuffle vector. 937 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 938 } 939 940 Ops.clear(); 941 } 942} 943 944 945//===----------------------------------------------------------------------===// 946// Operand Vector Splitting 947//===----------------------------------------------------------------------===// 948 949/// SplitVectorOperand - This method is called when the specified operand of the 950/// specified node is found to need vector splitting. At this point, all of the 951/// result types of the node are known to be legal, but other operands of the 952/// node may need legalization as well as the specified one. 953bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 954 DEBUG(dbgs() << "Split node operand: "; 955 N->dump(&DAG); 956 dbgs() << "\n"); 957 SDValue Res = SDValue(); 958 959 if (Res.getNode() == 0) { 960 switch (N->getOpcode()) { 961 default: 962#ifndef NDEBUG 963 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 964 N->dump(&DAG); 965 dbgs() << "\n"; 966#endif 967 llvm_unreachable("Do not know how to split this operator's operand!"); 968 969 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 970 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 971 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 972 case ISD::STORE: 973 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 974 break; 975 976 case ISD::CTTZ: 977 case ISD::CTLZ: 978 case ISD::CTPOP: 979 case ISD::FP_TO_SINT: 980 case ISD::FP_TO_UINT: 981 case ISD::SINT_TO_FP: 982 case ISD::UINT_TO_FP: 983 case ISD::TRUNCATE: 984 case ISD::SIGN_EXTEND: 985 case ISD::ZERO_EXTEND: 986 case ISD::ANY_EXTEND: 987 Res = SplitVecOp_UnaryOp(N); 988 break; 989 } 990 } 991 992 // If the result is null, the sub-method took care of registering results etc. 993 if (!Res.getNode()) return false; 994 995 // If the result is N, the sub-method updated N in place. Tell the legalizer 996 // core about this. 997 if (Res.getNode() == N) 998 return true; 999 1000 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1001 "Invalid operand expansion"); 1002 1003 ReplaceValueWith(SDValue(N, 0), Res); 1004 return false; 1005} 1006 1007SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1008 // The result has a legal vector type, but the input needs splitting. 1009 EVT ResVT = N->getValueType(0); 1010 SDValue Lo, Hi; 1011 DebugLoc dl = N->getDebugLoc(); 1012 GetSplitVector(N->getOperand(0), Lo, Hi); 1013 EVT InVT = Lo.getValueType(); 1014 1015 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1016 InVT.getVectorNumElements()); 1017 1018 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1019 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1020 1021 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1022} 1023 1024SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 1025 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 1026 // end up being split all the way down to individual components. Convert the 1027 // split pieces into integers and reassemble. 1028 SDValue Lo, Hi; 1029 GetSplitVector(N->getOperand(0), Lo, Hi); 1030 Lo = BitConvertToInteger(Lo); 1031 Hi = BitConvertToInteger(Hi); 1032 1033 if (TLI.isBigEndian()) 1034 std::swap(Lo, Hi); 1035 1036 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0), 1037 JoinIntegers(Lo, Hi)); 1038} 1039 1040SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1041 // We know that the extracted result type is legal. For now, assume the index 1042 // is a constant. 1043 EVT SubVT = N->getValueType(0); 1044 SDValue Idx = N->getOperand(1); 1045 DebugLoc dl = N->getDebugLoc(); 1046 SDValue Lo, Hi; 1047 GetSplitVector(N->getOperand(0), Lo, Hi); 1048 1049 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1050 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1051 1052 if (IdxVal < LoElts) { 1053 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1054 "Extracted subvector crosses vector split!"); 1055 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1056 } else { 1057 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1058 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1059 } 1060} 1061 1062SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1063 SDValue Vec = N->getOperand(0); 1064 SDValue Idx = N->getOperand(1); 1065 EVT VecVT = Vec.getValueType(); 1066 1067 if (isa<ConstantSDNode>(Idx)) { 1068 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1069 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1070 1071 SDValue Lo, Hi; 1072 GetSplitVector(Vec, Lo, Hi); 1073 1074 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1075 1076 if (IdxVal < LoElts) 1077 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 1078 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 1079 DAG.getConstant(IdxVal - LoElts, 1080 Idx.getValueType())); 1081 } 1082 1083 // Store the vector to the stack. 1084 EVT EltVT = VecVT.getVectorElementType(); 1085 DebugLoc dl = N->getDebugLoc(); 1086 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1087 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1088 const Value *SV = PseudoSourceValue::getFixedStack(SPFI); 1089 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0); 1090 1091 // Load back the required element. 1092 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1093 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1094 SV, 0, EltVT); 1095} 1096 1097SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1098 assert(N->isUnindexed() && "Indexed store of vector?"); 1099 assert(OpNo == 1 && "Can only split the stored value"); 1100 DebugLoc dl = N->getDebugLoc(); 1101 1102 bool isTruncating = N->isTruncatingStore(); 1103 SDValue Ch = N->getChain(); 1104 SDValue Ptr = N->getBasePtr(); 1105 int SVOffset = N->getSrcValueOffset(); 1106 EVT MemoryVT = N->getMemoryVT(); 1107 unsigned Alignment = N->getOriginalAlignment(); 1108 bool isVol = N->isVolatile(); 1109 SDValue Lo, Hi; 1110 GetSplitVector(N->getOperand(1), Lo, Hi); 1111 1112 EVT LoMemVT, HiMemVT; 1113 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1114 1115 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1116 1117 if (isTruncating) 1118 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1119 LoMemVT, isVol, Alignment); 1120 else 1121 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, 1122 isVol, Alignment); 1123 1124 // Increment the pointer to the other half. 1125 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1126 DAG.getIntPtrConstant(IncrementSize)); 1127 SVOffset += IncrementSize; 1128 1129 if (isTruncating) 1130 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1131 HiMemVT, isVol, Alignment); 1132 else 1133 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, 1134 isVol, Alignment); 1135 1136 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 1137} 1138 1139 1140//===----------------------------------------------------------------------===// 1141// Result Vector Widening 1142//===----------------------------------------------------------------------===// 1143 1144void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1145 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1146 N->dump(&DAG); 1147 dbgs() << "\n"); 1148 1149 // See if the target wants to custom widen this node. 1150 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1151 return; 1152 1153 SDValue Res = SDValue(); 1154 switch (N->getOpcode()) { 1155 default: 1156#ifndef NDEBUG 1157 dbgs() << "WidenVectorResult #" << ResNo << ": "; 1158 N->dump(&DAG); 1159 dbgs() << "\n"; 1160#endif 1161 llvm_unreachable("Do not know how to widen the result of this operator!"); 1162 1163 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break; 1164 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1165 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1166 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1167 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1168 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1169 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1170 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1171 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1172 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1173 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1174 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1175 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1176 case ISD::VECTOR_SHUFFLE: 1177 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1178 break; 1179 case ISD::VSETCC: 1180 Res = WidenVecRes_VSETCC(N); 1181 break; 1182 1183 case ISD::ADD: 1184 case ISD::AND: 1185 case ISD::BSWAP: 1186 case ISD::FADD: 1187 case ISD::FCOPYSIGN: 1188 case ISD::FDIV: 1189 case ISD::FMUL: 1190 case ISD::FPOW: 1191 case ISD::FPOWI: 1192 case ISD::FREM: 1193 case ISD::FSUB: 1194 case ISD::MUL: 1195 case ISD::MULHS: 1196 case ISD::MULHU: 1197 case ISD::OR: 1198 case ISD::SDIV: 1199 case ISD::SREM: 1200 case ISD::UDIV: 1201 case ISD::UREM: 1202 case ISD::SUB: 1203 case ISD::XOR: 1204 Res = WidenVecRes_Binary(N); 1205 break; 1206 1207 case ISD::SHL: 1208 case ISD::SRA: 1209 case ISD::SRL: 1210 Res = WidenVecRes_Shift(N); 1211 break; 1212 1213 case ISD::FP_ROUND: 1214 case ISD::FP_TO_SINT: 1215 case ISD::FP_TO_UINT: 1216 case ISD::SINT_TO_FP: 1217 case ISD::UINT_TO_FP: 1218 case ISD::TRUNCATE: 1219 case ISD::SIGN_EXTEND: 1220 case ISD::ZERO_EXTEND: 1221 case ISD::ANY_EXTEND: 1222 Res = WidenVecRes_Convert(N); 1223 break; 1224 1225 case ISD::CTLZ: 1226 case ISD::CTPOP: 1227 case ISD::CTTZ: 1228 case ISD::FABS: 1229 case ISD::FCOS: 1230 case ISD::FNEG: 1231 case ISD::FSIN: 1232 case ISD::FSQRT: 1233 Res = WidenVecRes_Unary(N); 1234 break; 1235 } 1236 1237 // If Res is null, the sub-method took care of registering the result. 1238 if (Res.getNode()) 1239 SetWidenedVector(SDValue(N, ResNo), Res); 1240} 1241 1242SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1243 // Binary op widening. 1244 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1245 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1246 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1247 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2); 1248} 1249 1250SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1251 SDValue InOp = N->getOperand(0); 1252 DebugLoc dl = N->getDebugLoc(); 1253 1254 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1255 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1256 1257 EVT InVT = InOp.getValueType(); 1258 EVT InEltVT = InVT.getVectorElementType(); 1259 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1260 1261 unsigned Opcode = N->getOpcode(); 1262 unsigned InVTNumElts = InVT.getVectorNumElements(); 1263 1264 if (getTypeAction(InVT) == WidenVector) { 1265 InOp = GetWidenedVector(N->getOperand(0)); 1266 InVT = InOp.getValueType(); 1267 InVTNumElts = InVT.getVectorNumElements(); 1268 if (InVTNumElts == WidenNumElts) 1269 return DAG.getNode(Opcode, dl, WidenVT, InOp); 1270 } 1271 1272 if (TLI.isTypeLegal(InWidenVT)) { 1273 // Because the result and the input are different vector types, widening 1274 // the result could create a legal type but widening the input might make 1275 // it an illegal type that might lead to repeatedly splitting the input 1276 // and then widening it. To avoid this, we widen the input only if 1277 // it results in a legal type. 1278 if (WidenNumElts % InVTNumElts == 0) { 1279 // Widen the input and call convert on the widened input vector. 1280 unsigned NumConcat = WidenNumElts/InVTNumElts; 1281 SmallVector<SDValue, 16> Ops(NumConcat); 1282 Ops[0] = InOp; 1283 SDValue UndefVal = DAG.getUNDEF(InVT); 1284 for (unsigned i = 1; i != NumConcat; ++i) 1285 Ops[i] = UndefVal; 1286 return DAG.getNode(Opcode, dl, WidenVT, 1287 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, 1288 &Ops[0], NumConcat)); 1289 } 1290 1291 if (InVTNumElts % WidenNumElts == 0) { 1292 // Extract the input and convert the shorten input vector. 1293 return DAG.getNode(Opcode, dl, WidenVT, 1294 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, 1295 InOp, DAG.getIntPtrConstant(0))); 1296 } 1297 } 1298 1299 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1300 SmallVector<SDValue, 16> Ops(WidenNumElts); 1301 EVT EltVT = WidenVT.getVectorElementType(); 1302 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1303 unsigned i; 1304 for (i=0; i < MinElts; ++i) 1305 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1306 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1307 DAG.getIntPtrConstant(i))); 1308 1309 SDValue UndefVal = DAG.getUNDEF(EltVT); 1310 for (; i < WidenNumElts; ++i) 1311 Ops[i] = UndefVal; 1312 1313 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1314} 1315 1316SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1317 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1318 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1319 SDValue ShOp = N->getOperand(1); 1320 1321 EVT ShVT = ShOp.getValueType(); 1322 if (getTypeAction(ShVT) == WidenVector) { 1323 ShOp = GetWidenedVector(ShOp); 1324 ShVT = ShOp.getValueType(); 1325 } 1326 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(), 1327 WidenVT.getVectorNumElements()); 1328 if (ShVT != ShWidenVT) 1329 ShOp = ModifyToType(ShOp, ShWidenVT); 1330 1331 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1332} 1333 1334SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1335 // Unary op widening. 1336 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1337 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1338 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1339} 1340 1341SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1342 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1343 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1344 cast<VTSDNode>(N->getOperand(1))->getVT() 1345 .getVectorElementType(), 1346 WidenVT.getVectorNumElements()); 1347 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1348 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 1349 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1350} 1351 1352SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { 1353 SDValue InOp = N->getOperand(0); 1354 EVT InVT = InOp.getValueType(); 1355 EVT VT = N->getValueType(0); 1356 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1357 DebugLoc dl = N->getDebugLoc(); 1358 1359 switch (getTypeAction(InVT)) { 1360 default: 1361 assert(false && "Unknown type action!"); 1362 break; 1363 case Legal: 1364 break; 1365 case PromoteInteger: 1366 // If the InOp is promoted to the same size, convert it. Otherwise, 1367 // fall out of the switch and widen the promoted input. 1368 InOp = GetPromotedInteger(InOp); 1369 InVT = InOp.getValueType(); 1370 if (WidenVT.bitsEq(InVT)) 1371 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1372 break; 1373 case SoftenFloat: 1374 case ExpandInteger: 1375 case ExpandFloat: 1376 case ScalarizeVector: 1377 case SplitVector: 1378 break; 1379 case WidenVector: 1380 // If the InOp is widened to the same size, convert it. Otherwise, fall 1381 // out of the switch and widen the widened input. 1382 InOp = GetWidenedVector(InOp); 1383 InVT = InOp.getValueType(); 1384 if (WidenVT.bitsEq(InVT)) 1385 // The input widens to the same size. Convert to the widen value. 1386 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp); 1387 break; 1388 } 1389 1390 unsigned WidenSize = WidenVT.getSizeInBits(); 1391 unsigned InSize = InVT.getSizeInBits(); 1392 if (WidenSize % InSize == 0) { 1393 // Determine new input vector type. The new input vector type will use 1394 // the same element type (if its a vector) or use the input type as a 1395 // vector. It is the same size as the type to widen to. 1396 EVT NewInVT; 1397 unsigned NewNumElts = WidenSize / InSize; 1398 if (InVT.isVector()) { 1399 EVT InEltVT = InVT.getVectorElementType(); 1400 NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits()); 1401 } else { 1402 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1403 } 1404 1405 if (TLI.isTypeLegal(NewInVT)) { 1406 // Because the result and the input are different vector types, widening 1407 // the result could create a legal type but widening the input might make 1408 // it an illegal type that might lead to repeatedly splitting the input 1409 // and then widening it. To avoid this, we widen the input only if 1410 // it results in a legal type. 1411 SmallVector<SDValue, 16> Ops(NewNumElts); 1412 SDValue UndefVal = DAG.getUNDEF(InVT); 1413 Ops[0] = InOp; 1414 for (unsigned i = 1; i < NewNumElts; ++i) 1415 Ops[i] = UndefVal; 1416 1417 SDValue NewVec; 1418 if (InVT.isVector()) 1419 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1420 NewInVT, &Ops[0], NewNumElts); 1421 else 1422 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1423 NewInVT, &Ops[0], NewNumElts); 1424 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec); 1425 } 1426 } 1427 1428 return CreateStackStoreLoad(InOp, WidenVT); 1429} 1430 1431SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1432 DebugLoc dl = N->getDebugLoc(); 1433 // Build a vector with undefined for the new nodes. 1434 EVT VT = N->getValueType(0); 1435 EVT EltVT = VT.getVectorElementType(); 1436 unsigned NumElts = VT.getVectorNumElements(); 1437 1438 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1439 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1440 1441 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1442 NewOps.reserve(WidenNumElts); 1443 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1444 NewOps.push_back(DAG.getUNDEF(EltVT)); 1445 1446 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1447} 1448 1449SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1450 EVT InVT = N->getOperand(0).getValueType(); 1451 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1452 DebugLoc dl = N->getDebugLoc(); 1453 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1454 unsigned NumOperands = N->getNumOperands(); 1455 1456 bool InputWidened = false; // Indicates we need to widen the input. 1457 if (getTypeAction(InVT) != WidenVector) { 1458 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1459 // Add undef vectors to widen to correct length. 1460 unsigned NumConcat = WidenVT.getVectorNumElements() / 1461 InVT.getVectorNumElements(); 1462 SDValue UndefVal = DAG.getUNDEF(InVT); 1463 SmallVector<SDValue, 16> Ops(NumConcat); 1464 for (unsigned i=0; i < NumOperands; ++i) 1465 Ops[i] = N->getOperand(i); 1466 for (unsigned i = NumOperands; i != NumConcat; ++i) 1467 Ops[i] = UndefVal; 1468 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1469 } 1470 } else { 1471 InputWidened = true; 1472 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1473 // The inputs and the result are widen to the same value. 1474 unsigned i; 1475 for (i=1; i < NumOperands; ++i) 1476 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1477 break; 1478 1479 if (i > NumOperands) 1480 // Everything but the first operand is an UNDEF so just return the 1481 // widened first operand. 1482 return GetWidenedVector(N->getOperand(0)); 1483 1484 if (NumOperands == 2) { 1485 // Replace concat of two operands with a shuffle. 1486 SmallVector<int, 16> MaskOps(WidenNumElts); 1487 for (unsigned i=0; i < WidenNumElts/2; ++i) { 1488 MaskOps[i] = i; 1489 MaskOps[i+WidenNumElts/2] = i+WidenNumElts; 1490 } 1491 return DAG.getVectorShuffle(WidenVT, dl, 1492 GetWidenedVector(N->getOperand(0)), 1493 GetWidenedVector(N->getOperand(1)), 1494 &MaskOps[0]); 1495 } 1496 } 1497 } 1498 1499 // Fall back to use extracts and build vector. 1500 EVT EltVT = WidenVT.getVectorElementType(); 1501 unsigned NumInElts = InVT.getVectorNumElements(); 1502 SmallVector<SDValue, 16> Ops(WidenNumElts); 1503 unsigned Idx = 0; 1504 for (unsigned i=0; i < NumOperands; ++i) { 1505 SDValue InOp = N->getOperand(i); 1506 if (InputWidened) 1507 InOp = GetWidenedVector(InOp); 1508 for (unsigned j=0; j < NumInElts; ++j) 1509 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1510 DAG.getIntPtrConstant(j)); 1511 } 1512 SDValue UndefVal = DAG.getUNDEF(EltVT); 1513 for (; Idx < WidenNumElts; ++Idx) 1514 Ops[Idx] = UndefVal; 1515 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1516} 1517 1518SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1519 DebugLoc dl = N->getDebugLoc(); 1520 SDValue InOp = N->getOperand(0); 1521 SDValue RndOp = N->getOperand(3); 1522 SDValue SatOp = N->getOperand(4); 1523 1524 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1525 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1526 1527 EVT InVT = InOp.getValueType(); 1528 EVT InEltVT = InVT.getVectorElementType(); 1529 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1530 1531 SDValue DTyOp = DAG.getValueType(WidenVT); 1532 SDValue STyOp = DAG.getValueType(InWidenVT); 1533 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1534 1535 unsigned InVTNumElts = InVT.getVectorNumElements(); 1536 if (getTypeAction(InVT) == WidenVector) { 1537 InOp = GetWidenedVector(InOp); 1538 InVT = InOp.getValueType(); 1539 InVTNumElts = InVT.getVectorNumElements(); 1540 if (InVTNumElts == WidenNumElts) 1541 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1542 SatOp, CvtCode); 1543 } 1544 1545 if (TLI.isTypeLegal(InWidenVT)) { 1546 // Because the result and the input are different vector types, widening 1547 // the result could create a legal type but widening the input might make 1548 // it an illegal type that might lead to repeatedly splitting the input 1549 // and then widening it. To avoid this, we widen the input only if 1550 // it results in a legal type. 1551 if (WidenNumElts % InVTNumElts == 0) { 1552 // Widen the input and call convert on the widened input vector. 1553 unsigned NumConcat = WidenNumElts/InVTNumElts; 1554 SmallVector<SDValue, 16> Ops(NumConcat); 1555 Ops[0] = InOp; 1556 SDValue UndefVal = DAG.getUNDEF(InVT); 1557 for (unsigned i = 1; i != NumConcat; ++i) { 1558 Ops[i] = UndefVal; 1559 } 1560 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1561 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1562 SatOp, CvtCode); 1563 } 1564 1565 if (InVTNumElts % WidenNumElts == 0) { 1566 // Extract the input and convert the shorten input vector. 1567 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1568 DAG.getIntPtrConstant(0)); 1569 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1570 SatOp, CvtCode); 1571 } 1572 } 1573 1574 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1575 SmallVector<SDValue, 16> Ops(WidenNumElts); 1576 EVT EltVT = WidenVT.getVectorElementType(); 1577 DTyOp = DAG.getValueType(EltVT); 1578 STyOp = DAG.getValueType(InEltVT); 1579 1580 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1581 unsigned i; 1582 for (i=0; i < MinElts; ++i) { 1583 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1584 DAG.getIntPtrConstant(i)); 1585 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1586 SatOp, CvtCode); 1587 } 1588 1589 SDValue UndefVal = DAG.getUNDEF(EltVT); 1590 for (; i < WidenNumElts; ++i) 1591 Ops[i] = UndefVal; 1592 1593 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1594} 1595 1596SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1597 EVT VT = N->getValueType(0); 1598 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1599 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1600 SDValue InOp = N->getOperand(0); 1601 SDValue Idx = N->getOperand(1); 1602 DebugLoc dl = N->getDebugLoc(); 1603 1604 if (getTypeAction(InOp.getValueType()) == WidenVector) 1605 InOp = GetWidenedVector(InOp); 1606 1607 EVT InVT = InOp.getValueType(); 1608 1609 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 1610 if (CIdx) { 1611 unsigned IdxVal = CIdx->getZExtValue(); 1612 // Check if we can just return the input vector after widening. 1613 if (IdxVal == 0 && InVT == WidenVT) 1614 return InOp; 1615 1616 // Check if we can extract from the vector. 1617 unsigned InNumElts = InVT.getVectorNumElements(); 1618 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1619 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1620 } 1621 1622 // We could try widening the input to the right length but for now, extract 1623 // the original elements, fill the rest with undefs and build a vector. 1624 SmallVector<SDValue, 16> Ops(WidenNumElts); 1625 EVT EltVT = VT.getVectorElementType(); 1626 EVT IdxVT = Idx.getValueType(); 1627 unsigned NumElts = VT.getVectorNumElements(); 1628 unsigned i; 1629 if (CIdx) { 1630 unsigned IdxVal = CIdx->getZExtValue(); 1631 for (i=0; i < NumElts; ++i) 1632 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1633 DAG.getConstant(IdxVal+i, IdxVT)); 1634 } else { 1635 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx); 1636 for (i=1; i < NumElts; ++i) { 1637 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1638 DAG.getConstant(i, IdxVT)); 1639 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx); 1640 } 1641 } 1642 1643 SDValue UndefVal = DAG.getUNDEF(EltVT); 1644 for (; i < WidenNumElts; ++i) 1645 Ops[i] = UndefVal; 1646 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1647} 1648 1649SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1650 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1651 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1652 InOp.getValueType(), InOp, 1653 N->getOperand(1), N->getOperand(2)); 1654} 1655 1656SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1657 LoadSDNode *LD = cast<LoadSDNode>(N); 1658 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1659 EVT LdVT = LD->getMemoryVT(); 1660 DebugLoc dl = N->getDebugLoc(); 1661 assert(LdVT.isVector() && WidenVT.isVector()); 1662 1663 // Load information 1664 SDValue Chain = LD->getChain(); 1665 SDValue BasePtr = LD->getBasePtr(); 1666 int SVOffset = LD->getSrcValueOffset(); 1667 unsigned Align = LD->getAlignment(); 1668 bool isVolatile = LD->isVolatile(); 1669 const Value *SV = LD->getSrcValue(); 1670 ISD::LoadExtType ExtType = LD->getExtensionType(); 1671 1672 SDValue Result; 1673 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1674 if (ExtType != ISD::NON_EXTLOAD) { 1675 // For extension loads, we can not play the tricks of chopping legal 1676 // vector types and bit cast it to the right type. Instead, we unroll 1677 // the load and build a vector. 1678 EVT EltVT = WidenVT.getVectorElementType(); 1679 EVT LdEltVT = LdVT.getVectorElementType(); 1680 unsigned NumElts = LdVT.getVectorNumElements(); 1681 1682 // Load each element and widen 1683 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1684 SmallVector<SDValue, 16> Ops(WidenNumElts); 1685 unsigned Increment = LdEltVT.getSizeInBits() / 8; 1686 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset, 1687 LdEltVT, isVolatile, Align); 1688 LdChain.push_back(Ops[0].getValue(1)); 1689 unsigned i = 0, Offset = Increment; 1690 for (i=1; i < NumElts; ++i, Offset += Increment) { 1691 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1692 BasePtr, DAG.getIntPtrConstant(Offset)); 1693 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV, 1694 SVOffset + Offset, LdEltVT, isVolatile, Align); 1695 LdChain.push_back(Ops[i].getValue(1)); 1696 } 1697 1698 // Fill the rest with undefs 1699 SDValue UndefVal = DAG.getUNDEF(EltVT); 1700 for (; i != WidenNumElts; ++i) 1701 Ops[i] = UndefVal; 1702 1703 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 1704 } else { 1705 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 1706 unsigned int LdWidth = LdVT.getSizeInBits(); 1707 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, 1708 Align, isVolatile, LdWidth, WidenVT, dl); 1709 } 1710 1711 // If we generate a single load, we can use that for the chain. Otherwise, 1712 // build a factor node to remember the multiple loads are independent and 1713 // chain to that. 1714 SDValue NewChain; 1715 if (LdChain.size() == 1) 1716 NewChain = LdChain[0]; 1717 else 1718 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0], 1719 LdChain.size()); 1720 1721 // Modified the chain - switch anything that used the old chain to use 1722 // the new one. 1723 ReplaceValueWith(SDValue(N, 1), NewChain); 1724 1725 return Result; 1726} 1727 1728SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1729 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1730 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1731 WidenVT, N->getOperand(0)); 1732} 1733 1734SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1735 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1736 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1737 1738 SDValue Cond1 = N->getOperand(0); 1739 EVT CondVT = Cond1.getValueType(); 1740 if (CondVT.isVector()) { 1741 EVT CondEltVT = CondVT.getVectorElementType(); 1742 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts); 1743 if (getTypeAction(CondVT) == WidenVector) 1744 Cond1 = GetWidenedVector(Cond1); 1745 1746 if (Cond1.getValueType() != CondWidenVT) 1747 Cond1 = ModifyToType(Cond1, CondWidenVT); 1748 } 1749 1750 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1751 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 1752 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 1753 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 1754 WidenVT, Cond1, InOp1, InOp2); 1755} 1756 1757SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 1758 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 1759 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 1760 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 1761 InOp1.getValueType(), N->getOperand(0), 1762 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 1763} 1764 1765SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 1766 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1767 return DAG.getUNDEF(WidenVT); 1768} 1769 1770SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 1771 EVT VT = N->getValueType(0); 1772 DebugLoc dl = N->getDebugLoc(); 1773 1774 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1775 unsigned NumElts = VT.getVectorNumElements(); 1776 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1777 1778 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1779 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1780 1781 // Adjust mask based on new input vector length. 1782 SmallVector<int, 16> NewMask; 1783 for (unsigned i = 0; i != NumElts; ++i) { 1784 int Idx = N->getMaskElt(i); 1785 if (Idx < (int)NumElts) 1786 NewMask.push_back(Idx); 1787 else 1788 NewMask.push_back(Idx - NumElts + WidenNumElts); 1789 } 1790 for (unsigned i = NumElts; i != WidenNumElts; ++i) 1791 NewMask.push_back(-1); 1792 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 1793} 1794 1795SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 1796 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1797 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1798 1799 SDValue InOp1 = N->getOperand(0); 1800 EVT InVT = InOp1.getValueType(); 1801 assert(InVT.isVector() && "can not widen non vector type"); 1802 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); 1803 InOp1 = GetWidenedVector(InOp1); 1804 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1805 1806 // Assume that the input and output will be widen appropriately. If not, 1807 // we will have to unroll it at some point. 1808 assert(InOp1.getValueType() == WidenInVT && 1809 InOp2.getValueType() == WidenInVT && 1810 "Input not widened to expected type!"); 1811 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), 1812 WidenVT, InOp1, InOp2, N->getOperand(2)); 1813} 1814 1815 1816//===----------------------------------------------------------------------===// 1817// Widen Vector Operand 1818//===----------------------------------------------------------------------===// 1819bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 1820 DEBUG(dbgs() << "Widen node operand " << ResNo << ": "; 1821 N->dump(&DAG); 1822 dbgs() << "\n"); 1823 SDValue Res = SDValue(); 1824 1825 switch (N->getOpcode()) { 1826 default: 1827#ifndef NDEBUG 1828 dbgs() << "WidenVectorOperand op #" << ResNo << ": "; 1829 N->dump(&DAG); 1830 dbgs() << "\n"; 1831#endif 1832 llvm_unreachable("Do not know how to widen this operator's operand!"); 1833 1834 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break; 1835 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 1836 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 1837 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 1838 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 1839 1840 case ISD::FP_ROUND: 1841 case ISD::FP_TO_SINT: 1842 case ISD::FP_TO_UINT: 1843 case ISD::SINT_TO_FP: 1844 case ISD::UINT_TO_FP: 1845 case ISD::TRUNCATE: 1846 case ISD::SIGN_EXTEND: 1847 case ISD::ZERO_EXTEND: 1848 case ISD::ANY_EXTEND: 1849 Res = WidenVecOp_Convert(N); 1850 break; 1851 } 1852 1853 // If Res is null, the sub-method took care of registering the result. 1854 if (!Res.getNode()) return false; 1855 1856 // If the result is N, the sub-method updated N in place. Tell the legalizer 1857 // core about this. 1858 if (Res.getNode() == N) 1859 return true; 1860 1861 1862 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1863 "Invalid operand expansion"); 1864 1865 ReplaceValueWith(SDValue(N, 0), Res); 1866 return false; 1867} 1868 1869SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 1870 // Since the result is legal and the input is illegal, it is unlikely 1871 // that we can fix the input to a legal type so unroll the convert 1872 // into some scalar code and create a nasty build vector. 1873 EVT VT = N->getValueType(0); 1874 EVT EltVT = VT.getVectorElementType(); 1875 DebugLoc dl = N->getDebugLoc(); 1876 unsigned NumElts = VT.getVectorNumElements(); 1877 SDValue InOp = N->getOperand(0); 1878 if (getTypeAction(InOp.getValueType()) == WidenVector) 1879 InOp = GetWidenedVector(InOp); 1880 EVT InVT = InOp.getValueType(); 1881 EVT InEltVT = InVT.getVectorElementType(); 1882 1883 unsigned Opcode = N->getOpcode(); 1884 SmallVector<SDValue, 16> Ops(NumElts); 1885 for (unsigned i=0; i < NumElts; ++i) 1886 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 1887 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1888 DAG.getIntPtrConstant(i))); 1889 1890 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1891} 1892 1893SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) { 1894 EVT VT = N->getValueType(0); 1895 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1896 EVT InWidenVT = InOp.getValueType(); 1897 DebugLoc dl = N->getDebugLoc(); 1898 1899 // Check if we can convert between two legal vector types and extract. 1900 unsigned InWidenSize = InWidenVT.getSizeInBits(); 1901 unsigned Size = VT.getSizeInBits(); 1902 if (InWidenSize % Size == 0 && !VT.isVector()) { 1903 unsigned NewNumElts = InWidenSize / Size; 1904 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 1905 if (TLI.isTypeLegal(NewVT)) { 1906 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp); 1907 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 1908 DAG.getIntPtrConstant(0)); 1909 } 1910 } 1911 1912 return CreateStackStoreLoad(InOp, VT); 1913} 1914 1915SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 1916 // If the input vector is not legal, it is likely that we will not find a 1917 // legal vector of the same size. Replace the concatenate vector with a 1918 // nasty build vector. 1919 EVT VT = N->getValueType(0); 1920 EVT EltVT = VT.getVectorElementType(); 1921 DebugLoc dl = N->getDebugLoc(); 1922 unsigned NumElts = VT.getVectorNumElements(); 1923 SmallVector<SDValue, 16> Ops(NumElts); 1924 1925 EVT InVT = N->getOperand(0).getValueType(); 1926 unsigned NumInElts = InVT.getVectorNumElements(); 1927 1928 unsigned Idx = 0; 1929 unsigned NumOperands = N->getNumOperands(); 1930 for (unsigned i=0; i < NumOperands; ++i) { 1931 SDValue InOp = N->getOperand(i); 1932 if (getTypeAction(InOp.getValueType()) == WidenVector) 1933 InOp = GetWidenedVector(InOp); 1934 for (unsigned j=0; j < NumInElts; ++j) 1935 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1936 DAG.getIntPtrConstant(j)); 1937 } 1938 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 1939} 1940 1941SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1942 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1943 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), 1944 N->getValueType(0), InOp, N->getOperand(1)); 1945} 1946 1947SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1948 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1949 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 1950 N->getValueType(0), InOp, N->getOperand(1)); 1951} 1952 1953SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 1954 // We have to widen the value but we want only to store the original 1955 // vector type. 1956 StoreSDNode *ST = cast<StoreSDNode>(N); 1957 SDValue Chain = ST->getChain(); 1958 SDValue BasePtr = ST->getBasePtr(); 1959 const Value *SV = ST->getSrcValue(); 1960 int SVOffset = ST->getSrcValueOffset(); 1961 unsigned Align = ST->getAlignment(); 1962 bool isVolatile = ST->isVolatile(); 1963 SDValue ValOp = GetWidenedVector(ST->getValue()); 1964 DebugLoc dl = N->getDebugLoc(); 1965 1966 EVT StVT = ST->getMemoryVT(); 1967 EVT ValVT = ValOp.getValueType(); 1968 // It must be true that we the widen vector type is bigger than where 1969 // we need to store. 1970 assert(StVT.isVector() && ValOp.getValueType().isVector()); 1971 assert(StVT.bitsLT(ValOp.getValueType())); 1972 1973 SmallVector<SDValue, 16> StChain; 1974 if (ST->isTruncatingStore()) { 1975 // For truncating stores, we can not play the tricks of chopping legal 1976 // vector types and bit cast it to the right type. Instead, we unroll 1977 // the store. 1978 EVT StEltVT = StVT.getVectorElementType(); 1979 EVT ValEltVT = ValVT.getVectorElementType(); 1980 unsigned Increment = ValEltVT.getSizeInBits() / 8; 1981 unsigned NumElts = StVT.getVectorNumElements(); 1982 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1983 DAG.getIntPtrConstant(0)); 1984 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV, 1985 SVOffset, StEltVT, 1986 isVolatile, Align)); 1987 unsigned Offset = Increment; 1988 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 1989 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 1990 BasePtr, DAG.getIntPtrConstant(Offset)); 1991 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 1992 DAG.getIntPtrConstant(0)); 1993 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV, 1994 SVOffset + Offset, StEltVT, 1995 isVolatile, MinAlign(Align, Offset))); 1996 } 1997 } 1998 else { 1999 assert(StVT.getVectorElementType() == ValVT.getVectorElementType()); 2000 // Store value 2001 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset, 2002 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl); 2003 } 2004 if (StChain.size() == 1) 2005 return StChain[0]; 2006 else 2007 return DAG.getNode(ISD::TokenFactor, dl, 2008 MVT::Other,&StChain[0],StChain.size()); 2009} 2010 2011//===----------------------------------------------------------------------===// 2012// Vector Widening Utilities 2013//===----------------------------------------------------------------------===// 2014 2015 2016// Utility function to find a vector type and its associated element 2017// type from a preferred width and whose vector type must be the same size 2018// as the VecVT. 2019// TLI: Target lowering used to determine legal types. 2020// Width: Preferred width to store. 2021// VecVT: Vector value type whose size we must match. 2022// Returns NewVecVT and NewEltVT - the vector type and its associated 2023// element type. 2024static void FindAssocWidenVecType(SelectionDAG& DAG, 2025 const TargetLowering &TLI, unsigned Width, 2026 EVT VecVT, 2027 EVT& NewEltVT, EVT& NewVecVT) { 2028 unsigned EltWidth = Width + 1; 2029 if (TLI.isTypeLegal(VecVT)) { 2030 // We start with the preferred with, making it a power of 2 and find a 2031 // legal vector type of that width. If not, we reduce it by another of 2. 2032 // For incoming type is legal, this process will end as a vector of the 2033 // smallest loadable type should always be legal. 2034 do { 2035 assert(EltWidth > 0); 2036 EltWidth = 1 << Log2_32(EltWidth - 1); 2037 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 2038 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2039 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 2040 } while (!TLI.isTypeLegal(NewVecVT) || 2041 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2042 } else { 2043 // The incoming vector type is illegal and is the result of widening 2044 // a vector to a power of 2. In this case, we will use the preferred 2045 // with as long as it is a multiple of the incoming vector length. 2046 // The legalization process will eventually make this into a legal type 2047 // and remove the illegal bit converts (which would turn to stack converts 2048 // if they are allow to exist). 2049 do { 2050 assert(EltWidth > 0); 2051 EltWidth = 1 << Log2_32(EltWidth - 1); 2052 NewEltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth); 2053 unsigned NumElts = VecVT.getSizeInBits() / EltWidth; 2054 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, NumElts); 2055 } while (!TLI.isTypeLegal(NewEltVT) || 2056 VecVT.getSizeInBits() != NewVecVT.getSizeInBits()); 2057 } 2058} 2059 2060SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, 2061 SDValue Chain, 2062 SDValue BasePtr, 2063 const Value *SV, 2064 int SVOffset, 2065 unsigned Alignment, 2066 bool isVolatile, 2067 unsigned LdWidth, 2068 EVT ResType, 2069 DebugLoc dl) { 2070 // The strategy assumes that we can efficiently load powers of two widths. 2071 // The routines chops the vector into the largest power of 2 load and 2072 // can be inserted into a legal vector and then cast the result into the 2073 // vector type we want. This avoids unnecessary stack converts. 2074 2075 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and 2076 // the load is nonvolatile, we an use a wider load for the value. 2077 2078 // Find the vector type that can load from. 2079 EVT NewEltVT, NewVecVT; 2080 unsigned NewEltVTWidth; 2081 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2082 NewEltVTWidth = NewEltVT.getSizeInBits(); 2083 2084 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset, 2085 isVolatile, Alignment); 2086 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2087 LdChain.push_back(LdOp.getValue(1)); 2088 2089 // Check if we can load the element with one instruction 2090 if (LdWidth == NewEltVTWidth) { 2091 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2092 } 2093 2094 unsigned Idx = 1; 2095 LdWidth -= NewEltVTWidth; 2096 unsigned Offset = 0; 2097 2098 while (LdWidth > 0) { 2099 unsigned Increment = NewEltVTWidth / 8; 2100 Offset += Increment; 2101 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2102 DAG.getIntPtrConstant(Increment)); 2103 2104 if (LdWidth < NewEltVTWidth) { 2105 // Our current type we are using is too large, use a smaller size by 2106 // using a smaller power of 2 2107 unsigned oNewEltVTWidth = NewEltVTWidth; 2108 FindAssocWidenVecType(DAG, TLI, LdWidth, ResType, NewEltVT, NewVecVT); 2109 NewEltVTWidth = NewEltVT.getSizeInBits(); 2110 // Readjust position and vector position based on new load type 2111 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2112 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2113 } 2114 2115 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, 2116 SVOffset+Offset, isVolatile, 2117 MinAlign(Alignment, Offset)); 2118 LdChain.push_back(LdOp.getValue(1)); 2119 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp, 2120 DAG.getIntPtrConstant(Idx++)); 2121 2122 LdWidth -= NewEltVTWidth; 2123 } 2124 2125 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); 2126} 2127 2128void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2129 SDValue Chain, 2130 SDValue BasePtr, 2131 const Value *SV, 2132 int SVOffset, 2133 unsigned Alignment, 2134 bool isVolatile, 2135 SDValue ValOp, 2136 unsigned StWidth, 2137 DebugLoc dl) { 2138 // Breaks the stores into a series of power of 2 width stores. For any 2139 // width, we convert the vector to the vector of element size that we 2140 // want to store. This avoids requiring a stack convert. 2141 2142 // Find a width of the element type we can store with 2143 EVT WidenVT = ValOp.getValueType(); 2144 EVT NewEltVT, NewVecVT; 2145 2146 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2147 unsigned NewEltVTWidth = NewEltVT.getSizeInBits(); 2148 2149 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp); 2150 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2151 DAG.getIntPtrConstant(0)); 2152 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, 2153 isVolatile, Alignment); 2154 StChain.push_back(StOp); 2155 2156 // Check if we are done 2157 if (StWidth == NewEltVTWidth) { 2158 return; 2159 } 2160 2161 unsigned Idx = 1; 2162 StWidth -= NewEltVTWidth; 2163 unsigned Offset = 0; 2164 2165 while (StWidth > 0) { 2166 unsigned Increment = NewEltVTWidth / 8; 2167 Offset += Increment; 2168 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2169 DAG.getIntPtrConstant(Increment)); 2170 2171 if (StWidth < NewEltVTWidth) { 2172 // Our current type we are using is too large, use a smaller size by 2173 // using a smaller power of 2 2174 unsigned oNewEltVTWidth = NewEltVTWidth; 2175 FindAssocWidenVecType(DAG, TLI, StWidth, WidenVT, NewEltVT, NewVecVT); 2176 NewEltVTWidth = NewEltVT.getSizeInBits(); 2177 // Readjust position and vector position based on new load type 2178 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth); 2179 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp); 2180 } 2181 2182 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp, 2183 DAG.getIntPtrConstant(Idx++)); 2184 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, 2185 SVOffset + Offset, isVolatile, 2186 MinAlign(Alignment, Offset))); 2187 StWidth -= NewEltVTWidth; 2188 } 2189} 2190 2191/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2192/// input vector must have the same element type as NVT. 2193SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2194 // Note that InOp might have been widened so it might already have 2195 // the right width or it might need be narrowed. 2196 EVT InVT = InOp.getValueType(); 2197 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2198 "input and widen element type must match"); 2199 DebugLoc dl = InOp.getDebugLoc(); 2200 2201 // Check if InOp already has the right width. 2202 if (InVT == NVT) 2203 return InOp; 2204 2205 unsigned InNumElts = InVT.getVectorNumElements(); 2206 unsigned WidenNumElts = NVT.getVectorNumElements(); 2207 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2208 unsigned NumConcat = WidenNumElts / InNumElts; 2209 SmallVector<SDValue, 16> Ops(NumConcat); 2210 SDValue UndefVal = DAG.getUNDEF(InVT); 2211 Ops[0] = InOp; 2212 for (unsigned i = 1; i != NumConcat; ++i) 2213 Ops[i] = UndefVal; 2214 2215 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2216 } 2217 2218 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2219 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2220 DAG.getIntPtrConstant(0)); 2221 2222 // Fall back to extract and build. 2223 SmallVector<SDValue, 16> Ops(WidenNumElts); 2224 EVT EltVT = NVT.getVectorElementType(); 2225 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2226 unsigned Idx; 2227 for (Idx = 0; Idx < MinNumElts; ++Idx) 2228 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2229 DAG.getIntPtrConstant(Idx)); 2230 2231 SDValue UndefVal = DAG.getUNDEF(EltVT); 2232 for ( ; Idx < WidenNumElts; ++Idx) 2233 Ops[Idx] = UndefVal; 2234 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2235} 2236