LegalizeVectorTypes.cpp revision 11e56cb4dc73bbb0bdc083042657ea3a5aad63f2
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in multiple vectors of a smaller type. For example, 19// implementing <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24using namespace llvm; 25 26//===----------------------------------------------------------------------===// 27// Result Vector Scalarization: <1 x ty> -> ty. 28//===----------------------------------------------------------------------===// 29 30void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 31 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 32 cerr << "\n"); 33 SDOperand R = SDOperand(); 34 35 switch (N->getOpcode()) { 36 default: 37#ifndef NDEBUG 38 cerr << "ScalarizeVectorResult #" << ResNo << ": "; 39 N->dump(&DAG); cerr << "\n"; 40#endif 41 assert(0 && "Do not know how to scalarize the result of this operator!"); 42 abort(); 43 44 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 45 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 46 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 47 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 48 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 49 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 50 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 51 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 52 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 53 54 case ISD::ADD: 55 case ISD::FADD: 56 case ISD::SUB: 57 case ISD::FSUB: 58 case ISD::MUL: 59 case ISD::FMUL: 60 case ISD::SDIV: 61 case ISD::UDIV: 62 case ISD::FDIV: 63 case ISD::SREM: 64 case ISD::UREM: 65 case ISD::FREM: 66 case ISD::FPOW: 67 case ISD::AND: 68 case ISD::OR: 69 case ISD::XOR: R = ScalarizeVecRes_BinOp(N); break; 70 71 case ISD::FNEG: 72 case ISD::FABS: 73 case ISD::FSQRT: 74 case ISD::FSIN: 75 case ISD::FCOS: R = ScalarizeVecRes_UnaryOp(N); break; 76 } 77 78 // If R is null, the sub-method took care of registering the result. 79 if (R.Val) 80 SetScalarizedVector(SDOperand(N, ResNo), R); 81} 82 83SDOperand DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 84 SDOperand LHS = GetScalarizedVector(N->getOperand(0)); 85 SDOperand RHS = GetScalarizedVector(N->getOperand(1)); 86 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 87} 88 89SDOperand DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 90 MVT NewVT = N->getValueType(0).getVectorElementType(); 91 return DAG.getNode(ISD::BIT_CONVERT, NewVT, N->getOperand(0)); 92} 93 94SDOperand DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 95 SDOperand Op = GetScalarizedVector(N->getOperand(0)); 96 return DAG.getNode(ISD::FPOWI, Op.getValueType(), Op, N->getOperand(1)); 97} 98 99SDOperand DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 100 // The value to insert may have a wider type than the vector element type, 101 // so be sure to truncate it to the element type if necessary. 102 SDOperand Op = N->getOperand(1); 103 MVT EltVT = N->getValueType(0).getVectorElementType(); 104 if (Op.getValueType() != EltVT) 105 // FIXME: Can this happen for floating point types? 106 Op = DAG.getNode(ISD::TRUNCATE, EltVT, Op); 107 return Op; 108} 109 110SDOperand DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 111 assert(ISD::isNormalLoad(N) && "Extending load of one-element vector?"); 112 SDOperand Result = DAG.getLoad(N->getValueType(0).getVectorElementType(), 113 N->getChain(), N->getBasePtr(), 114 N->getSrcValue(), N->getSrcValueOffset(), 115 N->isVolatile(), N->getAlignment()); 116 117 // Legalized the chain result - switch anything that used the old chain to 118 // use the new one. 119 ReplaceValueWith(SDOperand(N, 1), Result.getValue(1)); 120 return Result; 121} 122 123SDOperand DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 124 SDOperand Op = GetScalarizedVector(N->getOperand(0)); 125 return DAG.getNode(N->getOpcode(), Op.getValueType(), Op); 126} 127 128SDOperand DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 129 return DAG.getNode(ISD::UNDEF, N->getValueType(0).getVectorElementType()); 130} 131 132SDOperand DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 133 SDOperand LHS = GetScalarizedVector(N->getOperand(1)); 134 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0), LHS, 135 GetScalarizedVector(N->getOperand(2))); 136} 137 138SDOperand DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 139 // Figure out if the scalar is the LHS or RHS and return it. 140 SDOperand EltNum = N->getOperand(2).getOperand(0); 141 unsigned Op = cast<ConstantSDNode>(EltNum)->getValue() != 0; 142 return GetScalarizedVector(N->getOperand(Op)); 143} 144 145SDOperand DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 146 MVT NewVT = N->getValueType(0).getVectorElementType(); 147 SDOperand LHS = GetScalarizedVector(N->getOperand(0)); 148 SDOperand RHS = GetScalarizedVector(N->getOperand(1)); 149 LHS = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHS), LHS, RHS, 150 N->getOperand(2)); 151 return 152 DAG.getNode(ISD::SELECT, NewVT, LHS, 153 DAG.getConstant(APInt::getAllOnesValue(NewVT.getSizeInBits()), 154 NewVT), 155 DAG.getConstant(0ULL, NewVT)); 156} 157 158 159//===----------------------------------------------------------------------===// 160// Operand Vector Scalarization <1 x ty> -> ty. 161//===----------------------------------------------------------------------===// 162 163bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 164 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 165 cerr << "\n"); 166 SDOperand Res = SDOperand(); 167 168 if (Res.Val == 0) { 169 switch (N->getOpcode()) { 170 default: 171#ifndef NDEBUG 172 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": "; 173 N->dump(&DAG); cerr << "\n"; 174#endif 175 assert(0 && "Do not know how to scalarize this operator's operand!"); 176 abort(); 177 178 case ISD::BIT_CONVERT: 179 Res = ScalarizeVecOp_BIT_CONVERT(N); break; 180 181 case ISD::EXTRACT_VECTOR_ELT: 182 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); break; 183 184 case ISD::STORE: 185 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); break; 186 } 187 } 188 189 // If the result is null, the sub-method took care of registering results etc. 190 if (!Res.Val) return false; 191 192 // If the result is N, the sub-method updated N in place. Check to see if any 193 // operands are new, and if so, mark them. 194 if (Res.Val == N) { 195 // Mark N as new and remark N and its operands. This allows us to correctly 196 // revisit N if it needs another step of promotion and allows us to visit 197 // any new operands to N. 198 ReanalyzeNode(N); 199 return true; 200 } 201 202 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 203 "Invalid operand expansion"); 204 205 ReplaceValueWith(SDOperand(N, 0), Res); 206 return false; 207} 208 209/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 210/// to be scalarized, it must be <1 x ty>. Convert the element instead. 211SDOperand DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 212 SDOperand Elt = GetScalarizedVector(N->getOperand(0)); 213 return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Elt); 214} 215 216/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 217/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 218/// index. 219SDOperand DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 220 return GetScalarizedVector(N->getOperand(0)); 221} 222 223/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 224/// scalarized, it must be <1 x ty>. Just store the element. 225SDOperand DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 226 assert(ISD::isNormalStore(N) && "Truncating store of one-element vector?"); 227 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 228 return DAG.getStore(N->getChain(), GetScalarizedVector(N->getOperand(1)), 229 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 230 N->isVolatile(), N->getAlignment()); 231} 232 233 234//===----------------------------------------------------------------------===// 235// Result Vector Splitting 236//===----------------------------------------------------------------------===// 237 238/// SplitVectorResult - This method is called when the specified result of the 239/// specified node is found to need vector splitting. At this point, the node 240/// may also have invalid operands or may have other results that need 241/// legalization, we just know that (at least) one result needs vector 242/// splitting. 243void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 244 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n"); 245 SDOperand Lo, Hi; 246 247 switch (N->getOpcode()) { 248 default: 249#ifndef NDEBUG 250 cerr << "SplitVectorResult #" << ResNo << ": "; 251 N->dump(&DAG); cerr << "\n"; 252#endif 253 assert(0 && "Do not know how to split the result of this operator!"); 254 abort(); 255 256 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 257 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 258 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 259 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 260 261 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 262 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 263 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 264 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 265 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 266 case ISD::LOAD: SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);break; 267 case ISD::VECTOR_SHUFFLE: SplitVecRes_VECTOR_SHUFFLE(N, Lo, Hi); break; 268 case ISD::VSETCC: SplitVecRes_VSETCC(N, Lo, Hi); break; 269 270 case ISD::CTTZ: 271 case ISD::CTLZ: 272 case ISD::CTPOP: 273 case ISD::FNEG: 274 case ISD::FABS: 275 case ISD::FSQRT: 276 case ISD::FSIN: 277 case ISD::FCOS: 278 case ISD::FP_TO_SINT: 279 case ISD::FP_TO_UINT: 280 case ISD::SINT_TO_FP: 281 case ISD::UINT_TO_FP: SplitVecRes_UnOp(N, Lo, Hi); break; 282 283 case ISD::ADD: 284 case ISD::SUB: 285 case ISD::MUL: 286 case ISD::FADD: 287 case ISD::FSUB: 288 case ISD::FMUL: 289 case ISD::SDIV: 290 case ISD::UDIV: 291 case ISD::FDIV: 292 case ISD::FPOW: 293 case ISD::AND: 294 case ISD::OR: 295 case ISD::XOR: 296 case ISD::UREM: 297 case ISD::SREM: 298 case ISD::FREM: SplitVecRes_BinOp(N, Lo, Hi); break; 299 } 300 301 // If Lo/Hi is null, the sub-method took care of registering results etc. 302 if (Lo.Val) 303 SetSplitVector(SDOperand(N, ResNo), Lo, Hi); 304} 305 306void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDOperand &Lo, 307 SDOperand &Hi) { 308 SDOperand LHSLo, LHSHi; 309 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 310 SDOperand RHSLo, RHSHi; 311 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 312 313 Lo = DAG.getNode(N->getOpcode(), LHSLo.getValueType(), LHSLo, RHSLo); 314 Hi = DAG.getNode(N->getOpcode(), LHSHi.getValueType(), LHSHi, RHSHi); 315} 316 317void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDOperand &Lo, 318 SDOperand &Hi) { 319 // We know the result is a vector. The input may be either a vector or a 320 // scalar value. 321 MVT LoVT, HiVT; 322 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 323 324 SDOperand InOp = N->getOperand(0); 325 MVT InVT = InOp.getValueType(); 326 327 // Handle some special cases efficiently. 328 switch (getTypeAction(InVT)) { 329 default: 330 assert(false && "Unknown type action!"); 331 case Legal: 332 case PromoteInteger: 333 case SoftenFloat: 334 case ScalarizeVector: 335 break; 336 case ExpandInteger: 337 case ExpandFloat: 338 // A scalar to vector conversion, where the scalar needs expansion. 339 // If the vector is being split in two then we can just convert the 340 // expanded pieces. 341 if (LoVT == HiVT) { 342 GetExpandedOp(InOp, Lo, Hi); 343 if (TLI.isBigEndian()) 344 std::swap(Lo, Hi); 345 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 346 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 347 return; 348 } 349 break; 350 case SplitVector: 351 // If the input is a vector that needs to be split, convert each split 352 // piece of the input now. 353 GetSplitVector(InOp, Lo, Hi); 354 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 355 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 356 return; 357 } 358 359 // In the general case, convert the input to an integer and split it by hand. 360 MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits()); 361 MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits()); 362 if (TLI.isBigEndian()) 363 std::swap(LoIntVT, HiIntVT); 364 365 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 366 367 if (TLI.isBigEndian()) 368 std::swap(Lo, Hi); 369 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 370 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 371} 372 373void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDOperand &Lo, 374 SDOperand &Hi) { 375 MVT LoVT, HiVT; 376 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 377 unsigned LoNumElts = LoVT.getVectorNumElements(); 378 SmallVector<SDOperand, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 379 Lo = DAG.getNode(ISD::BUILD_VECTOR, LoVT, &LoOps[0], LoOps.size()); 380 381 SmallVector<SDOperand, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 382 Hi = DAG.getNode(ISD::BUILD_VECTOR, HiVT, &HiOps[0], HiOps.size()); 383} 384 385void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDOperand &Lo, 386 SDOperand &Hi) { 387 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 388 unsigned NumSubvectors = N->getNumOperands() / 2; 389 if (NumSubvectors == 1) { 390 Lo = N->getOperand(0); 391 Hi = N->getOperand(1); 392 return; 393 } 394 395 MVT LoVT, HiVT; 396 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 397 398 SmallVector<SDOperand, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 399 Lo = DAG.getNode(ISD::CONCAT_VECTORS, LoVT, &LoOps[0], LoOps.size()); 400 401 SmallVector<SDOperand, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 402 Hi = DAG.getNode(ISD::CONCAT_VECTORS, HiVT, &HiOps[0], HiOps.size()); 403} 404 405void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDOperand &Lo, 406 SDOperand &Hi) { 407 GetSplitVector(N->getOperand(0), Lo, Hi); 408 Lo = DAG.getNode(ISD::FPOWI, Lo.getValueType(), Lo, N->getOperand(1)); 409 Hi = DAG.getNode(ISD::FPOWI, Hi.getValueType(), Hi, N->getOperand(1)); 410} 411 412void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDOperand &Lo, 413 SDOperand &Hi) { 414 SDOperand Vec = N->getOperand(0); 415 SDOperand Elt = N->getOperand(1); 416 SDOperand Idx = N->getOperand(2); 417 GetSplitVector(Vec, Lo, Hi); 418 419 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 420 unsigned IdxVal = CIdx->getValue(); 421 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 422 if (IdxVal < LoNumElts) 423 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, Lo.getValueType(), Lo, Elt, Idx); 424 else 425 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, Hi.getValueType(), Hi, Elt, 426 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 427 return; 428 } 429 430 // Spill the vector to the stack. 431 MVT VecVT = Vec.getValueType(); 432 MVT EltVT = VecVT.getVectorElementType(); 433 SDOperand StackPtr = DAG.CreateStackTemporary(VecVT); 434 SDOperand Store = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); 435 436 // Store the new element. This may be larger than the vector element type, 437 // so use a truncating store. 438 SDOperand EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 439 Store = DAG.getTruncStore(Store, Elt, EltPtr, NULL, 0, EltVT); 440 441 // Reload the vector from the stack. 442 SDOperand Load = DAG.getLoad(VecVT, Store, StackPtr, NULL, 0); 443 444 // Split it. 445 SplitVecRes_LOAD(cast<LoadSDNode>(Load.Val), Lo, Hi); 446} 447 448void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDOperand &Lo, 449 SDOperand &Hi) { 450 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 451 MVT LoVT, HiVT; 452 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 453 454 SDOperand Ch = LD->getChain(); 455 SDOperand Ptr = LD->getBasePtr(); 456 const Value *SV = LD->getSrcValue(); 457 int SVOffset = LD->getSrcValueOffset(); 458 unsigned Alignment = LD->getAlignment(); 459 bool isVolatile = LD->isVolatile(); 460 461 Lo = DAG.getLoad(LoVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); 462 463 if (LD->getExtensionType() == ISD::NON_EXTLOAD) { 464 unsigned IncrementSize = LoVT.getSizeInBits()/8; 465 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 466 DAG.getIntPtrConstant(IncrementSize)); 467 SVOffset += IncrementSize; 468 Alignment = MinAlign(Alignment, IncrementSize); 469 Hi = DAG.getLoad(HiVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); 470 471 // Build a factor node to remember that this load is independent of the 472 // other one. 473 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 474 Hi.getValue(1)); 475 } else { 476 assert(LD->getExtensionType() == ISD::EXTLOAD && 477 "Unsupported vector extending load!"); 478 Hi = DAG.getNode(ISD::UNDEF, HiVT); 479 Ch = Lo.getValue(1); 480 } 481 482 // Legalized the chain result - switch anything that used the old chain to 483 // use the new one. 484 ReplaceValueWith(SDOperand(LD, 1), Ch); 485} 486 487void DAGTypeLegalizer::SplitVecRes_UnOp(SDNode *N, SDOperand &Lo, 488 SDOperand &Hi) { 489 // Get the dest types. This doesn't always match input types, e.g. int_to_fp. 490 MVT LoVT, HiVT; 491 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 492 493 GetSplitVector(N->getOperand(0), Lo, Hi); 494 Lo = DAG.getNode(N->getOpcode(), LoVT, Lo); 495 Hi = DAG.getNode(N->getOpcode(), HiVT, Hi); 496} 497 498void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(SDNode *N, SDOperand &Lo, 499 SDOperand &Hi) { 500 // Build the low part. 501 SDOperand Mask = N->getOperand(2); 502 SmallVector<SDOperand, 16> Ops; 503 MVT LoVT, HiVT; 504 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 505 MVT EltVT = LoVT.getVectorElementType(); 506 unsigned LoNumElts = LoVT.getVectorNumElements(); 507 unsigned NumElements = Mask.getNumOperands(); 508 509 // Insert all of the elements from the input that are needed. We use 510 // buildvector of extractelement here because the input vectors will have 511 // to be legalized, so this makes the code simpler. 512 for (unsigned i = 0; i != LoNumElts; ++i) { 513 unsigned Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getValue(); 514 SDOperand InVec = N->getOperand(0); 515 if (Idx >= NumElements) { 516 InVec = N->getOperand(1); 517 Idx -= NumElements; 518 } 519 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, InVec, 520 DAG.getIntPtrConstant(Idx))); 521 } 522 Lo = DAG.getNode(ISD::BUILD_VECTOR, LoVT, &Ops[0], Ops.size()); 523 Ops.clear(); 524 525 for (unsigned i = LoNumElts; i != NumElements; ++i) { 526 unsigned Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getValue(); 527 SDOperand InVec = N->getOperand(0); 528 if (Idx >= NumElements) { 529 InVec = N->getOperand(1); 530 Idx -= NumElements; 531 } 532 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, InVec, 533 DAG.getIntPtrConstant(Idx))); 534 } 535 Hi = DAG.getNode(ISD::BUILD_VECTOR, HiVT, &Ops[0], Ops.size()); 536} 537 538void DAGTypeLegalizer::SplitVecRes_VSETCC(SDNode *N, SDOperand &Lo, 539 SDOperand &Hi) { 540 MVT LoVT, HiVT; 541 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 542 543 SDOperand LL, LH, RL, RH; 544 GetSplitVector(N->getOperand(0), LL, LH); 545 GetSplitVector(N->getOperand(1), RL, RH); 546 547 Lo = DAG.getNode(ISD::VSETCC, LoVT, LL, RL, N->getOperand(2)); 548 Hi = DAG.getNode(ISD::VSETCC, HiVT, LH, RH, N->getOperand(2)); 549} 550 551 552//===----------------------------------------------------------------------===// 553// Operand Vector Splitting 554//===----------------------------------------------------------------------===// 555 556/// SplitVectorOperand - This method is called when the specified operand of the 557/// specified node is found to need vector splitting. At this point, all of the 558/// result types of the node are known to be legal, but other operands of the 559/// node may need legalization as well as the specified one. 560bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 561 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n"); 562 SDOperand Res = SDOperand(); 563 564 if (Res.Val == 0) { 565 switch (N->getOpcode()) { 566 default: 567#ifndef NDEBUG 568 cerr << "SplitVectorOperand Op #" << OpNo << ": "; 569 N->dump(&DAG); cerr << "\n"; 570#endif 571 assert(0 && "Do not know how to split this operator's operand!"); 572 abort(); 573 574 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 575 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 576 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 577 case ISD::STORE: Res = SplitVecOp_STORE(cast<StoreSDNode>(N), 578 OpNo); break; 579 case ISD::VECTOR_SHUFFLE: Res = SplitVecOp_VECTOR_SHUFFLE(N, OpNo);break; 580 } 581 } 582 583 // If the result is null, the sub-method took care of registering results etc. 584 if (!Res.Val) return false; 585 586 // If the result is N, the sub-method updated N in place. Check to see if any 587 // operands are new, and if so, mark them. 588 if (Res.Val == N) { 589 // Mark N as new and remark N and its operands. This allows us to correctly 590 // revisit N if it needs another step of promotion and allows us to visit 591 // any new operands to N. 592 ReanalyzeNode(N); 593 return true; 594 } 595 596 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 597 "Invalid operand expansion"); 598 599 ReplaceValueWith(SDOperand(N, 0), Res); 600 return false; 601} 602 603SDOperand DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 604 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 605 // end up being split all the way down to individual components. Convert the 606 // split pieces into integers and reassemble. 607 SDOperand Lo, Hi; 608 GetSplitVector(N->getOperand(0), Lo, Hi); 609 Lo = BitConvertToInteger(Lo); 610 Hi = BitConvertToInteger(Hi); 611 612 if (TLI.isBigEndian()) 613 std::swap(Lo, Hi); 614 615 return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), 616 JoinIntegers(Lo, Hi)); 617} 618 619SDOperand DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 620 // We know that the extracted result type is legal. For now, assume the index 621 // is a constant. 622 MVT SubVT = N->getValueType(0); 623 SDOperand Idx = N->getOperand(1); 624 SDOperand Lo, Hi; 625 GetSplitVector(N->getOperand(0), Lo, Hi); 626 627 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 628 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getValue(); 629 630 if (IdxVal < LoElts) { 631 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 632 "Extracted subvector crosses vector split!"); 633 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SubVT, Lo, Idx); 634 } else { 635 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SubVT, Hi, 636 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 637 } 638} 639 640SDOperand DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 641 SDOperand Vec = N->getOperand(0); 642 SDOperand Idx = N->getOperand(1); 643 MVT VecVT = Vec.getValueType(); 644 645 if (isa<ConstantSDNode>(Idx)) { 646 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getValue(); 647 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 648 649 SDOperand Lo, Hi; 650 GetSplitVector(Vec, Lo, Hi); 651 652 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 653 654 if (IdxVal < LoElts) 655 return DAG.UpdateNodeOperands(SDOperand(N, 0), Lo, Idx); 656 else 657 return DAG.UpdateNodeOperands(SDOperand(N, 0), Hi, 658 DAG.getConstant(IdxVal - LoElts, 659 Idx.getValueType())); 660 } 661 662 // Store the vector to the stack. 663 MVT EltVT = VecVT.getVectorElementType(); 664 SDOperand StackPtr = DAG.CreateStackTemporary(VecVT); 665 SDOperand Store = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); 666 667 // Load back the required element. 668 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 669 return DAG.getLoad(EltVT, Store, StackPtr, NULL, 0); 670} 671 672SDOperand DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 673 assert(ISD::isNormalStore(N) && "Truncating store of vector?"); 674 assert(OpNo == 1 && "Can only split the stored value"); 675 676 SDOperand Ch = N->getChain(); 677 SDOperand Ptr = N->getBasePtr(); 678 int SVOffset = N->getSrcValueOffset(); 679 unsigned Alignment = N->getAlignment(); 680 bool isVol = N->isVolatile(); 681 SDOperand Lo, Hi; 682 GetSplitVector(N->getOperand(1), Lo, Hi); 683 684 unsigned IncrementSize = Lo.getValueType().getSizeInBits()/8; 685 686 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, isVol, Alignment); 687 688 // Increment the pointer to the other half. 689 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 690 DAG.getIntPtrConstant(IncrementSize)); 691 692 Hi = DAG.getStore(Ch, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 693 isVol, MinAlign(Alignment, IncrementSize)); 694 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 695} 696 697SDOperand DAGTypeLegalizer::SplitVecOp_VECTOR_SHUFFLE(SDNode *N, unsigned OpNo){ 698 assert(OpNo == 2 && "Shuffle source type differs from result type?"); 699 SDOperand Mask = N->getOperand(2); 700 unsigned MaskLength = Mask.getValueType().getVectorNumElements(); 701 unsigned LargestMaskEntryPlusOne = 2 * MaskLength; 702 unsigned MinimumBitWidth = Log2_32_Ceil(LargestMaskEntryPlusOne); 703 704 // Look for a legal vector type to place the mask values in. 705 // Note that there may not be *any* legal vector-of-integer 706 // type for which the element type is legal! 707 for (MVT::SimpleValueType EltVT = MVT::FIRST_INTEGER_VALUETYPE; 708 EltVT <= MVT::LAST_INTEGER_VALUETYPE; 709 // Integer values types are consecutively numbered. Exploit this. 710 EltVT = MVT::SimpleValueType(EltVT + 1)) { 711 712 // Is the element type big enough to hold the values? 713 if (MVT(EltVT).getSizeInBits() < MinimumBitWidth) 714 // Nope. 715 continue; 716 717 // Is the vector type legal? 718 MVT VecVT = MVT::getVectorVT(EltVT, MaskLength); 719 if (!isTypeLegal(VecVT)) 720 // Nope. 721 continue; 722 723 // If the element type is not legal, find a larger legal type to use for 724 // the BUILD_VECTOR operands. This is an ugly hack, but seems to work! 725 // FIXME: The real solution is to change VECTOR_SHUFFLE into a variadic 726 // node where the shuffle mask is a list of integer operands, #2 .. #2+n. 727 for (MVT::SimpleValueType OpVT = EltVT; OpVT <= MVT::LAST_INTEGER_VALUETYPE; 728 // Integer values types are consecutively numbered. Exploit this. 729 OpVT = MVT::SimpleValueType(OpVT + 1)) { 730 if (!isTypeLegal(OpVT)) 731 continue; 732 733 // Success! Rebuild the vector using the legal types. 734 SmallVector<SDOperand, 16> Ops(MaskLength); 735 for (unsigned i = 0; i < MaskLength; ++i) { 736 uint64_t Idx = 737 cast<ConstantSDNode>(Mask.getOperand(i))->getValue(); 738 Ops[i] = DAG.getConstant(Idx, OpVT); 739 } 740 return DAG.UpdateNodeOperands(SDOperand(N,0), 741 N->getOperand(0), N->getOperand(1), 742 DAG.getNode(ISD::BUILD_VECTOR, 743 VecVT, &Ops[0], Ops.size())); 744 } 745 746 // Continuing is pointless - failure is certain. 747 break; 748 } 749 assert(false && "Failed to find an appropriate mask type!"); 750 return SDOperand(N, 0); 751} 752