LegalizeVectorTypes.cpp revision faeb4a35197737b3674fcfc34fd6e84220aa437b
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in multiple vectors of a smaller type. For example, 19// implementing <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24using namespace llvm; 25 26//===----------------------------------------------------------------------===// 27// Result Vector Scalarization: <1 x ty> -> ty. 28//===----------------------------------------------------------------------===// 29 30void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 31 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 32 cerr << "\n"); 33 SDValue R = SDValue(); 34 35 switch (N->getOpcode()) { 36 default: 37#ifndef NDEBUG 38 cerr << "ScalarizeVectorResult #" << ResNo << ": "; 39 N->dump(&DAG); cerr << "\n"; 40#endif 41 assert(0 && "Do not know how to scalarize the result of this operator!"); 42 abort(); 43 44 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 45 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 46 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 47 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 48 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 49 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 50 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 51 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 52 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 53 54 case ISD::CTLZ: 55 case ISD::CTPOP: 56 case ISD::CTTZ: 57 case ISD::FABS: 58 case ISD::FCOS: 59 case ISD::FNEG: 60 case ISD::FP_TO_SINT: 61 case ISD::FP_TO_UINT: 62 case ISD::FSIN: 63 case ISD::FSQRT: 64 case ISD::FTRUNC: 65 case ISD::FFLOOR: 66 case ISD::FCEIL: 67 case ISD::FRINT: 68 case ISD::FNEARBYINT: 69 case ISD::SINT_TO_FP: 70 case ISD::UINT_TO_FP: R = ScalarizeVecRes_UnaryOp(N); break; 71 72 case ISD::ADD: 73 case ISD::AND: 74 case ISD::FADD: 75 case ISD::FDIV: 76 case ISD::FMUL: 77 case ISD::FPOW: 78 case ISD::FREM: 79 case ISD::FSUB: 80 case ISD::MUL: 81 case ISD::OR: 82 case ISD::SDIV: 83 case ISD::SREM: 84 case ISD::SUB: 85 case ISD::UDIV: 86 case ISD::UREM: 87 case ISD::XOR: R = ScalarizeVecRes_BinOp(N); break; 88 } 89 90 // If R is null, the sub-method took care of registering the result. 91 if (R.getNode()) 92 SetScalarizedVector(SDValue(N, ResNo), R); 93} 94 95SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 96 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 97 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 98 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 99} 100 101SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 102 MVT NewVT = N->getValueType(0).getVectorElementType(); 103 return DAG.getNode(ISD::BIT_CONVERT, NewVT, N->getOperand(0)); 104} 105 106SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 107 SDValue Op = GetScalarizedVector(N->getOperand(0)); 108 return DAG.getNode(ISD::FPOWI, Op.getValueType(), Op, N->getOperand(1)); 109} 110 111SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 112 // The value to insert may have a wider type than the vector element type, 113 // so be sure to truncate it to the element type if necessary. 114 SDValue Op = N->getOperand(1); 115 MVT EltVT = N->getValueType(0).getVectorElementType(); 116 if (Op.getValueType() != EltVT) 117 // FIXME: Can this happen for floating point types? 118 Op = DAG.getNode(ISD::TRUNCATE, EltVT, Op); 119 return Op; 120} 121 122SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 123 assert(N->isUnindexed() && "Indexed vector load?"); 124 125 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getExtensionType(), 126 N->getValueType(0).getVectorElementType(), 127 N->getChain(), N->getBasePtr(), 128 DAG.getNode(ISD::UNDEF, 129 N->getBasePtr().getValueType()), 130 N->getSrcValue(), N->getSrcValueOffset(), 131 N->getMemoryVT().getVectorElementType(), 132 N->isVolatile(), N->getAlignment()); 133 134 // Legalized the chain result - switch anything that used the old chain to 135 // use the new one. 136 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 137 return Result; 138} 139 140SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 141 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 142 MVT DestVT = TLI.getTypeToTransformTo(N->getValueType(0)); 143 SDValue Op = GetScalarizedVector(N->getOperand(0)); 144 return DAG.getNode(N->getOpcode(), DestVT, Op); 145} 146 147SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 148 return DAG.getNode(ISD::UNDEF, N->getValueType(0).getVectorElementType()); 149} 150 151SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 152 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 153 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0), LHS, 154 GetScalarizedVector(N->getOperand(2))); 155} 156 157SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 158 // Figure out if the scalar is the LHS or RHS and return it. 159 SDValue EltNum = N->getOperand(2).getOperand(0); 160 unsigned Op = cast<ConstantSDNode>(EltNum)->getZExtValue() != 0; 161 return GetScalarizedVector(N->getOperand(Op)); 162} 163 164SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 165 MVT NewVT = N->getValueType(0).getVectorElementType(); 166 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 167 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 168 LHS = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHS), LHS, RHS, 169 N->getOperand(2)); 170 return 171 DAG.getNode(ISD::SELECT, NewVT, LHS, 172 DAG.getConstant(APInt::getAllOnesValue(NewVT.getSizeInBits()), 173 NewVT), 174 DAG.getConstant(0ULL, NewVT)); 175} 176 177 178//===----------------------------------------------------------------------===// 179// Operand Vector Scalarization <1 x ty> -> ty. 180//===----------------------------------------------------------------------===// 181 182bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 183 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 184 cerr << "\n"); 185 SDValue Res = SDValue(); 186 187 if (Res.getNode() == 0) { 188 switch (N->getOpcode()) { 189 default: 190#ifndef NDEBUG 191 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": "; 192 N->dump(&DAG); cerr << "\n"; 193#endif 194 assert(0 && "Do not know how to scalarize this operator's operand!"); 195 abort(); 196 197 case ISD::BIT_CONVERT: 198 Res = ScalarizeVecOp_BIT_CONVERT(N); break; 199 200 case ISD::EXTRACT_VECTOR_ELT: 201 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); break; 202 203 case ISD::STORE: 204 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); break; 205 } 206 } 207 208 // If the result is null, the sub-method took care of registering results etc. 209 if (!Res.getNode()) return false; 210 211 // If the result is N, the sub-method updated N in place. Check to see if any 212 // operands are new, and if so, mark them. 213 if (Res.getNode() == N) { 214 // Mark N as new and remark N and its operands. This allows us to correctly 215 // revisit N if it needs another step of promotion and allows us to visit 216 // any new operands to N. 217 ReanalyzeNode(N); 218 return true; 219 } 220 221 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 222 "Invalid operand expansion"); 223 224 ReplaceValueWith(SDValue(N, 0), Res); 225 return false; 226} 227 228/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 229/// to be scalarized, it must be <1 x ty>. Convert the element instead. 230SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 231 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 232 return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Elt); 233} 234 235/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 236/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 237/// index. 238SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 239 return GetScalarizedVector(N->getOperand(0)); 240} 241 242/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 243/// scalarized, it must be <1 x ty>. Just store the element. 244SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 245 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 246 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 247 248 if (N->isTruncatingStore()) 249 return DAG.getTruncStore(N->getChain(), 250 GetScalarizedVector(N->getOperand(1)), 251 N->getBasePtr(), 252 N->getSrcValue(), N->getSrcValueOffset(), 253 N->getMemoryVT().getVectorElementType(), 254 N->isVolatile(), N->getAlignment()); 255 256 return DAG.getStore(N->getChain(), GetScalarizedVector(N->getOperand(1)), 257 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 258 N->isVolatile(), N->getAlignment()); 259} 260 261 262//===----------------------------------------------------------------------===// 263// Result Vector Splitting 264//===----------------------------------------------------------------------===// 265 266/// SplitVectorResult - This method is called when the specified result of the 267/// specified node is found to need vector splitting. At this point, the node 268/// may also have invalid operands or may have other results that need 269/// legalization, we just know that (at least) one result needs vector 270/// splitting. 271void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 272 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n"); 273 SDValue Lo, Hi; 274 275 switch (N->getOpcode()) { 276 default: 277#ifndef NDEBUG 278 cerr << "SplitVectorResult #" << ResNo << ": "; 279 N->dump(&DAG); cerr << "\n"; 280#endif 281 assert(0 && "Do not know how to split the result of this operator!"); 282 abort(); 283 284 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 285 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 286 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 287 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 288 289 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 290 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 291 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 292 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 293 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 294 case ISD::LOAD: SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);break; 295 case ISD::VECTOR_SHUFFLE: SplitVecRes_VECTOR_SHUFFLE(N, Lo, Hi); break; 296 case ISD::VSETCC: SplitVecRes_VSETCC(N, Lo, Hi); break; 297 298 case ISD::CTTZ: 299 case ISD::CTLZ: 300 case ISD::CTPOP: 301 case ISD::FNEG: 302 case ISD::FABS: 303 case ISD::FSQRT: 304 case ISD::FSIN: 305 case ISD::FCOS: 306 case ISD::FTRUNC: 307 case ISD::FFLOOR: 308 case ISD::FCEIL: 309 case ISD::FRINT: 310 case ISD::FNEARBYINT: 311 case ISD::FP_TO_SINT: 312 case ISD::FP_TO_UINT: 313 case ISD::SINT_TO_FP: 314 case ISD::UINT_TO_FP: SplitVecRes_UnaryOp(N, Lo, Hi); break; 315 316 case ISD::ADD: 317 case ISD::SUB: 318 case ISD::MUL: 319 case ISD::FADD: 320 case ISD::FSUB: 321 case ISD::FMUL: 322 case ISD::SDIV: 323 case ISD::UDIV: 324 case ISD::FDIV: 325 case ISD::FPOW: 326 case ISD::AND: 327 case ISD::OR: 328 case ISD::XOR: 329 case ISD::UREM: 330 case ISD::SREM: 331 case ISD::FREM: SplitVecRes_BinOp(N, Lo, Hi); break; 332 } 333 334 // If Lo/Hi is null, the sub-method took care of registering results etc. 335 if (Lo.getNode()) 336 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 337} 338 339void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 340 SDValue &Hi) { 341 SDValue LHSLo, LHSHi; 342 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 343 SDValue RHSLo, RHSHi; 344 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 345 346 Lo = DAG.getNode(N->getOpcode(), LHSLo.getValueType(), LHSLo, RHSLo); 347 Hi = DAG.getNode(N->getOpcode(), LHSHi.getValueType(), LHSHi, RHSHi); 348} 349 350void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 351 SDValue &Hi) { 352 // We know the result is a vector. The input may be either a vector or a 353 // scalar value. 354 MVT LoVT, HiVT; 355 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 356 357 SDValue InOp = N->getOperand(0); 358 MVT InVT = InOp.getValueType(); 359 360 // Handle some special cases efficiently. 361 switch (getTypeAction(InVT)) { 362 default: 363 assert(false && "Unknown type action!"); 364 case Legal: 365 case PromoteInteger: 366 case SoftenFloat: 367 case ScalarizeVector: 368 break; 369 case ExpandInteger: 370 case ExpandFloat: 371 // A scalar to vector conversion, where the scalar needs expansion. 372 // If the vector is being split in two then we can just convert the 373 // expanded pieces. 374 if (LoVT == HiVT) { 375 GetExpandedOp(InOp, Lo, Hi); 376 if (TLI.isBigEndian()) 377 std::swap(Lo, Hi); 378 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 379 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 380 return; 381 } 382 break; 383 case SplitVector: 384 // If the input is a vector that needs to be split, convert each split 385 // piece of the input now. 386 GetSplitVector(InOp, Lo, Hi); 387 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 388 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 389 return; 390 } 391 392 // In the general case, convert the input to an integer and split it by hand. 393 MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits()); 394 MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits()); 395 if (TLI.isBigEndian()) 396 std::swap(LoIntVT, HiIntVT); 397 398 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 399 400 if (TLI.isBigEndian()) 401 std::swap(Lo, Hi); 402 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 403 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 404} 405 406void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 407 SDValue &Hi) { 408 MVT LoVT, HiVT; 409 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 410 unsigned LoNumElts = LoVT.getVectorNumElements(); 411 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 412 Lo = DAG.getNode(ISD::BUILD_VECTOR, LoVT, &LoOps[0], LoOps.size()); 413 414 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 415 Hi = DAG.getNode(ISD::BUILD_VECTOR, HiVT, &HiOps[0], HiOps.size()); 416} 417 418void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 419 SDValue &Hi) { 420 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 421 unsigned NumSubvectors = N->getNumOperands() / 2; 422 if (NumSubvectors == 1) { 423 Lo = N->getOperand(0); 424 Hi = N->getOperand(1); 425 return; 426 } 427 428 MVT LoVT, HiVT; 429 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 430 431 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 432 Lo = DAG.getNode(ISD::CONCAT_VECTORS, LoVT, &LoOps[0], LoOps.size()); 433 434 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 435 Hi = DAG.getNode(ISD::CONCAT_VECTORS, HiVT, &HiOps[0], HiOps.size()); 436} 437 438void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 439 SDValue &Hi) { 440 GetSplitVector(N->getOperand(0), Lo, Hi); 441 Lo = DAG.getNode(ISD::FPOWI, Lo.getValueType(), Lo, N->getOperand(1)); 442 Hi = DAG.getNode(ISD::FPOWI, Hi.getValueType(), Hi, N->getOperand(1)); 443} 444 445void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 446 SDValue &Hi) { 447 SDValue Vec = N->getOperand(0); 448 SDValue Elt = N->getOperand(1); 449 SDValue Idx = N->getOperand(2); 450 GetSplitVector(Vec, Lo, Hi); 451 452 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 453 unsigned IdxVal = CIdx->getZExtValue(); 454 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 455 if (IdxVal < LoNumElts) 456 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, Lo.getValueType(), Lo, Elt, Idx); 457 else 458 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, Hi.getValueType(), Hi, Elt, 459 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 460 return; 461 } 462 463 // Spill the vector to the stack. 464 MVT VecVT = Vec.getValueType(); 465 MVT EltVT = VecVT.getVectorElementType(); 466 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 467 SDValue Store = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); 468 469 // Store the new element. This may be larger than the vector element type, 470 // so use a truncating store. 471 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 472 Store = DAG.getTruncStore(Store, Elt, EltPtr, NULL, 0, EltVT); 473 474 // Reload the vector from the stack. 475 SDValue Load = DAG.getLoad(VecVT, Store, StackPtr, NULL, 0); 476 477 // Split it. 478 SplitVecRes_LOAD(cast<LoadSDNode>(Load.getNode()), Lo, Hi); 479} 480 481void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 482 SDValue &Hi) { 483 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 484 MVT LoVT, HiVT; 485 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 486 487 ISD::LoadExtType ExtType = LD->getExtensionType(); 488 SDValue Ch = LD->getChain(); 489 SDValue Ptr = LD->getBasePtr(); 490 SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType()); 491 const Value *SV = LD->getSrcValue(); 492 int SVOffset = LD->getSrcValueOffset(); 493 MVT MemoryVT = LD->getMemoryVT(); 494 unsigned Alignment = LD->getAlignment(); 495 bool isVolatile = LD->isVolatile(); 496 497 MVT LoMemVT, HiMemVT; 498 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 499 500 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, Ch, Ptr, Offset, 501 SV, SVOffset, LoMemVT, isVolatile, Alignment); 502 503 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 504 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 505 DAG.getIntPtrConstant(IncrementSize)); 506 SVOffset += IncrementSize; 507 Alignment = MinAlign(Alignment, IncrementSize); 508 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, Ch, Ptr, Offset, 509 SV, SVOffset, HiMemVT, isVolatile, Alignment); 510 511 // Build a factor node to remember that this load is independent of the 512 // other one. 513 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 514 Hi.getValue(1)); 515 516 // Legalized the chain result - switch anything that used the old chain to 517 // use the new one. 518 ReplaceValueWith(SDValue(LD, 1), Ch); 519} 520 521void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 522 SDValue &Hi) { 523 // Get the dest types - they may not match the input types, e.g. int_to_fp. 524 MVT LoVT, HiVT; 525 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 526 527 // Split the input. 528 MVT InVT = N->getOperand(0).getValueType(); 529 switch (getTypeAction(InVT)) { 530 default: assert(0 && "Unexpected type action!"); 531 case Legal: { 532 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 533 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 534 LoVT.getVectorNumElements()); 535 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, InNVT, N->getOperand(0), 536 DAG.getIntPtrConstant(0)); 537 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, InNVT, N->getOperand(0), 538 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 539 break; 540 } 541 case SplitVector: 542 GetSplitVector(N->getOperand(0), Lo, Hi); 543 break; 544 } 545 546 Lo = DAG.getNode(N->getOpcode(), LoVT, Lo); 547 Hi = DAG.getNode(N->getOpcode(), HiVT, Hi); 548} 549 550void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(SDNode *N, SDValue &Lo, 551 SDValue &Hi) { 552 // Build the low part. 553 SDValue Mask = N->getOperand(2); 554 SmallVector<SDValue, 16> Ops; 555 MVT LoVT, HiVT; 556 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 557 MVT EltVT = LoVT.getVectorElementType(); 558 unsigned LoNumElts = LoVT.getVectorNumElements(); 559 unsigned NumElements = Mask.getNumOperands(); 560 561 // Insert all of the elements from the input that are needed. We use 562 // buildvector of extractelement here because the input vectors will have 563 // to be legalized, so this makes the code simpler. 564 for (unsigned i = 0; i != LoNumElts; ++i) { 565 unsigned Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getZExtValue(); 566 SDValue InVec = N->getOperand(0); 567 if (Idx >= NumElements) { 568 InVec = N->getOperand(1); 569 Idx -= NumElements; 570 } 571 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, InVec, 572 DAG.getIntPtrConstant(Idx))); 573 } 574 Lo = DAG.getNode(ISD::BUILD_VECTOR, LoVT, &Ops[0], Ops.size()); 575 Ops.clear(); 576 577 for (unsigned i = LoNumElts; i != NumElements; ++i) { 578 unsigned Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getZExtValue(); 579 SDValue InVec = N->getOperand(0); 580 if (Idx >= NumElements) { 581 InVec = N->getOperand(1); 582 Idx -= NumElements; 583 } 584 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, InVec, 585 DAG.getIntPtrConstant(Idx))); 586 } 587 Hi = DAG.getNode(ISD::BUILD_VECTOR, HiVT, &Ops[0], Ops.size()); 588} 589 590void DAGTypeLegalizer::SplitVecRes_VSETCC(SDNode *N, SDValue &Lo, 591 SDValue &Hi) { 592 MVT LoVT, HiVT; 593 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 594 595 SDValue LL, LH, RL, RH; 596 GetSplitVector(N->getOperand(0), LL, LH); 597 GetSplitVector(N->getOperand(1), RL, RH); 598 599 Lo = DAG.getNode(ISD::VSETCC, LoVT, LL, RL, N->getOperand(2)); 600 Hi = DAG.getNode(ISD::VSETCC, HiVT, LH, RH, N->getOperand(2)); 601} 602 603 604//===----------------------------------------------------------------------===// 605// Operand Vector Splitting 606//===----------------------------------------------------------------------===// 607 608/// SplitVectorOperand - This method is called when the specified operand of the 609/// specified node is found to need vector splitting. At this point, all of the 610/// result types of the node are known to be legal, but other operands of the 611/// node may need legalization as well as the specified one. 612bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 613 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n"); 614 SDValue Res = SDValue(); 615 616 if (Res.getNode() == 0) { 617 switch (N->getOpcode()) { 618 default: 619#ifndef NDEBUG 620 cerr << "SplitVectorOperand Op #" << OpNo << ": "; 621 N->dump(&DAG); cerr << "\n"; 622#endif 623 assert(0 && "Do not know how to split this operator's operand!"); 624 abort(); 625 626 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 627 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 628 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 629 case ISD::STORE: Res = SplitVecOp_STORE(cast<StoreSDNode>(N), 630 OpNo); break; 631 case ISD::VECTOR_SHUFFLE: Res = SplitVecOp_VECTOR_SHUFFLE(N, OpNo);break; 632 } 633 } 634 635 // If the result is null, the sub-method took care of registering results etc. 636 if (!Res.getNode()) return false; 637 638 // If the result is N, the sub-method updated N in place. Check to see if any 639 // operands are new, and if so, mark them. 640 if (Res.getNode() == N) { 641 // Mark N as new and remark N and its operands. This allows us to correctly 642 // revisit N if it needs another step of promotion and allows us to visit 643 // any new operands to N. 644 ReanalyzeNode(N); 645 return true; 646 } 647 648 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 649 "Invalid operand expansion"); 650 651 ReplaceValueWith(SDValue(N, 0), Res); 652 return false; 653} 654 655SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 656 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 657 // end up being split all the way down to individual components. Convert the 658 // split pieces into integers and reassemble. 659 SDValue Lo, Hi; 660 GetSplitVector(N->getOperand(0), Lo, Hi); 661 Lo = BitConvertToInteger(Lo); 662 Hi = BitConvertToInteger(Hi); 663 664 if (TLI.isBigEndian()) 665 std::swap(Lo, Hi); 666 667 return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), 668 JoinIntegers(Lo, Hi)); 669} 670 671SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 672 // We know that the extracted result type is legal. For now, assume the index 673 // is a constant. 674 MVT SubVT = N->getValueType(0); 675 SDValue Idx = N->getOperand(1); 676 SDValue Lo, Hi; 677 GetSplitVector(N->getOperand(0), Lo, Hi); 678 679 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 680 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 681 682 if (IdxVal < LoElts) { 683 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 684 "Extracted subvector crosses vector split!"); 685 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SubVT, Lo, Idx); 686 } else { 687 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SubVT, Hi, 688 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 689 } 690} 691 692SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 693 SDValue Vec = N->getOperand(0); 694 SDValue Idx = N->getOperand(1); 695 MVT VecVT = Vec.getValueType(); 696 697 if (isa<ConstantSDNode>(Idx)) { 698 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 699 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 700 701 SDValue Lo, Hi; 702 GetSplitVector(Vec, Lo, Hi); 703 704 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 705 706 if (IdxVal < LoElts) 707 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 708 else 709 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 710 DAG.getConstant(IdxVal - LoElts, 711 Idx.getValueType())); 712 } 713 714 // Store the vector to the stack. 715 MVT EltVT = VecVT.getVectorElementType(); 716 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 717 SDValue Store = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); 718 719 // Load back the required element. 720 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 721 return DAG.getLoad(EltVT, Store, StackPtr, NULL, 0); 722} 723 724SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 725 assert(N->isUnindexed() && "Indexed store of vector?"); 726 assert(OpNo == 1 && "Can only split the stored value"); 727 728 bool isTruncating = N->isTruncatingStore(); 729 SDValue Ch = N->getChain(); 730 SDValue Ptr = N->getBasePtr(); 731 int SVOffset = N->getSrcValueOffset(); 732 MVT MemoryVT = N->getMemoryVT(); 733 unsigned Alignment = N->getAlignment(); 734 bool isVol = N->isVolatile(); 735 SDValue Lo, Hi; 736 GetSplitVector(N->getOperand(1), Lo, Hi); 737 738 MVT LoMemVT, HiMemVT; 739 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 740 741 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 742 743 if (isTruncating) 744 Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 745 LoMemVT, isVol, Alignment); 746 else 747 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 748 isVol, Alignment); 749 750 // Increment the pointer to the other half. 751 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 752 DAG.getIntPtrConstant(IncrementSize)); 753 754 if (isTruncating) 755 Hi = DAG.getTruncStore(Ch, Hi, Ptr, 756 N->getSrcValue(), SVOffset+IncrementSize, 757 HiMemVT, 758 isVol, MinAlign(Alignment, IncrementSize)); 759 else 760 Hi = DAG.getStore(Ch, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 761 isVol, MinAlign(Alignment, IncrementSize)); 762 763 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 764} 765 766SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_SHUFFLE(SDNode *N, unsigned OpNo){ 767 assert(OpNo == 2 && "Shuffle source type differs from result type?"); 768 SDValue Mask = N->getOperand(2); 769 unsigned MaskLength = Mask.getValueType().getVectorNumElements(); 770 unsigned LargestMaskEntryPlusOne = 2 * MaskLength; 771 unsigned MinimumBitWidth = Log2_32_Ceil(LargestMaskEntryPlusOne); 772 773 // Look for a legal vector type to place the mask values in. 774 // Note that there may not be *any* legal vector-of-integer 775 // type for which the element type is legal! 776 for (MVT::SimpleValueType EltVT = MVT::FIRST_INTEGER_VALUETYPE; 777 EltVT <= MVT::LAST_INTEGER_VALUETYPE; 778 // Integer values types are consecutively numbered. Exploit this. 779 EltVT = MVT::SimpleValueType(EltVT + 1)) { 780 781 // Is the element type big enough to hold the values? 782 if (MVT(EltVT).getSizeInBits() < MinimumBitWidth) 783 // Nope. 784 continue; 785 786 // Is the vector type legal? 787 MVT VecVT = MVT::getVectorVT(EltVT, MaskLength); 788 if (!isTypeLegal(VecVT)) 789 // Nope. 790 continue; 791 792 // If the element type is not legal, find a larger legal type to use for 793 // the BUILD_VECTOR operands. This is an ugly hack, but seems to work! 794 // FIXME: The real solution is to change VECTOR_SHUFFLE into a variadic 795 // node where the shuffle mask is a list of integer operands, #2 .. #2+n. 796 for (MVT::SimpleValueType OpVT = EltVT; OpVT <= MVT::LAST_INTEGER_VALUETYPE; 797 // Integer values types are consecutively numbered. Exploit this. 798 OpVT = MVT::SimpleValueType(OpVT + 1)) { 799 if (!isTypeLegal(OpVT)) 800 continue; 801 802 // Success! Rebuild the vector using the legal types. 803 SmallVector<SDValue, 16> Ops(MaskLength); 804 for (unsigned i = 0; i < MaskLength; ++i) { 805 uint64_t Idx = 806 cast<ConstantSDNode>(Mask.getOperand(i))->getZExtValue(); 807 Ops[i] = DAG.getConstant(Idx, OpVT); 808 } 809 return DAG.UpdateNodeOperands(SDValue(N,0), 810 N->getOperand(0), N->getOperand(1), 811 DAG.getNode(ISD::BUILD_VECTOR, 812 VecVT, &Ops[0], Ops.size())); 813 } 814 815 // Continuing is pointless - failure is certain. 816 break; 817 } 818 assert(false && "Failed to find an appropriate mask type!"); 819 return SDValue(N, 0); 820} 821