LegalizeVectorTypes.cpp revision 77cdf30742284a173fe818417eb482224cdee8d4
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in multiple vectors of a smaller type. For example, 19// implementing <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24using namespace llvm; 25 26//===----------------------------------------------------------------------===// 27// Result Vector Scalarization: <1 x ty> -> ty. 28//===----------------------------------------------------------------------===// 29 30void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 31 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 32 cerr << "\n"); 33 SDValue R = SDValue(); 34 35 switch (N->getOpcode()) { 36 default: 37#ifndef NDEBUG 38 cerr << "ScalarizeVectorResult #" << ResNo << ": "; 39 N->dump(&DAG); cerr << "\n"; 40#endif 41 assert(0 && "Do not know how to scalarize the result of this operator!"); 42 abort(); 43 44 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break; 45 case ISD::BUILD_VECTOR: R = N->getOperand(0); break; 46 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 47 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 48 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 49 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 50 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 51 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 52 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 53 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 54 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 55 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; 56 57 case ISD::CTLZ: 58 case ISD::CTPOP: 59 case ISD::CTTZ: 60 case ISD::FABS: 61 case ISD::FCOS: 62 case ISD::FNEG: 63 case ISD::FP_TO_SINT: 64 case ISD::FP_TO_UINT: 65 case ISD::FSIN: 66 case ISD::FSQRT: 67 case ISD::FTRUNC: 68 case ISD::FFLOOR: 69 case ISD::FCEIL: 70 case ISD::FRINT: 71 case ISD::FNEARBYINT: 72 case ISD::SINT_TO_FP: 73 case ISD::TRUNCATE: 74 case ISD::UINT_TO_FP: R = ScalarizeVecRes_UnaryOp(N); break; 75 76 case ISD::ADD: 77 case ISD::AND: 78 case ISD::FADD: 79 case ISD::FDIV: 80 case ISD::FMUL: 81 case ISD::FPOW: 82 case ISD::FREM: 83 case ISD::FSUB: 84 case ISD::MUL: 85 case ISD::OR: 86 case ISD::SDIV: 87 case ISD::SREM: 88 case ISD::SUB: 89 case ISD::UDIV: 90 case ISD::UREM: 91 case ISD::XOR: R = ScalarizeVecRes_BinOp(N); break; 92 } 93 94 // If R is null, the sub-method took care of registering the result. 95 if (R.getNode()) 96 SetScalarizedVector(SDValue(N, ResNo), R); 97} 98 99SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 100 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 101 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 102 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS); 103} 104 105SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) { 106 MVT NewVT = N->getValueType(0).getVectorElementType(); 107 return DAG.getNode(ISD::BIT_CONVERT, NewVT, N->getOperand(0)); 108} 109 110SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 111 MVT NewVT = N->getValueType(0).getVectorElementType(); 112 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 113 return DAG.getConvertRndSat(NewVT, Op0, DAG.getValueType(NewVT), 114 DAG.getValueType(Op0.getValueType()), 115 N->getOperand(3), 116 N->getOperand(4), 117 cast<CvtRndSatSDNode>(N)->getCvtCode()); 118} 119 120SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 121 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, 122 N->getValueType(0).getVectorElementType(), 123 N->getOperand(0), N->getOperand(1)); 124} 125 126SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 127 SDValue Op = GetScalarizedVector(N->getOperand(0)); 128 return DAG.getNode(ISD::FPOWI, Op.getValueType(), Op, N->getOperand(1)); 129} 130 131SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 132 // The value to insert may have a wider type than the vector element type, 133 // so be sure to truncate it to the element type if necessary. 134 SDValue Op = N->getOperand(1); 135 MVT EltVT = N->getValueType(0).getVectorElementType(); 136 if (Op.getValueType() != EltVT) 137 // FIXME: Can this happen for floating point types? 138 Op = DAG.getNode(ISD::TRUNCATE, EltVT, Op); 139 return Op; 140} 141 142SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 143 assert(N->isUnindexed() && "Indexed vector load?"); 144 145 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getExtensionType(), 146 N->getValueType(0).getVectorElementType(), 147 N->getChain(), N->getBasePtr(), 148 DAG.getNode(ISD::UNDEF, 149 N->getBasePtr().getValueType()), 150 N->getSrcValue(), N->getSrcValueOffset(), 151 N->getMemoryVT().getVectorElementType(), 152 N->isVolatile(), N->getAlignment()); 153 154 // Legalized the chain result - switch anything that used the old chain to 155 // use the new one. 156 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 157 return Result; 158} 159 160SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 161 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 162 MVT DestVT = N->getValueType(0).getVectorElementType(); 163 SDValue Op = GetScalarizedVector(N->getOperand(0)); 164 return DAG.getNode(N->getOpcode(), DestVT, Op); 165} 166 167SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 168 return DAG.getNode(ISD::UNDEF, N->getValueType(0).getVectorElementType()); 169} 170 171SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 172 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 173 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0), LHS, 174 GetScalarizedVector(N->getOperand(2))); 175} 176 177SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 178 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 179 return DAG.getNode(ISD::SELECT_CC, LHS.getValueType(), 180 N->getOperand(0), N->getOperand(1), 181 LHS, GetScalarizedVector(N->getOperand(3)), 182 N->getOperand(4)); 183} 184 185SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 186 // Figure out if the scalar is the LHS or RHS and return it. 187 SDValue Arg = N->getOperand(2).getOperand(0); 188 if (Arg.getOpcode() == ISD::UNDEF) 189 return DAG.getNode(ISD::UNDEF, N->getValueType(0).getVectorElementType()); 190 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 191 return GetScalarizedVector(N->getOperand(Op)); 192} 193 194SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 195 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 196 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 197 MVT NVT = N->getValueType(0).getVectorElementType(); 198 MVT SVT = TLI.getSetCCResultType(LHS); 199 200 // Turn it into a scalar SETCC. 201 SDValue Res = DAG.getNode(ISD::SETCC, SVT, LHS, RHS, N->getOperand(2)); 202 203 // VSETCC always returns a sign-extended value, while SETCC may not. The 204 // SETCC result type may not match the vector element type. Correct these. 205 if (NVT.bitsLE(SVT)) { 206 // The SETCC result type is bigger than the vector element type. 207 // Ensure the SETCC result is sign-extended. 208 if (TLI.getSetCCResultContents() != 209 TargetLowering::ZeroOrNegativeOneSetCCResult) 210 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, SVT, Res, 211 DAG.getValueType(MVT::i1)); 212 // Truncate to the final type. 213 return DAG.getNode(ISD::TRUNCATE, NVT, Res); 214 } else { 215 // The SETCC result type is smaller than the vector element type. 216 // If the SetCC result is not sign-extended, chop it down to MVT::i1. 217 if (TLI.getSetCCResultContents() != 218 TargetLowering::ZeroOrNegativeOneSetCCResult) 219 Res = DAG.getNode(ISD::TRUNCATE, MVT::i1, Res); 220 // Sign extend to the final type. 221 return DAG.getNode(ISD::SIGN_EXTEND, NVT, Res); 222 } 223} 224 225 226//===----------------------------------------------------------------------===// 227// Operand Vector Scalarization <1 x ty> -> ty. 228//===----------------------------------------------------------------------===// 229 230bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 231 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 232 cerr << "\n"); 233 SDValue Res = SDValue(); 234 235 if (Res.getNode() == 0) { 236 switch (N->getOpcode()) { 237 default: 238#ifndef NDEBUG 239 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": "; 240 N->dump(&DAG); cerr << "\n"; 241#endif 242 assert(0 && "Do not know how to scalarize this operator's operand!"); 243 abort(); 244 245 case ISD::BIT_CONVERT: 246 Res = ScalarizeVecOp_BIT_CONVERT(N); break; 247 248 case ISD::CONCAT_VECTORS: 249 Res = ScalarizeVecOp_CONCAT_VECTORS(N); break; 250 251 case ISD::EXTRACT_VECTOR_ELT: 252 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); break; 253 254 case ISD::STORE: 255 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); break; 256 } 257 } 258 259 // If the result is null, the sub-method took care of registering results etc. 260 if (!Res.getNode()) return false; 261 262 // If the result is N, the sub-method updated N in place. Check to see if any 263 // operands are new, and if so, mark them. 264 if (Res.getNode() == N) { 265 // Mark N as new and remark N and its operands. This allows us to correctly 266 // revisit N if it needs another step of promotion and allows us to visit 267 // any new operands to N. 268 ReanalyzeNode(N); 269 return true; 270 } 271 272 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 273 "Invalid operand expansion"); 274 275 ReplaceValueWith(SDValue(N, 0), Res); 276 return false; 277} 278 279/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs 280/// to be scalarized, it must be <1 x ty>. Convert the element instead. 281SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) { 282 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 283 return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Elt); 284} 285 286/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 287/// use a BUILD_VECTOR instead. 288SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 289 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 290 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 291 Ops[i] = GetScalarizedVector(N->getOperand(i)); 292 return DAG.getNode(ISD::BUILD_VECTOR, N->getValueType(0), 293 &Ops[0], Ops.size()); 294} 295 296/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 297/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 298/// index. 299SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 300 return GetScalarizedVector(N->getOperand(0)); 301} 302 303/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 304/// scalarized, it must be <1 x ty>. Just store the element. 305SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 306 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 307 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 308 309 if (N->isTruncatingStore()) 310 return DAG.getTruncStore(N->getChain(), 311 GetScalarizedVector(N->getOperand(1)), 312 N->getBasePtr(), 313 N->getSrcValue(), N->getSrcValueOffset(), 314 N->getMemoryVT().getVectorElementType(), 315 N->isVolatile(), N->getAlignment()); 316 317 return DAG.getStore(N->getChain(), GetScalarizedVector(N->getOperand(1)), 318 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), 319 N->isVolatile(), N->getAlignment()); 320} 321 322 323//===----------------------------------------------------------------------===// 324// Result Vector Splitting 325//===----------------------------------------------------------------------===// 326 327/// SplitVectorResult - This method is called when the specified result of the 328/// specified node is found to need vector splitting. At this point, the node 329/// may also have invalid operands or may have other results that need 330/// legalization, we just know that (at least) one result needs vector 331/// splitting. 332void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 333 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n"); 334 SDValue Lo, Hi; 335 336 switch (N->getOpcode()) { 337 default: 338#ifndef NDEBUG 339 cerr << "SplitVectorResult #" << ResNo << ": "; 340 N->dump(&DAG); cerr << "\n"; 341#endif 342 assert(0 && "Do not know how to split the result of this operator!"); 343 abort(); 344 345 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; 346 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 347 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 348 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 349 350 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break; 351 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 352 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 353 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break; 354 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 355 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 356 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 357 case ISD::LOAD: SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);break; 358 case ISD::VECTOR_SHUFFLE: SplitVecRes_VECTOR_SHUFFLE(N, Lo, Hi); break; 359 case ISD::VSETCC: SplitVecRes_VSETCC(N, Lo, Hi); break; 360 361 case ISD::CTTZ: 362 case ISD::CTLZ: 363 case ISD::CTPOP: 364 case ISD::FNEG: 365 case ISD::FABS: 366 case ISD::FSQRT: 367 case ISD::FSIN: 368 case ISD::FCOS: 369 case ISD::FTRUNC: 370 case ISD::FFLOOR: 371 case ISD::FCEIL: 372 case ISD::FRINT: 373 case ISD::FNEARBYINT: 374 case ISD::FP_TO_SINT: 375 case ISD::FP_TO_UINT: 376 case ISD::SINT_TO_FP: 377 case ISD::TRUNCATE: 378 case ISD::UINT_TO_FP: SplitVecRes_UnaryOp(N, Lo, Hi); break; 379 380 case ISD::ADD: 381 case ISD::SUB: 382 case ISD::MUL: 383 case ISD::FADD: 384 case ISD::FSUB: 385 case ISD::FMUL: 386 case ISD::SDIV: 387 case ISD::UDIV: 388 case ISD::FDIV: 389 case ISD::FPOW: 390 case ISD::AND: 391 case ISD::OR: 392 case ISD::XOR: 393 case ISD::UREM: 394 case ISD::SREM: 395 case ISD::FREM: SplitVecRes_BinOp(N, Lo, Hi); break; 396 } 397 398 // If Lo/Hi is null, the sub-method took care of registering results etc. 399 if (Lo.getNode()) 400 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 401} 402 403void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 404 SDValue &Hi) { 405 SDValue LHSLo, LHSHi; 406 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 407 SDValue RHSLo, RHSHi; 408 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 409 410 Lo = DAG.getNode(N->getOpcode(), LHSLo.getValueType(), LHSLo, RHSLo); 411 Hi = DAG.getNode(N->getOpcode(), LHSHi.getValueType(), LHSHi, RHSHi); 412} 413 414void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo, 415 SDValue &Hi) { 416 // We know the result is a vector. The input may be either a vector or a 417 // scalar value. 418 MVT LoVT, HiVT; 419 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 420 421 SDValue InOp = N->getOperand(0); 422 MVT InVT = InOp.getValueType(); 423 424 // Handle some special cases efficiently. 425 switch (getTypeAction(InVT)) { 426 default: 427 assert(false && "Unknown type action!"); 428 case Legal: 429 case PromoteInteger: 430 case SoftenFloat: 431 case ScalarizeVector: 432 break; 433 case ExpandInteger: 434 case ExpandFloat: 435 // A scalar to vector conversion, where the scalar needs expansion. 436 // If the vector is being split in two then we can just convert the 437 // expanded pieces. 438 if (LoVT == HiVT) { 439 GetExpandedOp(InOp, Lo, Hi); 440 if (TLI.isBigEndian()) 441 std::swap(Lo, Hi); 442 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 443 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 444 return; 445 } 446 break; 447 case SplitVector: 448 // If the input is a vector that needs to be split, convert each split 449 // piece of the input now. 450 GetSplitVector(InOp, Lo, Hi); 451 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 452 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 453 return; 454 } 455 456 // In the general case, convert the input to an integer and split it by hand. 457 MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits()); 458 MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits()); 459 if (TLI.isBigEndian()) 460 std::swap(LoIntVT, HiIntVT); 461 462 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 463 464 if (TLI.isBigEndian()) 465 std::swap(Lo, Hi); 466 Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo); 467 Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi); 468} 469 470void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 471 SDValue &Hi) { 472 MVT LoVT, HiVT; 473 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 474 unsigned LoNumElts = LoVT.getVectorNumElements(); 475 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 476 Lo = DAG.getNode(ISD::BUILD_VECTOR, LoVT, &LoOps[0], LoOps.size()); 477 478 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 479 Hi = DAG.getNode(ISD::BUILD_VECTOR, HiVT, &HiOps[0], HiOps.size()); 480} 481 482void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 483 SDValue &Hi) { 484 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 485 unsigned NumSubvectors = N->getNumOperands() / 2; 486 if (NumSubvectors == 1) { 487 Lo = N->getOperand(0); 488 Hi = N->getOperand(1); 489 return; 490 } 491 492 MVT LoVT, HiVT; 493 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 494 495 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 496 Lo = DAG.getNode(ISD::CONCAT_VECTORS, LoVT, &LoOps[0], LoOps.size()); 497 498 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 499 Hi = DAG.getNode(ISD::CONCAT_VECTORS, HiVT, &HiOps[0], HiOps.size()); 500} 501 502void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo, 503 SDValue &Hi) { 504 MVT LoVT, HiVT; 505 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 506 SDValue VLo, VHi; 507 GetSplitVector(N->getOperand(0), VLo, VHi); 508 SDValue DTyOpLo = DAG.getValueType(LoVT); 509 SDValue DTyOpHi = DAG.getValueType(HiVT); 510 SDValue STyOpLo = DAG.getValueType(VLo.getValueType()); 511 SDValue STyOpHi = DAG.getValueType(VHi.getValueType()); 512 513 SDValue RndOp = N->getOperand(3); 514 SDValue SatOp = N->getOperand(4); 515 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 516 517 Lo = DAG.getConvertRndSat(LoVT, VLo, DTyOpLo, STyOpLo, RndOp, SatOp, CvtCode); 518 Hi = DAG.getConvertRndSat(HiVT, VHi, DTyOpHi, STyOpHi, RndOp, SatOp, CvtCode); 519} 520 521void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 522 SDValue &Hi) { 523 MVT LoVT, HiVT; 524 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 525 unsigned LoNumElts = LoVT.getVectorNumElements(); 526 527 SDValue Vec = N->getOperand(0); 528 SDValue Idx = N->getOperand(1); 529 MVT IdxVT = Idx.getValueType(); 530 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, LoVT, Vec, Idx); 531 532 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx); 533 if (CIdx) { 534 unsigned IdxVal = CIdx->getZExtValue(); 535 assert (IdxVal % LoVT.getVectorNumElements() == 0 && 536 (IdxVal+LoNumElts) % HiVT.getVectorNumElements()==0 && 537 "Index must be a multiple of the result type"); 538 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, HiVT, Vec, 539 DAG.getConstant(IdxVal + LoNumElts, IdxVT)); 540 } else { 541 assert(LoVT == HiVT && "Low and High value type should be the same"); 542 Idx = DAG.getNode(ISD::ADD, IdxVT, Idx, DAG.getConstant(LoNumElts, IdxVT)); 543 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, HiVT, Vec, Idx); 544 } 545} 546 547void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 548 SDValue &Hi) { 549 GetSplitVector(N->getOperand(0), Lo, Hi); 550 Lo = DAG.getNode(ISD::FPOWI, Lo.getValueType(), Lo, N->getOperand(1)); 551 Hi = DAG.getNode(ISD::FPOWI, Hi.getValueType(), Hi, N->getOperand(1)); 552} 553 554void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 555 SDValue &Hi) { 556 SDValue Vec = N->getOperand(0); 557 SDValue Elt = N->getOperand(1); 558 SDValue Idx = N->getOperand(2); 559 GetSplitVector(Vec, Lo, Hi); 560 561 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 562 unsigned IdxVal = CIdx->getZExtValue(); 563 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 564 if (IdxVal < LoNumElts) 565 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, Lo.getValueType(), Lo, Elt, Idx); 566 else 567 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, Hi.getValueType(), Hi, Elt, 568 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 569 return; 570 } 571 572 // Spill the vector to the stack. 573 MVT VecVT = Vec.getValueType(); 574 MVT EltVT = VecVT.getVectorElementType(); 575 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 576 SDValue Store = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); 577 578 // Store the new element. This may be larger than the vector element type, 579 // so use a truncating store. 580 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 581 Store = DAG.getTruncStore(Store, Elt, EltPtr, NULL, 0, EltVT); 582 583 // Reload the vector from the stack. 584 SDValue Load = DAG.getLoad(VecVT, Store, StackPtr, NULL, 0); 585 586 // Split it. 587 SplitVecRes_LOAD(cast<LoadSDNode>(Load.getNode()), Lo, Hi); 588} 589 590void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 591 SDValue &Hi) { 592 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 593 MVT LoVT, HiVT; 594 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 595 596 ISD::LoadExtType ExtType = LD->getExtensionType(); 597 SDValue Ch = LD->getChain(); 598 SDValue Ptr = LD->getBasePtr(); 599 SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType()); 600 const Value *SV = LD->getSrcValue(); 601 int SVOffset = LD->getSrcValueOffset(); 602 MVT MemoryVT = LD->getMemoryVT(); 603 unsigned Alignment = LD->getAlignment(); 604 bool isVolatile = LD->isVolatile(); 605 606 MVT LoMemVT, HiMemVT; 607 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 608 609 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, Ch, Ptr, Offset, 610 SV, SVOffset, LoMemVT, isVolatile, Alignment); 611 612 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 613 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 614 DAG.getIntPtrConstant(IncrementSize)); 615 SVOffset += IncrementSize; 616 Alignment = MinAlign(Alignment, IncrementSize); 617 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, Ch, Ptr, Offset, 618 SV, SVOffset, HiMemVT, isVolatile, Alignment); 619 620 // Build a factor node to remember that this load is independent of the 621 // other one. 622 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), 623 Hi.getValue(1)); 624 625 // Legalized the chain result - switch anything that used the old chain to 626 // use the new one. 627 ReplaceValueWith(SDValue(LD, 1), Ch); 628} 629 630void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 631 SDValue &Hi) { 632 // Get the dest types - they may not match the input types, e.g. int_to_fp. 633 MVT LoVT, HiVT; 634 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 635 636 // Split the input. 637 MVT InVT = N->getOperand(0).getValueType(); 638 switch (getTypeAction(InVT)) { 639 default: assert(0 && "Unexpected type action!"); 640 case Legal: { 641 assert(LoVT == HiVT && "Legal non-power-of-two vector type?"); 642 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(), 643 LoVT.getVectorNumElements()); 644 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, InNVT, N->getOperand(0), 645 DAG.getIntPtrConstant(0)); 646 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, InNVT, N->getOperand(0), 647 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 648 break; 649 } 650 case SplitVector: 651 GetSplitVector(N->getOperand(0), Lo, Hi); 652 break; 653 } 654 655 Lo = DAG.getNode(N->getOpcode(), LoVT, Lo); 656 Hi = DAG.getNode(N->getOpcode(), HiVT, Hi); 657} 658 659void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(SDNode *N, SDValue &Lo, 660 SDValue &Hi) { 661 // Build the low part. 662 SDValue Mask = N->getOperand(2); 663 SmallVector<SDValue, 16> Ops; 664 MVT LoVT, HiVT; 665 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 666 MVT EltVT = LoVT.getVectorElementType(); 667 unsigned LoNumElts = LoVT.getVectorNumElements(); 668 unsigned NumElements = Mask.getNumOperands(); 669 670 // Insert all of the elements from the input that are needed. We use 671 // buildvector of extractelement here because the input vectors will have 672 // to be legalized, so this makes the code simpler. 673 for (unsigned i = 0; i != LoNumElts; ++i) { 674 SDValue Arg = Mask.getOperand(i); 675 if (Arg.getOpcode() == ISD::UNDEF) { 676 Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 677 } else { 678 unsigned Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getZExtValue(); 679 SDValue InVec = N->getOperand(0); 680 if (Idx >= NumElements) { 681 InVec = N->getOperand(1); 682 Idx -= NumElements; 683 } 684 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, InVec, 685 DAG.getIntPtrConstant(Idx))); 686 } 687 } 688 Lo = DAG.getNode(ISD::BUILD_VECTOR, LoVT, &Ops[0], Ops.size()); 689 Ops.clear(); 690 691 for (unsigned i = LoNumElts; i != NumElements; ++i) { 692 SDValue Arg = Mask.getOperand(i); 693 if (Arg.getOpcode() == ISD::UNDEF) { 694 Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT)); 695 } else { 696 unsigned Idx = cast<ConstantSDNode>(Mask.getOperand(i))->getZExtValue(); 697 SDValue InVec = N->getOperand(0); 698 if (Idx >= NumElements) { 699 InVec = N->getOperand(1); 700 Idx -= NumElements; 701 } 702 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, InVec, 703 DAG.getIntPtrConstant(Idx))); 704 } 705 } 706 Hi = DAG.getNode(ISD::BUILD_VECTOR, HiVT, &Ops[0], Ops.size()); 707} 708 709void DAGTypeLegalizer::SplitVecRes_VSETCC(SDNode *N, SDValue &Lo, 710 SDValue &Hi) { 711 MVT LoVT, HiVT; 712 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 713 714 SDValue LL, LH, RL, RH; 715 GetSplitVector(N->getOperand(0), LL, LH); 716 GetSplitVector(N->getOperand(1), RL, RH); 717 718 Lo = DAG.getNode(ISD::VSETCC, LoVT, LL, RL, N->getOperand(2)); 719 Hi = DAG.getNode(ISD::VSETCC, HiVT, LH, RH, N->getOperand(2)); 720} 721 722 723//===----------------------------------------------------------------------===// 724// Operand Vector Splitting 725//===----------------------------------------------------------------------===// 726 727/// SplitVectorOperand - This method is called when the specified operand of the 728/// specified node is found to need vector splitting. At this point, all of the 729/// result types of the node are known to be legal, but other operands of the 730/// node may need legalization as well as the specified one. 731bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 732 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n"); 733 SDValue Res = SDValue(); 734 735 if (Res.getNode() == 0) { 736 switch (N->getOpcode()) { 737 default: 738#ifndef NDEBUG 739 cerr << "SplitVectorOperand Op #" << OpNo << ": "; 740 N->dump(&DAG); cerr << "\n"; 741#endif 742 assert(0 && "Do not know how to split this operator's operand!"); 743 abort(); 744 745 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break; 746 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 747 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 748 case ISD::STORE: Res = SplitVecOp_STORE(cast<StoreSDNode>(N), 749 OpNo); break; 750 case ISD::VECTOR_SHUFFLE: Res = SplitVecOp_VECTOR_SHUFFLE(N, OpNo);break; 751 752 case ISD::CTTZ: 753 case ISD::CTLZ: 754 case ISD::CTPOP: 755 case ISD::FP_TO_SINT: 756 case ISD::FP_TO_UINT: 757 case ISD::SINT_TO_FP: 758 case ISD::TRUNCATE: 759 case ISD::UINT_TO_FP: Res = SplitVecOp_UnaryOp(N); break; 760 } 761 } 762 763 // If the result is null, the sub-method took care of registering results etc. 764 if (!Res.getNode()) return false; 765 766 // If the result is N, the sub-method updated N in place. Check to see if any 767 // operands are new, and if so, mark them. 768 if (Res.getNode() == N) { 769 // Mark N as new and remark N and its operands. This allows us to correctly 770 // revisit N if it needs another step of promotion and allows us to visit 771 // any new operands to N. 772 ReanalyzeNode(N); 773 return true; 774 } 775 776 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 777 "Invalid operand expansion"); 778 779 ReplaceValueWith(SDValue(N, 0), Res); 780 return false; 781} 782 783SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 784 // The result has a legal vector type, but the input needs splitting. 785 MVT ResVT = N->getValueType(0); 786 SDValue Lo, Hi; 787 GetSplitVector(N->getOperand(0), Lo, Hi); 788 assert(Lo.getValueType() == Hi.getValueType() && 789 "Returns legal non-power-of-two vector type?"); 790 MVT InVT = Lo.getValueType(); 791 792 MVT OutVT = MVT::getVectorVT(ResVT.getVectorElementType(), 793 InVT.getVectorNumElements()); 794 795 Lo = DAG.getNode(N->getOpcode(), OutVT, Lo); 796 Hi = DAG.getNode(N->getOpcode(), OutVT, Hi); 797 798 return DAG.getNode(ISD::CONCAT_VECTORS, ResVT, Lo, Hi); 799} 800 801SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) { 802 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will 803 // end up being split all the way down to individual components. Convert the 804 // split pieces into integers and reassemble. 805 SDValue Lo, Hi; 806 GetSplitVector(N->getOperand(0), Lo, Hi); 807 Lo = BitConvertToInteger(Lo); 808 Hi = BitConvertToInteger(Hi); 809 810 if (TLI.isBigEndian()) 811 std::swap(Lo, Hi); 812 813 return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), 814 JoinIntegers(Lo, Hi)); 815} 816 817SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 818 // We know that the extracted result type is legal. For now, assume the index 819 // is a constant. 820 MVT SubVT = N->getValueType(0); 821 SDValue Idx = N->getOperand(1); 822 SDValue Lo, Hi; 823 GetSplitVector(N->getOperand(0), Lo, Hi); 824 825 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 826 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 827 828 if (IdxVal < LoElts) { 829 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 830 "Extracted subvector crosses vector split!"); 831 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SubVT, Lo, Idx); 832 } else { 833 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SubVT, Hi, 834 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 835 } 836} 837 838SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 839 SDValue Vec = N->getOperand(0); 840 SDValue Idx = N->getOperand(1); 841 MVT VecVT = Vec.getValueType(); 842 843 if (isa<ConstantSDNode>(Idx)) { 844 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 845 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 846 847 SDValue Lo, Hi; 848 GetSplitVector(Vec, Lo, Hi); 849 850 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 851 852 if (IdxVal < LoElts) 853 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx); 854 else 855 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi, 856 DAG.getConstant(IdxVal - LoElts, 857 Idx.getValueType())); 858 } 859 860 // Store the vector to the stack. 861 MVT EltVT = VecVT.getVectorElementType(); 862 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 863 SDValue Store = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); 864 865 // Load back the required element. 866 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 867 return DAG.getLoad(EltVT, Store, StackPtr, NULL, 0); 868} 869 870SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 871 assert(N->isUnindexed() && "Indexed store of vector?"); 872 assert(OpNo == 1 && "Can only split the stored value"); 873 874 bool isTruncating = N->isTruncatingStore(); 875 SDValue Ch = N->getChain(); 876 SDValue Ptr = N->getBasePtr(); 877 int SVOffset = N->getSrcValueOffset(); 878 MVT MemoryVT = N->getMemoryVT(); 879 unsigned Alignment = N->getAlignment(); 880 bool isVol = N->isVolatile(); 881 SDValue Lo, Hi; 882 GetSplitVector(N->getOperand(1), Lo, Hi); 883 884 MVT LoMemVT, HiMemVT; 885 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 886 887 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 888 889 if (isTruncating) 890 Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 891 LoMemVT, isVol, Alignment); 892 else 893 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, 894 isVol, Alignment); 895 896 // Increment the pointer to the other half. 897 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, 898 DAG.getIntPtrConstant(IncrementSize)); 899 900 if (isTruncating) 901 Hi = DAG.getTruncStore(Ch, Hi, Ptr, 902 N->getSrcValue(), SVOffset+IncrementSize, 903 HiMemVT, 904 isVol, MinAlign(Alignment, IncrementSize)); 905 else 906 Hi = DAG.getStore(Ch, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, 907 isVol, MinAlign(Alignment, IncrementSize)); 908 909 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); 910} 911 912SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_SHUFFLE(SDNode *N, unsigned OpNo) { 913 assert(OpNo == 2 && "Shuffle source type differs from result type?"); 914 SDValue Mask = N->getOperand(2); 915 unsigned MaskLength = Mask.getValueType().getVectorNumElements(); 916 unsigned LargestMaskEntryPlusOne = 2 * MaskLength; 917 unsigned MinimumBitWidth = Log2_32_Ceil(LargestMaskEntryPlusOne); 918 919 // Look for a legal vector type to place the mask values in. 920 // Note that there may not be *any* legal vector-of-integer 921 // type for which the element type is legal! 922 for (MVT::SimpleValueType EltVT = MVT::FIRST_INTEGER_VALUETYPE; 923 EltVT <= MVT::LAST_INTEGER_VALUETYPE; 924 // Integer values types are consecutively numbered. Exploit this. 925 EltVT = MVT::SimpleValueType(EltVT + 1)) { 926 927 // Is the element type big enough to hold the values? 928 if (MVT(EltVT).getSizeInBits() < MinimumBitWidth) 929 // Nope. 930 continue; 931 932 // Is the vector type legal? 933 MVT VecVT = MVT::getVectorVT(EltVT, MaskLength); 934 if (!isTypeLegal(VecVT)) 935 // Nope. 936 continue; 937 938 // If the element type is not legal, find a larger legal type to use for 939 // the BUILD_VECTOR operands. This is an ugly hack, but seems to work! 940 // FIXME: The real solution is to change VECTOR_SHUFFLE into a variadic 941 // node where the shuffle mask is a list of integer operands, #2 .. #2+n. 942 for (MVT::SimpleValueType OpVT = EltVT; OpVT <= MVT::LAST_INTEGER_VALUETYPE; 943 // Integer values types are consecutively numbered. Exploit this. 944 OpVT = MVT::SimpleValueType(OpVT + 1)) { 945 if (!isTypeLegal(OpVT)) 946 continue; 947 948 // Success! Rebuild the vector using the legal types. 949 SmallVector<SDValue, 16> Ops(MaskLength); 950 for (unsigned i = 0; i < MaskLength; ++i) { 951 SDValue Arg = Mask.getOperand(i); 952 if (Arg.getOpcode() == ISD::UNDEF) { 953 Ops[i] = DAG.getNode(ISD::UNDEF, OpVT); 954 } else { 955 uint64_t Idx = cast<ConstantSDNode>(Arg)->getZExtValue(); 956 Ops[i] = DAG.getConstant(Idx, OpVT); 957 } 958 } 959 return DAG.UpdateNodeOperands(SDValue(N,0), 960 N->getOperand(0), N->getOperand(1), 961 DAG.getNode(ISD::BUILD_VECTOR, 962 VecVT, &Ops[0], Ops.size())); 963 } 964 965 // Continuing is pointless - failure is certain. 966 break; 967 } 968 assert(false && "Failed to find an appropriate mask type!"); 969 return SDValue(N, 0); 970} 971