LegalizeVectorTypes.cpp revision f06aaf11fff0e6fa12d4ee959569a263bf7bd779
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file performs vector type splitting and scalarization for LegalizeTypes. 11// Scalarization is the act of changing a computation in an illegal one-element 12// vector type to be a computation in its scalar element type. For example, 13// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16// types. 17// Splitting is the act of changing a computation in an invalid vector type to 18// be a computation in two vectors of half the size. For example, implementing 19// <128 x f32> operations in terms of two <64 x f32> operations. 20// 21//===----------------------------------------------------------------------===// 22 23#include "LegalizeTypes.h" 24#include "llvm/IR/DataLayout.h" 25#include "llvm/Support/ErrorHandling.h" 26#include "llvm/Support/raw_ostream.h" 27using namespace llvm; 28 29#define DEBUG_TYPE "legalize-types" 30 31//===----------------------------------------------------------------------===// 32// Result Vector Scalarization: <1 x ty> -> ty. 33//===----------------------------------------------------------------------===// 34 35void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 36 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 37 N->dump(&DAG); 38 dbgs() << "\n"); 39 SDValue R = SDValue(); 40 41 switch (N->getOpcode()) { 42 default: 43#ifndef NDEBUG 44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 45 N->dump(&DAG); 46 dbgs() << "\n"; 47#endif 48 report_fatal_error("Do not know how to scalarize the result of this " 49 "operator!\n"); 50 51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 54 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 55 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 56 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 57 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 58 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 59 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 60 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 61 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 62 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 63 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 64 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 65 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 66 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 67 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 68 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 69 case ISD::ANY_EXTEND: 70 case ISD::BSWAP: 71 case ISD::CTLZ: 72 case ISD::CTLZ_ZERO_UNDEF: 73 case ISD::CTPOP: 74 case ISD::CTTZ: 75 case ISD::FABS: 76 case ISD::FCEIL: 77 case ISD::FCOS: 78 case ISD::FEXP: 79 case ISD::FEXP2: 80 case ISD::FFLOOR: 81 case ISD::FLOG: 82 case ISD::FLOG10: 83 case ISD::FLOG2: 84 case ISD::FNEARBYINT: 85 case ISD::FNEG: 86 case ISD::FP_EXTEND: 87 case ISD::FP_TO_SINT: 88 case ISD::FP_TO_UINT: 89 case ISD::FRINT: 90 case ISD::FROUND: 91 case ISD::FSIN: 92 case ISD::FSQRT: 93 case ISD::FTRUNC: 94 case ISD::SIGN_EXTEND: 95 case ISD::SINT_TO_FP: 96 case ISD::TRUNCATE: 97 case ISD::UINT_TO_FP: 98 case ISD::ZERO_EXTEND: 99 R = ScalarizeVecRes_UnaryOp(N); 100 break; 101 102 case ISD::ADD: 103 case ISD::AND: 104 case ISD::FADD: 105 case ISD::FCOPYSIGN: 106 case ISD::FDIV: 107 case ISD::FMUL: 108 case ISD::FPOW: 109 case ISD::FREM: 110 case ISD::FSUB: 111 case ISD::MUL: 112 case ISD::OR: 113 case ISD::SDIV: 114 case ISD::SREM: 115 case ISD::SUB: 116 case ISD::UDIV: 117 case ISD::UREM: 118 case ISD::XOR: 119 case ISD::SHL: 120 case ISD::SRA: 121 case ISD::SRL: 122 R = ScalarizeVecRes_BinOp(N); 123 break; 124 case ISD::FMA: 125 R = ScalarizeVecRes_TernaryOp(N); 126 break; 127 } 128 129 // If R is null, the sub-method took care of registering the result. 130 if (R.getNode()) 131 SetScalarizedVector(SDValue(N, ResNo), R); 132} 133 134SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 135 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 136 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 137 return DAG.getNode(N->getOpcode(), SDLoc(N), 138 LHS.getValueType(), LHS, RHS); 139} 140 141SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 142 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 143 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 144 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 145 return DAG.getNode(N->getOpcode(), SDLoc(N), 146 Op0.getValueType(), Op0, Op1, Op2); 147} 148 149SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 150 unsigned ResNo) { 151 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 152 return GetScalarizedVector(Op); 153} 154 155SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 156 EVT NewVT = N->getValueType(0).getVectorElementType(); 157 return DAG.getNode(ISD::BITCAST, SDLoc(N), 158 NewVT, N->getOperand(0)); 159} 160 161SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 162 EVT EltVT = N->getValueType(0).getVectorElementType(); 163 SDValue InOp = N->getOperand(0); 164 // The BUILD_VECTOR operands may be of wider element types and 165 // we may need to truncate them back to the requested return type. 166 if (EltVT.isInteger()) 167 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 168 return InOp; 169} 170 171SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 172 EVT NewVT = N->getValueType(0).getVectorElementType(); 173 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 174 return DAG.getConvertRndSat(NewVT, SDLoc(N), 175 Op0, DAG.getValueType(NewVT), 176 DAG.getValueType(Op0.getValueType()), 177 N->getOperand(3), 178 N->getOperand(4), 179 cast<CvtRndSatSDNode>(N)->getCvtCode()); 180} 181 182SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 183 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 184 N->getValueType(0).getVectorElementType(), 185 N->getOperand(0), N->getOperand(1)); 186} 187 188SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 189 EVT NewVT = N->getValueType(0).getVectorElementType(); 190 SDValue Op = GetScalarizedVector(N->getOperand(0)); 191 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 192 NewVT, Op, N->getOperand(1)); 193} 194 195SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 196 SDValue Op = GetScalarizedVector(N->getOperand(0)); 197 return DAG.getNode(ISD::FPOWI, SDLoc(N), 198 Op.getValueType(), Op, N->getOperand(1)); 199} 200 201SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 202 // The value to insert may have a wider type than the vector element type, 203 // so be sure to truncate it to the element type if necessary. 204 SDValue Op = N->getOperand(1); 205 EVT EltVT = N->getValueType(0).getVectorElementType(); 206 if (Op.getValueType() != EltVT) 207 // FIXME: Can this happen for floating point types? 208 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 209 return Op; 210} 211 212SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 213 assert(N->isUnindexed() && "Indexed vector load?"); 214 215 SDValue Result = DAG.getLoad(ISD::UNINDEXED, 216 N->getExtensionType(), 217 N->getValueType(0).getVectorElementType(), 218 SDLoc(N), 219 N->getChain(), N->getBasePtr(), 220 DAG.getUNDEF(N->getBasePtr().getValueType()), 221 N->getPointerInfo(), 222 N->getMemoryVT().getVectorElementType(), 223 N->isVolatile(), N->isNonTemporal(), 224 N->isInvariant(), N->getOriginalAlignment(), 225 N->getTBAAInfo()); 226 227 // Legalized the chain result - switch anything that used the old chain to 228 // use the new one. 229 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 230 return Result; 231} 232 233SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 234 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 235 EVT DestVT = N->getValueType(0).getVectorElementType(); 236 SDValue Op = GetScalarizedVector(N->getOperand(0)); 237 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op); 238} 239 240SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 241 EVT EltVT = N->getValueType(0).getVectorElementType(); 242 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 243 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 244 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 245 LHS, DAG.getValueType(ExtVT)); 246} 247 248SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 249 // If the operand is wider than the vector element type then it is implicitly 250 // truncated. Make that explicit here. 251 EVT EltVT = N->getValueType(0).getVectorElementType(); 252 SDValue InOp = N->getOperand(0); 253 if (InOp.getValueType() != EltVT) 254 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 255 return InOp; 256} 257 258SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 259 SDValue Cond = GetScalarizedVector(N->getOperand(0)); 260 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 261 TargetLowering::BooleanContent ScalarBool = 262 TLI.getBooleanContents(false, false); 263 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 264 265 // If integer and float booleans have different contents then we can't 266 // reliably optimize in all cases. There is a full explanation for this in 267 // DAGCombiner::visitSELECT() where the same issue affects folding 268 // (select C, 0, 1) to (xor C, 1). 269 if (TLI.getBooleanContents(false, false) != 270 TLI.getBooleanContents(false, true)) { 271 // At least try the common case where the boolean is generated by a 272 // comparison. 273 if (Cond->getOpcode() == ISD::SETCC) { 274 EVT OpVT = Cond->getOperand(0)->getValueType(0); 275 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 276 VecBool = TLI.getBooleanContents(OpVT); 277 } else 278 ScalarBool = TargetLowering::UndefinedBooleanContent; 279 } 280 281 if (ScalarBool != VecBool) { 282 EVT CondVT = Cond.getValueType(); 283 switch (ScalarBool) { 284 case TargetLowering::UndefinedBooleanContent: 285 break; 286 case TargetLowering::ZeroOrOneBooleanContent: 287 assert(VecBool == TargetLowering::UndefinedBooleanContent || 288 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 289 // Vector read from all ones, scalar expects a single 1 so mask. 290 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 291 Cond, DAG.getConstant(1, CondVT)); 292 break; 293 case TargetLowering::ZeroOrNegativeOneBooleanContent: 294 assert(VecBool == TargetLowering::UndefinedBooleanContent || 295 VecBool == TargetLowering::ZeroOrOneBooleanContent); 296 // Vector reads from a one, scalar from all ones so sign extend. 297 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 298 Cond, DAG.getValueType(MVT::i1)); 299 break; 300 } 301 } 302 303 return DAG.getSelect(SDLoc(N), 304 LHS.getValueType(), Cond, LHS, 305 GetScalarizedVector(N->getOperand(2))); 306} 307 308SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 309 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 310 return DAG.getSelect(SDLoc(N), 311 LHS.getValueType(), N->getOperand(0), LHS, 312 GetScalarizedVector(N->getOperand(2))); 313} 314 315SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 316 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 317 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 318 N->getOperand(0), N->getOperand(1), 319 LHS, GetScalarizedVector(N->getOperand(3)), 320 N->getOperand(4)); 321} 322 323SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 324 assert(N->getValueType(0).isVector() == 325 N->getOperand(0).getValueType().isVector() && 326 "Scalar/Vector type mismatch"); 327 328 if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 329 330 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 331 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 332 SDLoc DL(N); 333 334 // Turn it into a scalar SETCC. 335 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 336} 337 338SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 339 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 340} 341 342SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 343 // Figure out if the scalar is the LHS or RHS and return it. 344 SDValue Arg = N->getOperand(2).getOperand(0); 345 if (Arg.getOpcode() == ISD::UNDEF) 346 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 347 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 348 return GetScalarizedVector(N->getOperand(Op)); 349} 350 351SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 352 assert(N->getValueType(0).isVector() && 353 N->getOperand(0).getValueType().isVector() && 354 "Operand types must be vectors"); 355 SDValue LHS = N->getOperand(0); 356 SDValue RHS = N->getOperand(1); 357 EVT OpVT = LHS.getValueType(); 358 EVT NVT = N->getValueType(0).getVectorElementType(); 359 SDLoc DL(N); 360 361 // The result needs scalarizing, but it's not a given that the source does. 362 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 363 LHS = GetScalarizedVector(LHS); 364 RHS = GetScalarizedVector(RHS); 365 } else { 366 EVT VT = OpVT.getVectorElementType(); 367 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 368 DAG.getConstant(0, TLI.getVectorIdxTy())); 369 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 370 DAG.getConstant(0, TLI.getVectorIdxTy())); 371 } 372 373 // Turn it into a scalar SETCC. 374 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 375 N->getOperand(2)); 376 // Vectors may have a different boolean contents to scalars. Promote the 377 // value appropriately. 378 ISD::NodeType ExtendCode = 379 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 380 return DAG.getNode(ExtendCode, DL, NVT, Res); 381} 382 383 384//===----------------------------------------------------------------------===// 385// Operand Vector Scalarization <1 x ty> -> ty. 386//===----------------------------------------------------------------------===// 387 388bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 389 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 390 N->dump(&DAG); 391 dbgs() << "\n"); 392 SDValue Res = SDValue(); 393 394 if (!Res.getNode()) { 395 switch (N->getOpcode()) { 396 default: 397#ifndef NDEBUG 398 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 399 N->dump(&DAG); 400 dbgs() << "\n"; 401#endif 402 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 403 case ISD::BITCAST: 404 Res = ScalarizeVecOp_BITCAST(N); 405 break; 406 case ISD::ANY_EXTEND: 407 case ISD::ZERO_EXTEND: 408 case ISD::SIGN_EXTEND: 409 case ISD::TRUNCATE: 410 Res = ScalarizeVecOp_UnaryOp(N); 411 break; 412 case ISD::CONCAT_VECTORS: 413 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 414 break; 415 case ISD::EXTRACT_VECTOR_ELT: 416 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 417 break; 418 case ISD::VSELECT: 419 Res = ScalarizeVecOp_VSELECT(N); 420 break; 421 case ISD::STORE: 422 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 423 break; 424 case ISD::FP_ROUND: 425 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 426 break; 427 } 428 } 429 430 // If the result is null, the sub-method took care of registering results etc. 431 if (!Res.getNode()) return false; 432 433 // If the result is N, the sub-method updated N in place. Tell the legalizer 434 // core about this. 435 if (Res.getNode() == N) 436 return true; 437 438 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 439 "Invalid operand expansion"); 440 441 ReplaceValueWith(SDValue(N, 0), Res); 442 return false; 443} 444 445/// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs 446/// to be scalarized, it must be <1 x ty>. Convert the element instead. 447SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 448 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 449 return DAG.getNode(ISD::BITCAST, SDLoc(N), 450 N->getValueType(0), Elt); 451} 452 453/// ScalarizeVecOp_EXTEND - If the value to extend is a vector that needs 454/// to be scalarized, it must be <1 x ty>. Extend the element instead. 455SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 456 assert(N->getValueType(0).getVectorNumElements() == 1 && 457 "Unexected vector type!"); 458 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 459 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 460 N->getValueType(0).getScalarType(), Elt); 461 // Revectorize the result so the types line up with what the uses of this 462 // expression expect. 463 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Op); 464} 465 466/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 467/// use a BUILD_VECTOR instead. 468SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 469 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 470 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 471 Ops[i] = GetScalarizedVector(N->getOperand(i)); 472 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Ops); 473} 474 475/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 476/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 477/// index. 478SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 479 SDValue Res = GetScalarizedVector(N->getOperand(0)); 480 if (Res.getValueType() != N->getValueType(0)) 481 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), 482 Res); 483 return Res; 484} 485 486 487/// ScalarizeVecOp_VSELECT - If the input condition is a vector that needs to be 488/// scalarized, it must be <1 x i1>, so just convert to a normal ISD::SELECT 489/// (still with vector output type since that was acceptable if we got here). 490SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 491 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 492 EVT VT = N->getValueType(0); 493 494 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 495 N->getOperand(2)); 496} 497 498/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 499/// scalarized, it must be <1 x ty>. Just store the element. 500SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 501 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 502 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 503 SDLoc dl(N); 504 505 if (N->isTruncatingStore()) 506 return DAG.getTruncStore(N->getChain(), dl, 507 GetScalarizedVector(N->getOperand(1)), 508 N->getBasePtr(), N->getPointerInfo(), 509 N->getMemoryVT().getVectorElementType(), 510 N->isVolatile(), N->isNonTemporal(), 511 N->getAlignment(), N->getTBAAInfo()); 512 513 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 514 N->getBasePtr(), N->getPointerInfo(), 515 N->isVolatile(), N->isNonTemporal(), 516 N->getOriginalAlignment(), N->getTBAAInfo()); 517} 518 519/// ScalarizeVecOp_FP_ROUND - If the value to round is a vector that needs 520/// to be scalarized, it must be <1 x ty>. Convert the element instead. 521SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 522 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 523 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 524 N->getValueType(0).getVectorElementType(), Elt, 525 N->getOperand(1)); 526 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 527} 528 529//===----------------------------------------------------------------------===// 530// Result Vector Splitting 531//===----------------------------------------------------------------------===// 532 533/// SplitVectorResult - This method is called when the specified result of the 534/// specified node is found to need vector splitting. At this point, the node 535/// may also have invalid operands or may have other results that need 536/// legalization, we just know that (at least) one result needs vector 537/// splitting. 538void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 539 DEBUG(dbgs() << "Split node result: "; 540 N->dump(&DAG); 541 dbgs() << "\n"); 542 SDValue Lo, Hi; 543 544 // See if the target wants to custom expand this node. 545 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 546 return; 547 548 switch (N->getOpcode()) { 549 default: 550#ifndef NDEBUG 551 dbgs() << "SplitVectorResult #" << ResNo << ": "; 552 N->dump(&DAG); 553 dbgs() << "\n"; 554#endif 555 report_fatal_error("Do not know how to split the result of this " 556 "operator!\n"); 557 558 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 559 case ISD::VSELECT: 560 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 561 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 562 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 563 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 564 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 565 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 566 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 567 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 568 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 569 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 570 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 571 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 572 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 573 case ISD::LOAD: 574 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 575 break; 576 case ISD::SETCC: 577 SplitVecRes_SETCC(N, Lo, Hi); 578 break; 579 case ISD::VECTOR_SHUFFLE: 580 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 581 break; 582 583 case ISD::BSWAP: 584 case ISD::CONVERT_RNDSAT: 585 case ISD::CTLZ: 586 case ISD::CTTZ: 587 case ISD::CTLZ_ZERO_UNDEF: 588 case ISD::CTTZ_ZERO_UNDEF: 589 case ISD::CTPOP: 590 case ISD::FABS: 591 case ISD::FCEIL: 592 case ISD::FCOS: 593 case ISD::FEXP: 594 case ISD::FEXP2: 595 case ISD::FFLOOR: 596 case ISD::FLOG: 597 case ISD::FLOG10: 598 case ISD::FLOG2: 599 case ISD::FNEARBYINT: 600 case ISD::FNEG: 601 case ISD::FP_EXTEND: 602 case ISD::FP_ROUND: 603 case ISD::FP_TO_SINT: 604 case ISD::FP_TO_UINT: 605 case ISD::FRINT: 606 case ISD::FROUND: 607 case ISD::FSIN: 608 case ISD::FSQRT: 609 case ISD::FTRUNC: 610 case ISD::SINT_TO_FP: 611 case ISD::TRUNCATE: 612 case ISD::UINT_TO_FP: 613 SplitVecRes_UnaryOp(N, Lo, Hi); 614 break; 615 616 case ISD::ANY_EXTEND: 617 case ISD::SIGN_EXTEND: 618 case ISD::ZERO_EXTEND: 619 SplitVecRes_ExtendOp(N, Lo, Hi); 620 break; 621 622 case ISD::ADD: 623 case ISD::SUB: 624 case ISD::MUL: 625 case ISD::FADD: 626 case ISD::FCOPYSIGN: 627 case ISD::FSUB: 628 case ISD::FMUL: 629 case ISD::SDIV: 630 case ISD::UDIV: 631 case ISD::FDIV: 632 case ISD::FPOW: 633 case ISD::AND: 634 case ISD::OR: 635 case ISD::XOR: 636 case ISD::SHL: 637 case ISD::SRA: 638 case ISD::SRL: 639 case ISD::UREM: 640 case ISD::SREM: 641 case ISD::FREM: 642 SplitVecRes_BinOp(N, Lo, Hi); 643 break; 644 case ISD::FMA: 645 SplitVecRes_TernaryOp(N, Lo, Hi); 646 break; 647 } 648 649 // If Lo/Hi is null, the sub-method took care of registering results etc. 650 if (Lo.getNode()) 651 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 652} 653 654void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 655 SDValue &Hi) { 656 SDValue LHSLo, LHSHi; 657 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 658 SDValue RHSLo, RHSHi; 659 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 660 SDLoc dl(N); 661 662 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 663 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 664} 665 666void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 667 SDValue &Hi) { 668 SDValue Op0Lo, Op0Hi; 669 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 670 SDValue Op1Lo, Op1Hi; 671 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 672 SDValue Op2Lo, Op2Hi; 673 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 674 SDLoc dl(N); 675 676 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 677 Op0Lo, Op1Lo, Op2Lo); 678 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 679 Op0Hi, Op1Hi, Op2Hi); 680} 681 682void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 683 SDValue &Hi) { 684 // We know the result is a vector. The input may be either a vector or a 685 // scalar value. 686 EVT LoVT, HiVT; 687 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 688 SDLoc dl(N); 689 690 SDValue InOp = N->getOperand(0); 691 EVT InVT = InOp.getValueType(); 692 693 // Handle some special cases efficiently. 694 switch (getTypeAction(InVT)) { 695 case TargetLowering::TypeLegal: 696 case TargetLowering::TypePromoteInteger: 697 case TargetLowering::TypeSoftenFloat: 698 case TargetLowering::TypeScalarizeVector: 699 case TargetLowering::TypeWidenVector: 700 break; 701 case TargetLowering::TypeExpandInteger: 702 case TargetLowering::TypeExpandFloat: 703 // A scalar to vector conversion, where the scalar needs expansion. 704 // If the vector is being split in two then we can just convert the 705 // expanded pieces. 706 if (LoVT == HiVT) { 707 GetExpandedOp(InOp, Lo, Hi); 708 if (TLI.isBigEndian()) 709 std::swap(Lo, Hi); 710 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 711 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 712 return; 713 } 714 break; 715 case TargetLowering::TypeSplitVector: 716 // If the input is a vector that needs to be split, convert each split 717 // piece of the input now. 718 GetSplitVector(InOp, Lo, Hi); 719 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 720 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 721 return; 722 } 723 724 // In the general case, convert the input to an integer and split it by hand. 725 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 726 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 727 if (TLI.isBigEndian()) 728 std::swap(LoIntVT, HiIntVT); 729 730 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 731 732 if (TLI.isBigEndian()) 733 std::swap(Lo, Hi); 734 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 735 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 736} 737 738void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 739 SDValue &Hi) { 740 EVT LoVT, HiVT; 741 SDLoc dl(N); 742 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 743 unsigned LoNumElts = LoVT.getVectorNumElements(); 744 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 745 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, LoOps); 746 747 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 748 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, HiOps); 749} 750 751void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 752 SDValue &Hi) { 753 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 754 SDLoc dl(N); 755 unsigned NumSubvectors = N->getNumOperands() / 2; 756 if (NumSubvectors == 1) { 757 Lo = N->getOperand(0); 758 Hi = N->getOperand(1); 759 return; 760 } 761 762 EVT LoVT, HiVT; 763 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 764 765 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 766 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 767 768 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 769 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 770} 771 772void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 773 SDValue &Hi) { 774 SDValue Vec = N->getOperand(0); 775 SDValue Idx = N->getOperand(1); 776 SDLoc dl(N); 777 778 EVT LoVT, HiVT; 779 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 780 781 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 782 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 783 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 784 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), 785 TLI.getVectorIdxTy())); 786} 787 788void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 789 SDValue &Hi) { 790 SDValue Vec = N->getOperand(0); 791 SDValue SubVec = N->getOperand(1); 792 SDValue Idx = N->getOperand(2); 793 SDLoc dl(N); 794 GetSplitVector(Vec, Lo, Hi); 795 796 // Spill the vector to the stack. 797 EVT VecVT = Vec.getValueType(); 798 EVT SubVecVT = VecVT.getVectorElementType(); 799 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 800 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 801 MachinePointerInfo(), false, false, 0); 802 803 // Store the new subvector into the specified index. 804 SDValue SubVecPtr = GetVectorElementPointer(StackPtr, SubVecVT, Idx); 805 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 806 unsigned Alignment = TLI.getDataLayout()->getPrefTypeAlignment(VecType); 807 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo(), 808 false, false, 0); 809 810 // Load the Lo part from the stack slot. 811 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 812 false, false, false, 0); 813 814 // Increment the pointer to the other part. 815 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 816 StackPtr = 817 DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 818 DAG.getConstant(IncrementSize, StackPtr.getValueType())); 819 820 // Load the Hi part from the stack slot. 821 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 822 false, false, false, MinAlign(Alignment, IncrementSize)); 823} 824 825void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 826 SDValue &Hi) { 827 SDLoc dl(N); 828 GetSplitVector(N->getOperand(0), Lo, Hi); 829 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 830 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 831} 832 833void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 834 SDValue &Hi) { 835 SDValue LHSLo, LHSHi; 836 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 837 SDLoc dl(N); 838 839 EVT LoVT, HiVT; 840 std::tie(LoVT, HiVT) = 841 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 842 843 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 844 DAG.getValueType(LoVT)); 845 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 846 DAG.getValueType(HiVT)); 847} 848 849void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 850 SDValue &Hi) { 851 SDValue Vec = N->getOperand(0); 852 SDValue Elt = N->getOperand(1); 853 SDValue Idx = N->getOperand(2); 854 SDLoc dl(N); 855 GetSplitVector(Vec, Lo, Hi); 856 857 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 858 unsigned IdxVal = CIdx->getZExtValue(); 859 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 860 if (IdxVal < LoNumElts) 861 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 862 Lo.getValueType(), Lo, Elt, Idx); 863 else 864 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 865 DAG.getConstant(IdxVal - LoNumElts, 866 TLI.getVectorIdxTy())); 867 return; 868 } 869 870 // Spill the vector to the stack. 871 EVT VecVT = Vec.getValueType(); 872 EVT EltVT = VecVT.getVectorElementType(); 873 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 874 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 875 MachinePointerInfo(), false, false, 0); 876 877 // Store the new element. This may be larger than the vector element type, 878 // so use a truncating store. 879 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 880 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 881 unsigned Alignment = 882 TLI.getDataLayout()->getPrefTypeAlignment(VecType); 883 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 884 false, false, 0); 885 886 // Load the Lo part from the stack slot. 887 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 888 false, false, false, 0); 889 890 // Increment the pointer to the other part. 891 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 892 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 893 DAG.getConstant(IncrementSize, StackPtr.getValueType())); 894 895 // Load the Hi part from the stack slot. 896 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 897 false, false, false, MinAlign(Alignment, IncrementSize)); 898} 899 900void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 901 SDValue &Hi) { 902 EVT LoVT, HiVT; 903 SDLoc dl(N); 904 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 905 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 906 Hi = DAG.getUNDEF(HiVT); 907} 908 909void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 910 SDValue &Hi) { 911 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 912 EVT LoVT, HiVT; 913 SDLoc dl(LD); 914 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 915 916 ISD::LoadExtType ExtType = LD->getExtensionType(); 917 SDValue Ch = LD->getChain(); 918 SDValue Ptr = LD->getBasePtr(); 919 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 920 EVT MemoryVT = LD->getMemoryVT(); 921 unsigned Alignment = LD->getOriginalAlignment(); 922 bool isVolatile = LD->isVolatile(); 923 bool isNonTemporal = LD->isNonTemporal(); 924 bool isInvariant = LD->isInvariant(); 925 const MDNode *TBAAInfo = LD->getTBAAInfo(); 926 927 EVT LoMemVT, HiMemVT; 928 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 929 930 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 931 LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 932 isInvariant, Alignment, TBAAInfo); 933 934 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 935 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 936 DAG.getConstant(IncrementSize, Ptr.getValueType())); 937 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 938 LD->getPointerInfo().getWithOffset(IncrementSize), 939 HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment, 940 TBAAInfo); 941 942 // Build a factor node to remember that this load is independent of the 943 // other one. 944 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 945 Hi.getValue(1)); 946 947 // Legalized the chain result - switch anything that used the old chain to 948 // use the new one. 949 ReplaceValueWith(SDValue(LD, 1), Ch); 950} 951 952void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 953 assert(N->getValueType(0).isVector() && 954 N->getOperand(0).getValueType().isVector() && 955 "Operand types must be vectors"); 956 957 EVT LoVT, HiVT; 958 SDLoc DL(N); 959 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 960 961 // Split the input. 962 SDValue LL, LH, RL, RH; 963 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 964 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 965 966 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 967 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 968} 969 970void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 971 SDValue &Hi) { 972 // Get the dest types - they may not match the input types, e.g. int_to_fp. 973 EVT LoVT, HiVT; 974 SDLoc dl(N); 975 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 976 977 // If the input also splits, handle it directly for a compile time speedup. 978 // Otherwise split it by hand. 979 EVT InVT = N->getOperand(0).getValueType(); 980 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 981 GetSplitVector(N->getOperand(0), Lo, Hi); 982 else 983 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 984 985 if (N->getOpcode() == ISD::FP_ROUND) { 986 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 987 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 988 } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 989 SDValue DTyOpLo = DAG.getValueType(LoVT); 990 SDValue DTyOpHi = DAG.getValueType(HiVT); 991 SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 992 SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 993 SDValue RndOp = N->getOperand(3); 994 SDValue SatOp = N->getOperand(4); 995 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 996 Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 997 CvtCode); 998 Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 999 CvtCode); 1000 } else { 1001 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1002 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1003 } 1004} 1005 1006void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 1007 SDValue &Hi) { 1008 SDLoc dl(N); 1009 EVT SrcVT = N->getOperand(0).getValueType(); 1010 EVT DestVT = N->getValueType(0); 1011 EVT LoVT, HiVT; 1012 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 1013 1014 // We can do better than a generic split operation if the extend is doing 1015 // more than just doubling the width of the elements and the following are 1016 // true: 1017 // - The number of vector elements is even, 1018 // - the source type is legal, 1019 // - the type of a split source is illegal, 1020 // - the type of an extended (by doubling element size) source is legal, and 1021 // - the type of that extended source when split is legal. 1022 // 1023 // This won't necessarily completely legalize the operation, but it will 1024 // more effectively move in the right direction and prevent falling down 1025 // to scalarization in many cases due to the input vector being split too 1026 // far. 1027 unsigned NumElements = SrcVT.getVectorNumElements(); 1028 if ((NumElements & 1) == 0 && 1029 SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) { 1030 LLVMContext &Ctx = *DAG.getContext(); 1031 EVT NewSrcVT = EVT::getVectorVT( 1032 Ctx, EVT::getIntegerVT( 1033 Ctx, SrcVT.getVectorElementType().getSizeInBits() * 2), 1034 NumElements); 1035 EVT SplitSrcVT = 1036 EVT::getVectorVT(Ctx, SrcVT.getVectorElementType(), NumElements / 2); 1037 EVT SplitLoVT, SplitHiVT; 1038 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 1039 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 1040 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 1041 DEBUG(dbgs() << "Split vector extend via incremental extend:"; 1042 N->dump(&DAG); dbgs() << "\n"); 1043 // Extend the source vector by one step. 1044 SDValue NewSrc = 1045 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 1046 // Get the low and high halves of the new, extended one step, vector. 1047 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 1048 // Extend those vector halves the rest of the way. 1049 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1050 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1051 return; 1052 } 1053 } 1054 // Fall back to the generic unary operator splitting otherwise. 1055 SplitVecRes_UnaryOp(N, Lo, Hi); 1056} 1057 1058void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 1059 SDValue &Lo, SDValue &Hi) { 1060 // The low and high parts of the original input give four input vectors. 1061 SDValue Inputs[4]; 1062 SDLoc dl(N); 1063 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 1064 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 1065 EVT NewVT = Inputs[0].getValueType(); 1066 unsigned NewElts = NewVT.getVectorNumElements(); 1067 1068 // If Lo or Hi uses elements from at most two of the four input vectors, then 1069 // express it as a vector shuffle of those two inputs. Otherwise extract the 1070 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 1071 SmallVector<int, 16> Ops; 1072 for (unsigned High = 0; High < 2; ++High) { 1073 SDValue &Output = High ? Hi : Lo; 1074 1075 // Build a shuffle mask for the output, discovering on the fly which 1076 // input vectors to use as shuffle operands (recorded in InputUsed). 1077 // If building a suitable shuffle vector proves too hard, then bail 1078 // out with useBuildVector set. 1079 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 1080 unsigned FirstMaskIdx = High * NewElts; 1081 bool useBuildVector = false; 1082 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1083 // The mask element. This indexes into the input. 1084 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1085 1086 // The input vector this mask element indexes into. 1087 unsigned Input = (unsigned)Idx / NewElts; 1088 1089 if (Input >= array_lengthof(Inputs)) { 1090 // The mask element does not index into any input vector. 1091 Ops.push_back(-1); 1092 continue; 1093 } 1094 1095 // Turn the index into an offset from the start of the input vector. 1096 Idx -= Input * NewElts; 1097 1098 // Find or create a shuffle vector operand to hold this input. 1099 unsigned OpNo; 1100 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 1101 if (InputUsed[OpNo] == Input) { 1102 // This input vector is already an operand. 1103 break; 1104 } else if (InputUsed[OpNo] == -1U) { 1105 // Create a new operand for this input vector. 1106 InputUsed[OpNo] = Input; 1107 break; 1108 } 1109 } 1110 1111 if (OpNo >= array_lengthof(InputUsed)) { 1112 // More than two input vectors used! Give up on trying to create a 1113 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 1114 useBuildVector = true; 1115 break; 1116 } 1117 1118 // Add the mask index for the new shuffle vector. 1119 Ops.push_back(Idx + OpNo * NewElts); 1120 } 1121 1122 if (useBuildVector) { 1123 EVT EltVT = NewVT.getVectorElementType(); 1124 SmallVector<SDValue, 16> SVOps; 1125 1126 // Extract the input elements by hand. 1127 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1128 // The mask element. This indexes into the input. 1129 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1130 1131 // The input vector this mask element indexes into. 1132 unsigned Input = (unsigned)Idx / NewElts; 1133 1134 if (Input >= array_lengthof(Inputs)) { 1135 // The mask element is "undef" or indexes off the end of the input. 1136 SVOps.push_back(DAG.getUNDEF(EltVT)); 1137 continue; 1138 } 1139 1140 // Turn the index into an offset from the start of the input vector. 1141 Idx -= Input * NewElts; 1142 1143 // Extract the vector element by hand. 1144 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 1145 Inputs[Input], DAG.getConstant(Idx, 1146 TLI.getVectorIdxTy()))); 1147 } 1148 1149 // Construct the Lo/Hi output using a BUILD_VECTOR. 1150 Output = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, SVOps); 1151 } else if (InputUsed[0] == -1U) { 1152 // No input vectors were used! The result is undefined. 1153 Output = DAG.getUNDEF(NewVT); 1154 } else { 1155 SDValue Op0 = Inputs[InputUsed[0]]; 1156 // If only one input was used, use an undefined vector for the other. 1157 SDValue Op1 = InputUsed[1] == -1U ? 1158 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 1159 // At least one input vector was used. Create a new shuffle vector. 1160 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 1161 } 1162 1163 Ops.clear(); 1164 } 1165} 1166 1167 1168//===----------------------------------------------------------------------===// 1169// Operand Vector Splitting 1170//===----------------------------------------------------------------------===// 1171 1172/// SplitVectorOperand - This method is called when the specified operand of the 1173/// specified node is found to need vector splitting. At this point, all of the 1174/// result types of the node are known to be legal, but other operands of the 1175/// node may need legalization as well as the specified one. 1176bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1177 DEBUG(dbgs() << "Split node operand: "; 1178 N->dump(&DAG); 1179 dbgs() << "\n"); 1180 SDValue Res = SDValue(); 1181 1182 // See if the target wants to custom split this node. 1183 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1184 return false; 1185 1186 if (!Res.getNode()) { 1187 switch (N->getOpcode()) { 1188 default: 1189#ifndef NDEBUG 1190 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1191 N->dump(&DAG); 1192 dbgs() << "\n"; 1193#endif 1194 report_fatal_error("Do not know how to split this operator's " 1195 "operand!\n"); 1196 1197 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1198 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1199 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1200 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1201 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1202 case ISD::TRUNCATE: Res = SplitVecOp_TRUNCATE(N); break; 1203 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1204 case ISD::STORE: 1205 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1206 break; 1207 case ISD::VSELECT: 1208 Res = SplitVecOp_VSELECT(N, OpNo); 1209 break; 1210 case ISD::CTTZ: 1211 case ISD::CTLZ: 1212 case ISD::CTPOP: 1213 case ISD::FP_EXTEND: 1214 case ISD::FP_TO_SINT: 1215 case ISD::FP_TO_UINT: 1216 case ISD::SINT_TO_FP: 1217 case ISD::UINT_TO_FP: 1218 case ISD::FTRUNC: 1219 case ISD::SIGN_EXTEND: 1220 case ISD::ZERO_EXTEND: 1221 case ISD::ANY_EXTEND: 1222 Res = SplitVecOp_UnaryOp(N); 1223 break; 1224 } 1225 } 1226 1227 // If the result is null, the sub-method took care of registering results etc. 1228 if (!Res.getNode()) return false; 1229 1230 // If the result is N, the sub-method updated N in place. Tell the legalizer 1231 // core about this. 1232 if (Res.getNode() == N) 1233 return true; 1234 1235 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1236 "Invalid operand expansion"); 1237 1238 ReplaceValueWith(SDValue(N, 0), Res); 1239 return false; 1240} 1241 1242SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 1243 // The only possibility for an illegal operand is the mask, since result type 1244 // legalization would have handled this node already otherwise. 1245 assert(OpNo == 0 && "Illegal operand must be mask"); 1246 1247 SDValue Mask = N->getOperand(0); 1248 SDValue Src0 = N->getOperand(1); 1249 SDValue Src1 = N->getOperand(2); 1250 EVT Src0VT = Src0.getValueType(); 1251 SDLoc DL(N); 1252 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 1253 1254 SDValue Lo, Hi; 1255 GetSplitVector(N->getOperand(0), Lo, Hi); 1256 assert(Lo.getValueType() == Hi.getValueType() && 1257 "Lo and Hi have differing types"); 1258 1259 EVT LoOpVT, HiOpVT; 1260 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 1261 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 1262 1263 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 1264 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 1265 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 1266 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 1267 1268 SDValue LoSelect = 1269 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 1270 SDValue HiSelect = 1271 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 1272 1273 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 1274} 1275 1276SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1277 // The result has a legal vector type, but the input needs splitting. 1278 EVT ResVT = N->getValueType(0); 1279 SDValue Lo, Hi; 1280 SDLoc dl(N); 1281 GetSplitVector(N->getOperand(0), Lo, Hi); 1282 EVT InVT = Lo.getValueType(); 1283 1284 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1285 InVT.getVectorNumElements()); 1286 1287 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1288 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1289 1290 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1291} 1292 1293SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1294 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1295 // end up being split all the way down to individual components. Convert the 1296 // split pieces into integers and reassemble. 1297 SDValue Lo, Hi; 1298 GetSplitVector(N->getOperand(0), Lo, Hi); 1299 Lo = BitConvertToInteger(Lo); 1300 Hi = BitConvertToInteger(Hi); 1301 1302 if (TLI.isBigEndian()) 1303 std::swap(Lo, Hi); 1304 1305 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 1306 JoinIntegers(Lo, Hi)); 1307} 1308 1309SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1310 // We know that the extracted result type is legal. 1311 EVT SubVT = N->getValueType(0); 1312 SDValue Idx = N->getOperand(1); 1313 SDLoc dl(N); 1314 SDValue Lo, Hi; 1315 GetSplitVector(N->getOperand(0), Lo, Hi); 1316 1317 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1318 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1319 1320 if (IdxVal < LoElts) { 1321 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1322 "Extracted subvector crosses vector split!"); 1323 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1324 } else { 1325 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1326 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1327 } 1328} 1329 1330SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1331 SDValue Vec = N->getOperand(0); 1332 SDValue Idx = N->getOperand(1); 1333 EVT VecVT = Vec.getValueType(); 1334 1335 if (isa<ConstantSDNode>(Idx)) { 1336 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1337 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1338 1339 SDValue Lo, Hi; 1340 GetSplitVector(Vec, Lo, Hi); 1341 1342 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1343 1344 if (IdxVal < LoElts) 1345 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1346 return SDValue(DAG.UpdateNodeOperands(N, Hi, 1347 DAG.getConstant(IdxVal - LoElts, 1348 Idx.getValueType())), 0); 1349 } 1350 1351 // Store the vector to the stack. 1352 EVT EltVT = VecVT.getVectorElementType(); 1353 SDLoc dl(N); 1354 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1355 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1356 MachinePointerInfo(), false, false, 0); 1357 1358 // Load back the required element. 1359 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1360 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1361 MachinePointerInfo(), EltVT, false, false, 0); 1362} 1363 1364SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1365 assert(N->isUnindexed() && "Indexed store of vector?"); 1366 assert(OpNo == 1 && "Can only split the stored value"); 1367 SDLoc DL(N); 1368 1369 bool isTruncating = N->isTruncatingStore(); 1370 SDValue Ch = N->getChain(); 1371 SDValue Ptr = N->getBasePtr(); 1372 EVT MemoryVT = N->getMemoryVT(); 1373 unsigned Alignment = N->getOriginalAlignment(); 1374 bool isVol = N->isVolatile(); 1375 bool isNT = N->isNonTemporal(); 1376 const MDNode *TBAAInfo = N->getTBAAInfo(); 1377 SDValue Lo, Hi; 1378 GetSplitVector(N->getOperand(1), Lo, Hi); 1379 1380 EVT LoMemVT, HiMemVT; 1381 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1382 1383 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1384 1385 if (isTruncating) 1386 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1387 LoMemVT, isVol, isNT, Alignment, TBAAInfo); 1388 else 1389 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1390 isVol, isNT, Alignment, TBAAInfo); 1391 1392 // Increment the pointer to the other half. 1393 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1394 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1395 1396 if (isTruncating) 1397 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1398 N->getPointerInfo().getWithOffset(IncrementSize), 1399 HiMemVT, isVol, isNT, Alignment, TBAAInfo); 1400 else 1401 Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1402 N->getPointerInfo().getWithOffset(IncrementSize), 1403 isVol, isNT, Alignment, TBAAInfo); 1404 1405 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1406} 1407 1408SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1409 SDLoc DL(N); 1410 1411 // The input operands all must have the same type, and we know the result 1412 // type is valid. Convert this to a buildvector which extracts all the 1413 // input elements. 1414 // TODO: If the input elements are power-two vectors, we could convert this to 1415 // a new CONCAT_VECTORS node with elements that are half-wide. 1416 SmallVector<SDValue, 32> Elts; 1417 EVT EltVT = N->getValueType(0).getVectorElementType(); 1418 for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 1419 SDValue Op = N->getOperand(op); 1420 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 1421 i != e; ++i) { 1422 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 1423 Op, DAG.getConstant(i, TLI.getVectorIdxTy()))); 1424 1425 } 1426 } 1427 1428 return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts); 1429} 1430 1431SDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) { 1432 // The result type is legal, but the input type is illegal. If splitting 1433 // ends up with the result type of each half still being legal, just 1434 // do that. If, however, that would result in an illegal result type, 1435 // we can try to get more clever with power-two vectors. Specifically, 1436 // split the input type, but also widen the result element size, then 1437 // concatenate the halves and truncate again. For example, consider a target 1438 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 1439 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 1440 // %inlo = v4i32 extract_subvector %in, 0 1441 // %inhi = v4i32 extract_subvector %in, 4 1442 // %lo16 = v4i16 trunc v4i32 %inlo 1443 // %hi16 = v4i16 trunc v4i32 %inhi 1444 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 1445 // %res = v8i8 trunc v8i16 %in16 1446 // 1447 // Without this transform, the original truncate would end up being 1448 // scalarized, which is pretty much always a last resort. 1449 SDValue InVec = N->getOperand(0); 1450 EVT InVT = InVec->getValueType(0); 1451 EVT OutVT = N->getValueType(0); 1452 unsigned NumElements = OutVT.getVectorNumElements(); 1453 // Widening should have already made sure this is a power-two vector 1454 // if we're trying to split it at all. assert() that's true, just in case. 1455 assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 1456 1457 unsigned InElementSize = InVT.getVectorElementType().getSizeInBits(); 1458 unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits(); 1459 1460 // If the input elements are only 1/2 the width of the result elements, 1461 // just use the normal splitting. Our trick only work if there's room 1462 // to split more than once. 1463 if (InElementSize <= OutElementSize * 2) 1464 return SplitVecOp_UnaryOp(N); 1465 SDLoc DL(N); 1466 1467 // Extract the halves of the input via extract_subvector. 1468 SDValue InLoVec, InHiVec; 1469 std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL); 1470 // Truncate them to 1/2 the element size. 1471 EVT HalfElementVT = EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 1472 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 1473 NumElements/2); 1474 SDValue HalfLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InLoVec); 1475 SDValue HalfHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InHiVec); 1476 // Concatenate them to get the full intermediate truncation result. 1477 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 1478 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 1479 HalfHi); 1480 // Now finish up by truncating all the way down to the original result 1481 // type. This should normally be something that ends up being legal directly, 1482 // but in theory if a target has very wide vectors and an annoyingly 1483 // restricted set of legal types, this split can chain to build things up. 1484 return DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 1485} 1486 1487SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 1488 assert(N->getValueType(0).isVector() && 1489 N->getOperand(0).getValueType().isVector() && 1490 "Operand types must be vectors"); 1491 // The result has a legal vector type, but the input needs splitting. 1492 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1493 SDLoc DL(N); 1494 GetSplitVector(N->getOperand(0), Lo0, Hi0); 1495 GetSplitVector(N->getOperand(1), Lo1, Hi1); 1496 unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 1497 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 1498 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 1499 1500 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 1501 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 1502 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 1503 return PromoteTargetBoolean(Con, N->getValueType(0)); 1504} 1505 1506 1507SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 1508 // The result has a legal vector type, but the input needs splitting. 1509 EVT ResVT = N->getValueType(0); 1510 SDValue Lo, Hi; 1511 SDLoc DL(N); 1512 GetSplitVector(N->getOperand(0), Lo, Hi); 1513 EVT InVT = Lo.getValueType(); 1514 1515 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1516 InVT.getVectorNumElements()); 1517 1518 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 1519 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 1520 1521 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 1522} 1523 1524 1525 1526//===----------------------------------------------------------------------===// 1527// Result Vector Widening 1528//===----------------------------------------------------------------------===// 1529 1530void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1531 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1532 N->dump(&DAG); 1533 dbgs() << "\n"); 1534 1535 // See if the target wants to custom widen this node. 1536 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1537 return; 1538 1539 SDValue Res = SDValue(); 1540 switch (N->getOpcode()) { 1541 default: 1542#ifndef NDEBUG 1543 dbgs() << "WidenVectorResult #" << ResNo << ": "; 1544 N->dump(&DAG); 1545 dbgs() << "\n"; 1546#endif 1547 llvm_unreachable("Do not know how to widen the result of this operator!"); 1548 1549 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 1550 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 1551 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1552 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1553 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1554 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1555 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1556 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1557 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1558 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1559 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1560 case ISD::VSELECT: 1561 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1562 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1563 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 1564 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1565 case ISD::VECTOR_SHUFFLE: 1566 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1567 break; 1568 1569 case ISD::ADD: 1570 case ISD::AND: 1571 case ISD::MUL: 1572 case ISD::MULHS: 1573 case ISD::MULHU: 1574 case ISD::OR: 1575 case ISD::SUB: 1576 case ISD::XOR: 1577 Res = WidenVecRes_Binary(N); 1578 break; 1579 1580 case ISD::FADD: 1581 case ISD::FCOPYSIGN: 1582 case ISD::FMUL: 1583 case ISD::FPOW: 1584 case ISD::FSUB: 1585 case ISD::FDIV: 1586 case ISD::FREM: 1587 case ISD::SDIV: 1588 case ISD::UDIV: 1589 case ISD::SREM: 1590 case ISD::UREM: 1591 Res = WidenVecRes_BinaryCanTrap(N); 1592 break; 1593 1594 case ISD::FPOWI: 1595 Res = WidenVecRes_POWI(N); 1596 break; 1597 1598 case ISD::SHL: 1599 case ISD::SRA: 1600 case ISD::SRL: 1601 Res = WidenVecRes_Shift(N); 1602 break; 1603 1604 case ISD::ANY_EXTEND: 1605 case ISD::FP_EXTEND: 1606 case ISD::FP_ROUND: 1607 case ISD::FP_TO_SINT: 1608 case ISD::FP_TO_UINT: 1609 case ISD::SIGN_EXTEND: 1610 case ISD::SINT_TO_FP: 1611 case ISD::TRUNCATE: 1612 case ISD::UINT_TO_FP: 1613 case ISD::ZERO_EXTEND: 1614 Res = WidenVecRes_Convert(N); 1615 break; 1616 1617 case ISD::BSWAP: 1618 case ISD::CTLZ: 1619 case ISD::CTPOP: 1620 case ISD::CTTZ: 1621 case ISD::FABS: 1622 case ISD::FCEIL: 1623 case ISD::FCOS: 1624 case ISD::FEXP: 1625 case ISD::FEXP2: 1626 case ISD::FFLOOR: 1627 case ISD::FLOG: 1628 case ISD::FLOG10: 1629 case ISD::FLOG2: 1630 case ISD::FNEARBYINT: 1631 case ISD::FNEG: 1632 case ISD::FRINT: 1633 case ISD::FROUND: 1634 case ISD::FSIN: 1635 case ISD::FSQRT: 1636 case ISD::FTRUNC: 1637 Res = WidenVecRes_Unary(N); 1638 break; 1639 case ISD::FMA: 1640 Res = WidenVecRes_Ternary(N); 1641 break; 1642 } 1643 1644 // If Res is null, the sub-method took care of registering the result. 1645 if (Res.getNode()) 1646 SetWidenedVector(SDValue(N, ResNo), Res); 1647} 1648 1649SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 1650 // Ternary op widening. 1651 SDLoc dl(N); 1652 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1653 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1654 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1655 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 1656 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 1657} 1658 1659SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1660 // Binary op widening. 1661 SDLoc dl(N); 1662 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1663 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1664 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1665 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1666} 1667 1668SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 1669 // Binary op widening for operations that can trap. 1670 unsigned Opcode = N->getOpcode(); 1671 SDLoc dl(N); 1672 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1673 EVT WidenEltVT = WidenVT.getVectorElementType(); 1674 EVT VT = WidenVT; 1675 unsigned NumElts = VT.getVectorNumElements(); 1676 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 1677 NumElts = NumElts / 2; 1678 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1679 } 1680 1681 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 1682 // Operation doesn't trap so just widen as normal. 1683 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1684 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1685 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1686 } 1687 1688 // No legal vector version so unroll the vector operation and then widen. 1689 if (NumElts == 1) 1690 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 1691 1692 // Since the operation can trap, apply operation on the original vector. 1693 EVT MaxVT = VT; 1694 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1695 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1696 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 1697 1698 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 1699 unsigned ConcatEnd = 0; // Current ConcatOps index. 1700 int Idx = 0; // Current Idx into input vectors. 1701 1702 // NumElts := greatest legal vector size (at most WidenVT) 1703 // while (orig. vector has unhandled elements) { 1704 // take munches of size NumElts from the beginning and add to ConcatOps 1705 // NumElts := next smaller supported vector size or 1 1706 // } 1707 while (CurNumElts != 0) { 1708 while (CurNumElts >= NumElts) { 1709 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 1710 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1711 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 1712 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1713 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2); 1714 Idx += NumElts; 1715 CurNumElts -= NumElts; 1716 } 1717 do { 1718 NumElts = NumElts / 2; 1719 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1720 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 1721 1722 if (NumElts == 1) { 1723 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 1724 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1725 InOp1, DAG.getConstant(Idx, 1726 TLI.getVectorIdxTy())); 1727 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1728 InOp2, DAG.getConstant(Idx, 1729 TLI.getVectorIdxTy())); 1730 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 1731 EOp1, EOp2); 1732 } 1733 CurNumElts = 0; 1734 } 1735 } 1736 1737 // Check to see if we have a single operation with the widen type. 1738 if (ConcatEnd == 1) { 1739 VT = ConcatOps[0].getValueType(); 1740 if (VT == WidenVT) 1741 return ConcatOps[0]; 1742 } 1743 1744 // while (Some element of ConcatOps is not of type MaxVT) { 1745 // From the end of ConcatOps, collect elements of the same type and put 1746 // them into an op of the next larger supported type 1747 // } 1748 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 1749 Idx = ConcatEnd - 1; 1750 VT = ConcatOps[Idx--].getValueType(); 1751 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 1752 Idx--; 1753 1754 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 1755 EVT NextVT; 1756 do { 1757 NextSize *= 2; 1758 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 1759 } while (!TLI.isTypeLegal(NextVT)); 1760 1761 if (!VT.isVector()) { 1762 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 1763 SDValue VecOp = DAG.getUNDEF(NextVT); 1764 unsigned NumToInsert = ConcatEnd - Idx - 1; 1765 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 1766 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 1767 ConcatOps[OpIdx], DAG.getConstant(i, 1768 TLI.getVectorIdxTy())); 1769 } 1770 ConcatOps[Idx+1] = VecOp; 1771 ConcatEnd = Idx + 2; 1772 } else { 1773 // Vector type, create a CONCAT_VECTORS of type NextVT 1774 SDValue undefVec = DAG.getUNDEF(VT); 1775 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 1776 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 1777 unsigned RealVals = ConcatEnd - Idx - 1; 1778 unsigned SubConcatEnd = 0; 1779 unsigned SubConcatIdx = Idx + 1; 1780 while (SubConcatEnd < RealVals) 1781 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 1782 while (SubConcatEnd < OpsToConcat) 1783 SubConcatOps[SubConcatEnd++] = undefVec; 1784 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1785 NextVT, SubConcatOps); 1786 ConcatEnd = SubConcatIdx + 1; 1787 } 1788 } 1789 1790 // Check to see if we have a single operation with the widen type. 1791 if (ConcatEnd == 1) { 1792 VT = ConcatOps[0].getValueType(); 1793 if (VT == WidenVT) 1794 return ConcatOps[0]; 1795 } 1796 1797 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 1798 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 1799 if (NumOps != ConcatEnd ) { 1800 SDValue UndefVal = DAG.getUNDEF(MaxVT); 1801 for (unsigned j = ConcatEnd; j < NumOps; ++j) 1802 ConcatOps[j] = UndefVal; 1803 } 1804 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 1805 makeArrayRef(ConcatOps.data(), NumOps)); 1806} 1807 1808SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1809 SDValue InOp = N->getOperand(0); 1810 SDLoc DL(N); 1811 1812 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1813 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1814 1815 EVT InVT = InOp.getValueType(); 1816 EVT InEltVT = InVT.getVectorElementType(); 1817 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1818 1819 unsigned Opcode = N->getOpcode(); 1820 unsigned InVTNumElts = InVT.getVectorNumElements(); 1821 1822 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1823 InOp = GetWidenedVector(N->getOperand(0)); 1824 InVT = InOp.getValueType(); 1825 InVTNumElts = InVT.getVectorNumElements(); 1826 if (InVTNumElts == WidenNumElts) { 1827 if (N->getNumOperands() == 1) 1828 return DAG.getNode(Opcode, DL, WidenVT, InOp); 1829 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1)); 1830 } 1831 } 1832 1833 if (TLI.isTypeLegal(InWidenVT)) { 1834 // Because the result and the input are different vector types, widening 1835 // the result could create a legal type but widening the input might make 1836 // it an illegal type that might lead to repeatedly splitting the input 1837 // and then widening it. To avoid this, we widen the input only if 1838 // it results in a legal type. 1839 if (WidenNumElts % InVTNumElts == 0) { 1840 // Widen the input and call convert on the widened input vector. 1841 unsigned NumConcat = WidenNumElts/InVTNumElts; 1842 SmallVector<SDValue, 16> Ops(NumConcat); 1843 Ops[0] = InOp; 1844 SDValue UndefVal = DAG.getUNDEF(InVT); 1845 for (unsigned i = 1; i != NumConcat; ++i) 1846 Ops[i] = UndefVal; 1847 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 1848 if (N->getNumOperands() == 1) 1849 return DAG.getNode(Opcode, DL, WidenVT, InVec); 1850 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1)); 1851 } 1852 1853 if (InVTNumElts % WidenNumElts == 0) { 1854 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, 1855 InOp, DAG.getConstant(0, 1856 TLI.getVectorIdxTy())); 1857 // Extract the input and convert the shorten input vector. 1858 if (N->getNumOperands() == 1) 1859 return DAG.getNode(Opcode, DL, WidenVT, InVal); 1860 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1)); 1861 } 1862 } 1863 1864 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1865 SmallVector<SDValue, 16> Ops(WidenNumElts); 1866 EVT EltVT = WidenVT.getVectorElementType(); 1867 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1868 unsigned i; 1869 for (i=0; i < MinElts; ++i) { 1870 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 1871 DAG.getConstant(i, TLI.getVectorIdxTy())); 1872 if (N->getNumOperands() == 1) 1873 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 1874 else 1875 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1)); 1876 } 1877 1878 SDValue UndefVal = DAG.getUNDEF(EltVT); 1879 for (; i < WidenNumElts; ++i) 1880 Ops[i] = UndefVal; 1881 1882 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops); 1883} 1884 1885SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 1886 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1887 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1888 SDValue ShOp = N->getOperand(1); 1889 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 1890} 1891 1892SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1893 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1894 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1895 SDValue ShOp = N->getOperand(1); 1896 1897 EVT ShVT = ShOp.getValueType(); 1898 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 1899 ShOp = GetWidenedVector(ShOp); 1900 ShVT = ShOp.getValueType(); 1901 } 1902 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 1903 ShVT.getVectorElementType(), 1904 WidenVT.getVectorNumElements()); 1905 if (ShVT != ShWidenVT) 1906 ShOp = ModifyToType(ShOp, ShWidenVT); 1907 1908 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 1909} 1910 1911SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1912 // Unary op widening. 1913 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1914 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1915 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 1916} 1917 1918SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1919 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1920 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1921 cast<VTSDNode>(N->getOperand(1))->getVT() 1922 .getVectorElementType(), 1923 WidenVT.getVectorNumElements()); 1924 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1925 return DAG.getNode(N->getOpcode(), SDLoc(N), 1926 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1927} 1928 1929SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 1930 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 1931 return GetWidenedVector(WidenVec); 1932} 1933 1934SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 1935 SDValue InOp = N->getOperand(0); 1936 EVT InVT = InOp.getValueType(); 1937 EVT VT = N->getValueType(0); 1938 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1939 SDLoc dl(N); 1940 1941 switch (getTypeAction(InVT)) { 1942 case TargetLowering::TypeLegal: 1943 break; 1944 case TargetLowering::TypePromoteInteger: 1945 // If the incoming type is a vector that is being promoted, then 1946 // we know that the elements are arranged differently and that we 1947 // must perform the conversion using a stack slot. 1948 if (InVT.isVector()) 1949 break; 1950 1951 // If the InOp is promoted to the same size, convert it. Otherwise, 1952 // fall out of the switch and widen the promoted input. 1953 InOp = GetPromotedInteger(InOp); 1954 InVT = InOp.getValueType(); 1955 if (WidenVT.bitsEq(InVT)) 1956 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1957 break; 1958 case TargetLowering::TypeSoftenFloat: 1959 case TargetLowering::TypeExpandInteger: 1960 case TargetLowering::TypeExpandFloat: 1961 case TargetLowering::TypeScalarizeVector: 1962 case TargetLowering::TypeSplitVector: 1963 break; 1964 case TargetLowering::TypeWidenVector: 1965 // If the InOp is widened to the same size, convert it. Otherwise, fall 1966 // out of the switch and widen the widened input. 1967 InOp = GetWidenedVector(InOp); 1968 InVT = InOp.getValueType(); 1969 if (WidenVT.bitsEq(InVT)) 1970 // The input widens to the same size. Convert to the widen value. 1971 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1972 break; 1973 } 1974 1975 unsigned WidenSize = WidenVT.getSizeInBits(); 1976 unsigned InSize = InVT.getSizeInBits(); 1977 // x86mmx is not an acceptable vector element type, so don't try. 1978 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 1979 // Determine new input vector type. The new input vector type will use 1980 // the same element type (if its a vector) or use the input type as a 1981 // vector. It is the same size as the type to widen to. 1982 EVT NewInVT; 1983 unsigned NewNumElts = WidenSize / InSize; 1984 if (InVT.isVector()) { 1985 EVT InEltVT = InVT.getVectorElementType(); 1986 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 1987 WidenSize / InEltVT.getSizeInBits()); 1988 } else { 1989 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1990 } 1991 1992 if (TLI.isTypeLegal(NewInVT)) { 1993 // Because the result and the input are different vector types, widening 1994 // the result could create a legal type but widening the input might make 1995 // it an illegal type that might lead to repeatedly splitting the input 1996 // and then widening it. To avoid this, we widen the input only if 1997 // it results in a legal type. 1998 SmallVector<SDValue, 16> Ops(NewNumElts); 1999 SDValue UndefVal = DAG.getUNDEF(InVT); 2000 Ops[0] = InOp; 2001 for (unsigned i = 1; i < NewNumElts; ++i) 2002 Ops[i] = UndefVal; 2003 2004 SDValue NewVec; 2005 if (InVT.isVector()) 2006 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 2007 else 2008 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops); 2009 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 2010 } 2011 } 2012 2013 return CreateStackStoreLoad(InOp, WidenVT); 2014} 2015 2016SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 2017 SDLoc dl(N); 2018 // Build a vector with undefined for the new nodes. 2019 EVT VT = N->getValueType(0); 2020 2021 // Integer BUILD_VECTOR operands may be larger than the node's vector element 2022 // type. The UNDEFs need to have the same type as the existing operands. 2023 EVT EltVT = N->getOperand(0).getValueType(); 2024 unsigned NumElts = VT.getVectorNumElements(); 2025 2026 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2027 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2028 2029 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 2030 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 2031 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 2032 2033 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, NewOps); 2034} 2035 2036SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 2037 EVT InVT = N->getOperand(0).getValueType(); 2038 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2039 SDLoc dl(N); 2040 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2041 unsigned NumInElts = InVT.getVectorNumElements(); 2042 unsigned NumOperands = N->getNumOperands(); 2043 2044 bool InputWidened = false; // Indicates we need to widen the input. 2045 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 2046 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 2047 // Add undef vectors to widen to correct length. 2048 unsigned NumConcat = WidenVT.getVectorNumElements() / 2049 InVT.getVectorNumElements(); 2050 SDValue UndefVal = DAG.getUNDEF(InVT); 2051 SmallVector<SDValue, 16> Ops(NumConcat); 2052 for (unsigned i=0; i < NumOperands; ++i) 2053 Ops[i] = N->getOperand(i); 2054 for (unsigned i = NumOperands; i != NumConcat; ++i) 2055 Ops[i] = UndefVal; 2056 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 2057 } 2058 } else { 2059 InputWidened = true; 2060 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 2061 // The inputs and the result are widen to the same value. 2062 unsigned i; 2063 for (i=1; i < NumOperands; ++i) 2064 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 2065 break; 2066 2067 if (i == NumOperands) 2068 // Everything but the first operand is an UNDEF so just return the 2069 // widened first operand. 2070 return GetWidenedVector(N->getOperand(0)); 2071 2072 if (NumOperands == 2) { 2073 // Replace concat of two operands with a shuffle. 2074 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 2075 for (unsigned i = 0; i < NumInElts; ++i) { 2076 MaskOps[i] = i; 2077 MaskOps[i + NumInElts] = i + WidenNumElts; 2078 } 2079 return DAG.getVectorShuffle(WidenVT, dl, 2080 GetWidenedVector(N->getOperand(0)), 2081 GetWidenedVector(N->getOperand(1)), 2082 &MaskOps[0]); 2083 } 2084 } 2085 } 2086 2087 // Fall back to use extracts and build vector. 2088 EVT EltVT = WidenVT.getVectorElementType(); 2089 SmallVector<SDValue, 16> Ops(WidenNumElts); 2090 unsigned Idx = 0; 2091 for (unsigned i=0; i < NumOperands; ++i) { 2092 SDValue InOp = N->getOperand(i); 2093 if (InputWidened) 2094 InOp = GetWidenedVector(InOp); 2095 for (unsigned j=0; j < NumInElts; ++j) 2096 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2097 DAG.getConstant(j, TLI.getVectorIdxTy())); 2098 } 2099 SDValue UndefVal = DAG.getUNDEF(EltVT); 2100 for (; Idx < WidenNumElts; ++Idx) 2101 Ops[Idx] = UndefVal; 2102 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2103} 2104 2105SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 2106 SDLoc dl(N); 2107 SDValue InOp = N->getOperand(0); 2108 SDValue RndOp = N->getOperand(3); 2109 SDValue SatOp = N->getOperand(4); 2110 2111 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2112 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2113 2114 EVT InVT = InOp.getValueType(); 2115 EVT InEltVT = InVT.getVectorElementType(); 2116 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 2117 2118 SDValue DTyOp = DAG.getValueType(WidenVT); 2119 SDValue STyOp = DAG.getValueType(InWidenVT); 2120 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 2121 2122 unsigned InVTNumElts = InVT.getVectorNumElements(); 2123 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 2124 InOp = GetWidenedVector(InOp); 2125 InVT = InOp.getValueType(); 2126 InVTNumElts = InVT.getVectorNumElements(); 2127 if (InVTNumElts == WidenNumElts) 2128 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2129 SatOp, CvtCode); 2130 } 2131 2132 if (TLI.isTypeLegal(InWidenVT)) { 2133 // Because the result and the input are different vector types, widening 2134 // the result could create a legal type but widening the input might make 2135 // it an illegal type that might lead to repeatedly splitting the input 2136 // and then widening it. To avoid this, we widen the input only if 2137 // it results in a legal type. 2138 if (WidenNumElts % InVTNumElts == 0) { 2139 // Widen the input and call convert on the widened input vector. 2140 unsigned NumConcat = WidenNumElts/InVTNumElts; 2141 SmallVector<SDValue, 16> Ops(NumConcat); 2142 Ops[0] = InOp; 2143 SDValue UndefVal = DAG.getUNDEF(InVT); 2144 for (unsigned i = 1; i != NumConcat; ++i) 2145 Ops[i] = UndefVal; 2146 2147 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, Ops); 2148 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2149 SatOp, CvtCode); 2150 } 2151 2152 if (InVTNumElts % WidenNumElts == 0) { 2153 // Extract the input and convert the shorten input vector. 2154 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 2155 DAG.getConstant(0, TLI.getVectorIdxTy())); 2156 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2157 SatOp, CvtCode); 2158 } 2159 } 2160 2161 // Otherwise unroll into some nasty scalar code and rebuild the vector. 2162 SmallVector<SDValue, 16> Ops(WidenNumElts); 2163 EVT EltVT = WidenVT.getVectorElementType(); 2164 DTyOp = DAG.getValueType(EltVT); 2165 STyOp = DAG.getValueType(InEltVT); 2166 2167 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 2168 unsigned i; 2169 for (i=0; i < MinElts; ++i) { 2170 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2171 DAG.getConstant(i, TLI.getVectorIdxTy())); 2172 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 2173 SatOp, CvtCode); 2174 } 2175 2176 SDValue UndefVal = DAG.getUNDEF(EltVT); 2177 for (; i < WidenNumElts; ++i) 2178 Ops[i] = UndefVal; 2179 2180 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2181} 2182 2183SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 2184 EVT VT = N->getValueType(0); 2185 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2186 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2187 SDValue InOp = N->getOperand(0); 2188 SDValue Idx = N->getOperand(1); 2189 SDLoc dl(N); 2190 2191 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2192 InOp = GetWidenedVector(InOp); 2193 2194 EVT InVT = InOp.getValueType(); 2195 2196 // Check if we can just return the input vector after widening. 2197 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2198 if (IdxVal == 0 && InVT == WidenVT) 2199 return InOp; 2200 2201 // Check if we can extract from the vector. 2202 unsigned InNumElts = InVT.getVectorNumElements(); 2203 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 2204 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 2205 2206 // We could try widening the input to the right length but for now, extract 2207 // the original elements, fill the rest with undefs and build a vector. 2208 SmallVector<SDValue, 16> Ops(WidenNumElts); 2209 EVT EltVT = VT.getVectorElementType(); 2210 unsigned NumElts = VT.getVectorNumElements(); 2211 unsigned i; 2212 for (i=0; i < NumElts; ++i) 2213 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2214 DAG.getConstant(IdxVal+i, TLI.getVectorIdxTy())); 2215 2216 SDValue UndefVal = DAG.getUNDEF(EltVT); 2217 for (; i < WidenNumElts; ++i) 2218 Ops[i] = UndefVal; 2219 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2220} 2221 2222SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 2223 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2224 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 2225 InOp.getValueType(), InOp, 2226 N->getOperand(1), N->getOperand(2)); 2227} 2228 2229SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 2230 LoadSDNode *LD = cast<LoadSDNode>(N); 2231 ISD::LoadExtType ExtType = LD->getExtensionType(); 2232 2233 SDValue Result; 2234 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 2235 if (ExtType != ISD::NON_EXTLOAD) 2236 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 2237 else 2238 Result = GenWidenVectorLoads(LdChain, LD); 2239 2240 // If we generate a single load, we can use that for the chain. Otherwise, 2241 // build a factor node to remember the multiple loads are independent and 2242 // chain to that. 2243 SDValue NewChain; 2244 if (LdChain.size() == 1) 2245 NewChain = LdChain[0]; 2246 else 2247 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 2248 2249 // Modified the chain - switch anything that used the old chain to use 2250 // the new one. 2251 ReplaceValueWith(SDValue(N, 1), NewChain); 2252 2253 return Result; 2254} 2255 2256SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 2257 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2258 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), 2259 WidenVT, N->getOperand(0)); 2260} 2261 2262SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 2263 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2264 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2265 2266 SDValue Cond1 = N->getOperand(0); 2267 EVT CondVT = Cond1.getValueType(); 2268 if (CondVT.isVector()) { 2269 EVT CondEltVT = CondVT.getVectorElementType(); 2270 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 2271 CondEltVT, WidenNumElts); 2272 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 2273 Cond1 = GetWidenedVector(Cond1); 2274 2275 // If we have to split the condition there is no point in widening the 2276 // select. This would result in an cycle of widening the select -> 2277 // widening the condition operand -> splitting the condition operand -> 2278 // splitting the select -> widening the select. Instead split this select 2279 // further and widen the resulting type. 2280 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 2281 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 2282 SDValue Res = ModifyToType(SplitSelect, WidenVT); 2283 return Res; 2284 } 2285 2286 if (Cond1.getValueType() != CondWidenVT) 2287 Cond1 = ModifyToType(Cond1, CondWidenVT); 2288 } 2289 2290 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2291 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 2292 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2293 return DAG.getNode(N->getOpcode(), SDLoc(N), 2294 WidenVT, Cond1, InOp1, InOp2); 2295} 2296 2297SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 2298 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2299 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2300 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2301 InOp1.getValueType(), N->getOperand(0), 2302 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 2303} 2304 2305SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 2306 assert(N->getValueType(0).isVector() == 2307 N->getOperand(0).getValueType().isVector() && 2308 "Scalar/Vector type mismatch"); 2309 if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 2310 2311 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2312 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2313 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2314 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, 2315 InOp1, InOp2, N->getOperand(2)); 2316} 2317 2318SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 2319 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2320 return DAG.getUNDEF(WidenVT); 2321} 2322 2323SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2324 EVT VT = N->getValueType(0); 2325 SDLoc dl(N); 2326 2327 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2328 unsigned NumElts = VT.getVectorNumElements(); 2329 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2330 2331 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2332 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2333 2334 // Adjust mask based on new input vector length. 2335 SmallVector<int, 16> NewMask; 2336 for (unsigned i = 0; i != NumElts; ++i) { 2337 int Idx = N->getMaskElt(i); 2338 if (Idx < (int)NumElts) 2339 NewMask.push_back(Idx); 2340 else 2341 NewMask.push_back(Idx - NumElts + WidenNumElts); 2342 } 2343 for (unsigned i = NumElts; i != WidenNumElts; ++i) 2344 NewMask.push_back(-1); 2345 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 2346} 2347 2348SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 2349 assert(N->getValueType(0).isVector() && 2350 N->getOperand(0).getValueType().isVector() && 2351 "Operands must be vectors"); 2352 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2353 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2354 2355 SDValue InOp1 = N->getOperand(0); 2356 EVT InVT = InOp1.getValueType(); 2357 assert(InVT.isVector() && "can not widen non-vector type"); 2358 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 2359 InVT.getVectorElementType(), WidenNumElts); 2360 InOp1 = GetWidenedVector(InOp1); 2361 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2362 2363 // Assume that the input and output will be widen appropriately. If not, 2364 // we will have to unroll it at some point. 2365 assert(InOp1.getValueType() == WidenInVT && 2366 InOp2.getValueType() == WidenInVT && 2367 "Input not widened to expected type!"); 2368 (void)WidenInVT; 2369 return DAG.getNode(ISD::SETCC, SDLoc(N), 2370 WidenVT, InOp1, InOp2, N->getOperand(2)); 2371} 2372 2373 2374//===----------------------------------------------------------------------===// 2375// Widen Vector Operand 2376//===----------------------------------------------------------------------===// 2377bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 2378 DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; 2379 N->dump(&DAG); 2380 dbgs() << "\n"); 2381 SDValue Res = SDValue(); 2382 2383 // See if the target wants to custom widen this node. 2384 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2385 return false; 2386 2387 switch (N->getOpcode()) { 2388 default: 2389#ifndef NDEBUG 2390 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 2391 N->dump(&DAG); 2392 dbgs() << "\n"; 2393#endif 2394 llvm_unreachable("Do not know how to widen this operator's operand!"); 2395 2396 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 2397 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 2398 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 2399 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 2400 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 2401 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 2402 2403 case ISD::ANY_EXTEND: 2404 case ISD::SIGN_EXTEND: 2405 case ISD::ZERO_EXTEND: 2406 Res = WidenVecOp_EXTEND(N); 2407 break; 2408 2409 case ISD::FP_EXTEND: 2410 case ISD::FP_TO_SINT: 2411 case ISD::FP_TO_UINT: 2412 case ISD::SINT_TO_FP: 2413 case ISD::UINT_TO_FP: 2414 case ISD::TRUNCATE: 2415 Res = WidenVecOp_Convert(N); 2416 break; 2417 } 2418 2419 // If Res is null, the sub-method took care of registering the result. 2420 if (!Res.getNode()) return false; 2421 2422 // If the result is N, the sub-method updated N in place. Tell the legalizer 2423 // core about this. 2424 if (Res.getNode() == N) 2425 return true; 2426 2427 2428 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2429 "Invalid operand expansion"); 2430 2431 ReplaceValueWith(SDValue(N, 0), Res); 2432 return false; 2433} 2434 2435SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 2436 SDLoc DL(N); 2437 EVT VT = N->getValueType(0); 2438 2439 SDValue InOp = N->getOperand(0); 2440 // If some legalization strategy other than widening is used on the operand, 2441 // we can't safely assume that just extending the low lanes is the correct 2442 // transformation. 2443 if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector) 2444 return WidenVecOp_Convert(N); 2445 InOp = GetWidenedVector(InOp); 2446 assert(VT.getVectorNumElements() < 2447 InOp.getValueType().getVectorNumElements() && 2448 "Input wasn't widened!"); 2449 2450 // We may need to further widen the operand until it has the same total 2451 // vector size as the result. 2452 EVT InVT = InOp.getValueType(); 2453 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 2454 EVT InEltVT = InVT.getVectorElementType(); 2455 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) { 2456 EVT FixedVT = (MVT::SimpleValueType)i; 2457 EVT FixedEltVT = FixedVT.getVectorElementType(); 2458 if (TLI.isTypeLegal(FixedVT) && 2459 FixedVT.getSizeInBits() == VT.getSizeInBits() && 2460 FixedEltVT == InEltVT) { 2461 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 2462 "Not enough elements in the fixed type for the operand!"); 2463 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 2464 "We can't have the same type as we started with!"); 2465 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 2466 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT, 2467 DAG.getUNDEF(FixedVT), InOp, 2468 DAG.getConstant(0, TLI.getVectorIdxTy())); 2469 else 2470 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 2471 DAG.getConstant(0, TLI.getVectorIdxTy())); 2472 break; 2473 } 2474 } 2475 InVT = InOp.getValueType(); 2476 if (InVT.getSizeInBits() != VT.getSizeInBits()) 2477 // We couldn't find a legal vector type that was a widening of the input 2478 // and could be extended in-register to the result type, so we have to 2479 // scalarize. 2480 return WidenVecOp_Convert(N); 2481 } 2482 2483 // Use special DAG nodes to represent the operation of extending the 2484 // low lanes. 2485 switch (N->getOpcode()) { 2486 default: 2487 llvm_unreachable("Extend legalization on on extend operation!"); 2488 case ISD::ANY_EXTEND: 2489 return DAG.getAnyExtendVectorInReg(InOp, DL, VT); 2490 case ISD::SIGN_EXTEND: 2491 return DAG.getSignExtendVectorInReg(InOp, DL, VT); 2492 case ISD::ZERO_EXTEND: 2493 return DAG.getZeroExtendVectorInReg(InOp, DL, VT); 2494 } 2495} 2496 2497SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 2498 // Since the result is legal and the input is illegal, it is unlikely 2499 // that we can fix the input to a legal type so unroll the convert 2500 // into some scalar code and create a nasty build vector. 2501 EVT VT = N->getValueType(0); 2502 EVT EltVT = VT.getVectorElementType(); 2503 SDLoc dl(N); 2504 unsigned NumElts = VT.getVectorNumElements(); 2505 SDValue InOp = N->getOperand(0); 2506 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2507 InOp = GetWidenedVector(InOp); 2508 EVT InVT = InOp.getValueType(); 2509 EVT InEltVT = InVT.getVectorElementType(); 2510 2511 unsigned Opcode = N->getOpcode(); 2512 SmallVector<SDValue, 16> Ops(NumElts); 2513 for (unsigned i=0; i < NumElts; ++i) 2514 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 2515 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2516 DAG.getConstant(i, TLI.getVectorIdxTy()))); 2517 2518 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 2519} 2520 2521SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 2522 EVT VT = N->getValueType(0); 2523 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2524 EVT InWidenVT = InOp.getValueType(); 2525 SDLoc dl(N); 2526 2527 // Check if we can convert between two legal vector types and extract. 2528 unsigned InWidenSize = InWidenVT.getSizeInBits(); 2529 unsigned Size = VT.getSizeInBits(); 2530 // x86mmx is not an acceptable vector element type, so don't try. 2531 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 2532 unsigned NewNumElts = InWidenSize / Size; 2533 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 2534 if (TLI.isTypeLegal(NewVT)) { 2535 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 2536 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 2537 DAG.getConstant(0, TLI.getVectorIdxTy())); 2538 } 2539 } 2540 2541 return CreateStackStoreLoad(InOp, VT); 2542} 2543 2544SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 2545 // If the input vector is not legal, it is likely that we will not find a 2546 // legal vector of the same size. Replace the concatenate vector with a 2547 // nasty build vector. 2548 EVT VT = N->getValueType(0); 2549 EVT EltVT = VT.getVectorElementType(); 2550 SDLoc dl(N); 2551 unsigned NumElts = VT.getVectorNumElements(); 2552 SmallVector<SDValue, 16> Ops(NumElts); 2553 2554 EVT InVT = N->getOperand(0).getValueType(); 2555 unsigned NumInElts = InVT.getVectorNumElements(); 2556 2557 unsigned Idx = 0; 2558 unsigned NumOperands = N->getNumOperands(); 2559 for (unsigned i=0; i < NumOperands; ++i) { 2560 SDValue InOp = N->getOperand(i); 2561 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2562 InOp = GetWidenedVector(InOp); 2563 for (unsigned j=0; j < NumInElts; ++j) 2564 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2565 DAG.getConstant(j, TLI.getVectorIdxTy())); 2566 } 2567 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 2568} 2569 2570SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2571 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2572 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 2573 N->getValueType(0), InOp, N->getOperand(1)); 2574} 2575 2576SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2577 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2578 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2579 N->getValueType(0), InOp, N->getOperand(1)); 2580} 2581 2582SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 2583 // We have to widen the value but we want only to store the original 2584 // vector type. 2585 StoreSDNode *ST = cast<StoreSDNode>(N); 2586 2587 SmallVector<SDValue, 16> StChain; 2588 if (ST->isTruncatingStore()) 2589 GenWidenVectorTruncStores(StChain, ST); 2590 else 2591 GenWidenVectorStores(StChain, ST); 2592 2593 if (StChain.size() == 1) 2594 return StChain[0]; 2595 else 2596 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 2597} 2598 2599SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 2600 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 2601 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2602 SDLoc dl(N); 2603 2604 // WARNING: In this code we widen the compare instruction with garbage. 2605 // This garbage may contain denormal floats which may be slow. Is this a real 2606 // concern ? Should we zero the unused lanes if this is a float compare ? 2607 2608 // Get a new SETCC node to compare the newly widened operands. 2609 // Only some of the compared elements are legal. 2610 EVT SVT = TLI.getSetCCResultType(*DAG.getContext(), InOp0.getValueType()); 2611 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 2612 SVT, InOp0, InOp1, N->getOperand(2)); 2613 2614 // Extract the needed results from the result vector. 2615 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 2616 SVT.getVectorElementType(), 2617 N->getValueType(0).getVectorNumElements()); 2618 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, 2619 ResVT, WideSETCC, DAG.getConstant(0, 2620 TLI.getVectorIdxTy())); 2621 2622 return PromoteTargetBoolean(CC, N->getValueType(0)); 2623} 2624 2625 2626//===----------------------------------------------------------------------===// 2627// Vector Widening Utilities 2628//===----------------------------------------------------------------------===// 2629 2630// Utility function to find the type to chop up a widen vector for load/store 2631// TLI: Target lowering used to determine legal types. 2632// Width: Width left need to load/store. 2633// WidenVT: The widen vector type to load to/store from 2634// Align: If 0, don't allow use of a wider type 2635// WidenEx: If Align is not 0, the amount additional we can load/store from. 2636 2637static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 2638 unsigned Width, EVT WidenVT, 2639 unsigned Align = 0, unsigned WidenEx = 0) { 2640 EVT WidenEltVT = WidenVT.getVectorElementType(); 2641 unsigned WidenWidth = WidenVT.getSizeInBits(); 2642 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 2643 unsigned AlignInBits = Align*8; 2644 2645 // If we have one element to load/store, return it. 2646 EVT RetVT = WidenEltVT; 2647 if (Width == WidenEltWidth) 2648 return RetVT; 2649 2650 // See if there is larger legal integer than the element type to load/store 2651 unsigned VT; 2652 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 2653 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 2654 EVT MemVT((MVT::SimpleValueType) VT); 2655 unsigned MemVTWidth = MemVT.getSizeInBits(); 2656 if (MemVT.getSizeInBits() <= WidenEltWidth) 2657 break; 2658 if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 2659 isPowerOf2_32(WidenWidth / MemVTWidth) && 2660 (MemVTWidth <= Width || 2661 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2662 RetVT = MemVT; 2663 break; 2664 } 2665 } 2666 2667 // See if there is a larger vector type to load/store that has the same vector 2668 // element type and is evenly divisible with the WidenVT. 2669 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 2670 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 2671 EVT MemVT = (MVT::SimpleValueType) VT; 2672 unsigned MemVTWidth = MemVT.getSizeInBits(); 2673 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 2674 (WidenWidth % MemVTWidth) == 0 && 2675 isPowerOf2_32(WidenWidth / MemVTWidth) && 2676 (MemVTWidth <= Width || 2677 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2678 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 2679 return MemVT; 2680 } 2681 } 2682 2683 return RetVT; 2684} 2685 2686// Builds a vector type from scalar loads 2687// VecTy: Resulting Vector type 2688// LDOps: Load operators to build a vector type 2689// [Start,End) the list of loads to use. 2690static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 2691 SmallVectorImpl<SDValue> &LdOps, 2692 unsigned Start, unsigned End) { 2693 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2694 SDLoc dl(LdOps[Start]); 2695 EVT LdTy = LdOps[Start].getValueType(); 2696 unsigned Width = VecTy.getSizeInBits(); 2697 unsigned NumElts = Width / LdTy.getSizeInBits(); 2698 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 2699 2700 unsigned Idx = 1; 2701 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2702 2703 for (unsigned i = Start + 1; i != End; ++i) { 2704 EVT NewLdTy = LdOps[i].getValueType(); 2705 if (NewLdTy != LdTy) { 2706 NumElts = Width / NewLdTy.getSizeInBits(); 2707 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2708 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 2709 // Readjust position and vector position based on new load type 2710 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2711 LdTy = NewLdTy; 2712 } 2713 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2714 DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 2715 } 2716 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 2717} 2718 2719SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 2720 LoadSDNode *LD) { 2721 // The strategy assumes that we can efficiently load powers of two widths. 2722 // The routines chops the vector into the largest vector loads with the same 2723 // element type or scalar loads and then recombines it to the widen vector 2724 // type. 2725 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2726 unsigned WidenWidth = WidenVT.getSizeInBits(); 2727 EVT LdVT = LD->getMemoryVT(); 2728 SDLoc dl(LD); 2729 assert(LdVT.isVector() && WidenVT.isVector()); 2730 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2731 2732 // Load information 2733 SDValue Chain = LD->getChain(); 2734 SDValue BasePtr = LD->getBasePtr(); 2735 unsigned Align = LD->getAlignment(); 2736 bool isVolatile = LD->isVolatile(); 2737 bool isNonTemporal = LD->isNonTemporal(); 2738 bool isInvariant = LD->isInvariant(); 2739 const MDNode *TBAAInfo = LD->getTBAAInfo(); 2740 2741 int LdWidth = LdVT.getSizeInBits(); 2742 int WidthDiff = WidenWidth - LdWidth; // Difference 2743 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 2744 2745 // Find the vector type that can load from. 2746 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2747 int NewVTWidth = NewVT.getSizeInBits(); 2748 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 2749 isVolatile, isNonTemporal, isInvariant, Align, 2750 TBAAInfo); 2751 LdChain.push_back(LdOp.getValue(1)); 2752 2753 // Check if we can load the element with one instruction 2754 if (LdWidth <= NewVTWidth) { 2755 if (!NewVT.isVector()) { 2756 unsigned NumElts = WidenWidth / NewVTWidth; 2757 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2758 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2759 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 2760 } 2761 if (NewVT == WidenVT) 2762 return LdOp; 2763 2764 assert(WidenWidth % NewVTWidth == 0); 2765 unsigned NumConcat = WidenWidth / NewVTWidth; 2766 SmallVector<SDValue, 16> ConcatOps(NumConcat); 2767 SDValue UndefVal = DAG.getUNDEF(NewVT); 2768 ConcatOps[0] = LdOp; 2769 for (unsigned i = 1; i != NumConcat; ++i) 2770 ConcatOps[i] = UndefVal; 2771 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 2772 } 2773 2774 // Load vector by using multiple loads from largest vector to scalar 2775 SmallVector<SDValue, 16> LdOps; 2776 LdOps.push_back(LdOp); 2777 2778 LdWidth -= NewVTWidth; 2779 unsigned Offset = 0; 2780 2781 while (LdWidth > 0) { 2782 unsigned Increment = NewVTWidth / 8; 2783 Offset += Increment; 2784 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2785 DAG.getConstant(Increment, BasePtr.getValueType())); 2786 2787 SDValue L; 2788 if (LdWidth < NewVTWidth) { 2789 // Our current type we are using is too large, find a better size 2790 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2791 NewVTWidth = NewVT.getSizeInBits(); 2792 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2793 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2794 isNonTemporal, isInvariant, MinAlign(Align, Increment), 2795 TBAAInfo); 2796 LdChain.push_back(L.getValue(1)); 2797 if (L->getValueType(0).isVector()) { 2798 SmallVector<SDValue, 16> Loads; 2799 Loads.push_back(L); 2800 unsigned size = L->getValueSizeInBits(0); 2801 while (size < LdOp->getValueSizeInBits(0)) { 2802 Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 2803 size += L->getValueSizeInBits(0); 2804 } 2805 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads); 2806 } 2807 } else { 2808 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2809 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2810 isNonTemporal, isInvariant, MinAlign(Align, Increment), 2811 TBAAInfo); 2812 LdChain.push_back(L.getValue(1)); 2813 } 2814 2815 LdOps.push_back(L); 2816 2817 2818 LdWidth -= NewVTWidth; 2819 } 2820 2821 // Build the vector from the loads operations 2822 unsigned End = LdOps.size(); 2823 if (!LdOps[0].getValueType().isVector()) 2824 // All the loads are scalar loads. 2825 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 2826 2827 // If the load contains vectors, build the vector using concat vector. 2828 // All of the vectors used to loads are power of 2 and the scalars load 2829 // can be combined to make a power of 2 vector. 2830 SmallVector<SDValue, 16> ConcatOps(End); 2831 int i = End - 1; 2832 int Idx = End; 2833 EVT LdTy = LdOps[i].getValueType(); 2834 // First combine the scalar loads to a vector 2835 if (!LdTy.isVector()) { 2836 for (--i; i >= 0; --i) { 2837 LdTy = LdOps[i].getValueType(); 2838 if (LdTy.isVector()) 2839 break; 2840 } 2841 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 2842 } 2843 ConcatOps[--Idx] = LdOps[i]; 2844 for (--i; i >= 0; --i) { 2845 EVT NewLdTy = LdOps[i].getValueType(); 2846 if (NewLdTy != LdTy) { 2847 // Create a larger vector 2848 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 2849 makeArrayRef(&ConcatOps[Idx], End - Idx)); 2850 Idx = End - 1; 2851 LdTy = NewLdTy; 2852 } 2853 ConcatOps[--Idx] = LdOps[i]; 2854 } 2855 2856 if (WidenWidth == LdTy.getSizeInBits()*(End - Idx)) 2857 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2858 makeArrayRef(&ConcatOps[Idx], End - Idx)); 2859 2860 // We need to fill the rest with undefs to build the vector 2861 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 2862 SmallVector<SDValue, 16> WidenOps(NumOps); 2863 SDValue UndefVal = DAG.getUNDEF(LdTy); 2864 { 2865 unsigned i = 0; 2866 for (; i != End-Idx; ++i) 2867 WidenOps[i] = ConcatOps[Idx+i]; 2868 for (; i != NumOps; ++i) 2869 WidenOps[i] = UndefVal; 2870 } 2871 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 2872} 2873 2874SDValue 2875DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 2876 LoadSDNode *LD, 2877 ISD::LoadExtType ExtType) { 2878 // For extension loads, it may not be more efficient to chop up the vector 2879 // and then extended it. Instead, we unroll the load and build a new vector. 2880 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2881 EVT LdVT = LD->getMemoryVT(); 2882 SDLoc dl(LD); 2883 assert(LdVT.isVector() && WidenVT.isVector()); 2884 2885 // Load information 2886 SDValue Chain = LD->getChain(); 2887 SDValue BasePtr = LD->getBasePtr(); 2888 unsigned Align = LD->getAlignment(); 2889 bool isVolatile = LD->isVolatile(); 2890 bool isNonTemporal = LD->isNonTemporal(); 2891 const MDNode *TBAAInfo = LD->getTBAAInfo(); 2892 2893 EVT EltVT = WidenVT.getVectorElementType(); 2894 EVT LdEltVT = LdVT.getVectorElementType(); 2895 unsigned NumElts = LdVT.getVectorNumElements(); 2896 2897 // Load each element and widen 2898 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2899 SmallVector<SDValue, 16> Ops(WidenNumElts); 2900 unsigned Increment = LdEltVT.getSizeInBits() / 8; 2901 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 2902 LD->getPointerInfo(), 2903 LdEltVT, isVolatile, isNonTemporal, Align, TBAAInfo); 2904 LdChain.push_back(Ops[0].getValue(1)); 2905 unsigned i = 0, Offset = Increment; 2906 for (i=1; i < NumElts; ++i, Offset += Increment) { 2907 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2908 BasePtr, 2909 DAG.getConstant(Offset, 2910 BasePtr.getValueType())); 2911 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 2912 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 2913 isVolatile, isNonTemporal, Align, TBAAInfo); 2914 LdChain.push_back(Ops[i].getValue(1)); 2915 } 2916 2917 // Fill the rest with undefs 2918 SDValue UndefVal = DAG.getUNDEF(EltVT); 2919 for (; i != WidenNumElts; ++i) 2920 Ops[i] = UndefVal; 2921 2922 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2923} 2924 2925 2926void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 2927 StoreSDNode *ST) { 2928 // The strategy assumes that we can efficiently store powers of two widths. 2929 // The routines chops the vector into the largest vector stores with the same 2930 // element type or scalar stores. 2931 SDValue Chain = ST->getChain(); 2932 SDValue BasePtr = ST->getBasePtr(); 2933 unsigned Align = ST->getAlignment(); 2934 bool isVolatile = ST->isVolatile(); 2935 bool isNonTemporal = ST->isNonTemporal(); 2936 const MDNode *TBAAInfo = ST->getTBAAInfo(); 2937 SDValue ValOp = GetWidenedVector(ST->getValue()); 2938 SDLoc dl(ST); 2939 2940 EVT StVT = ST->getMemoryVT(); 2941 unsigned StWidth = StVT.getSizeInBits(); 2942 EVT ValVT = ValOp.getValueType(); 2943 unsigned ValWidth = ValVT.getSizeInBits(); 2944 EVT ValEltVT = ValVT.getVectorElementType(); 2945 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 2946 assert(StVT.getVectorElementType() == ValEltVT); 2947 2948 int Idx = 0; // current index to store 2949 unsigned Offset = 0; // offset from base to store 2950 while (StWidth != 0) { 2951 // Find the largest vector type we can store with 2952 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 2953 unsigned NewVTWidth = NewVT.getSizeInBits(); 2954 unsigned Increment = NewVTWidth / 8; 2955 if (NewVT.isVector()) { 2956 unsigned NumVTElts = NewVT.getVectorNumElements(); 2957 do { 2958 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 2959 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 2960 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2961 ST->getPointerInfo().getWithOffset(Offset), 2962 isVolatile, isNonTemporal, 2963 MinAlign(Align, Offset), TBAAInfo)); 2964 StWidth -= NewVTWidth; 2965 Offset += Increment; 2966 Idx += NumVTElts; 2967 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2968 DAG.getConstant(Increment, BasePtr.getValueType())); 2969 } while (StWidth != 0 && StWidth >= NewVTWidth); 2970 } else { 2971 // Cast the vector to the scalar type we can store 2972 unsigned NumElts = ValWidth / NewVTWidth; 2973 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2974 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 2975 // Readjust index position based on new vector type 2976 Idx = Idx * ValEltWidth / NewVTWidth; 2977 do { 2978 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 2979 DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 2980 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2981 ST->getPointerInfo().getWithOffset(Offset), 2982 isVolatile, isNonTemporal, 2983 MinAlign(Align, Offset), TBAAInfo)); 2984 StWidth -= NewVTWidth; 2985 Offset += Increment; 2986 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2987 DAG.getConstant(Increment, BasePtr.getValueType())); 2988 } while (StWidth != 0 && StWidth >= NewVTWidth); 2989 // Restore index back to be relative to the original widen element type 2990 Idx = Idx * NewVTWidth / ValEltWidth; 2991 } 2992 } 2993} 2994 2995void 2996DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain, 2997 StoreSDNode *ST) { 2998 // For extension loads, it may not be more efficient to truncate the vector 2999 // and then store it. Instead, we extract each element and then store it. 3000 SDValue Chain = ST->getChain(); 3001 SDValue BasePtr = ST->getBasePtr(); 3002 unsigned Align = ST->getAlignment(); 3003 bool isVolatile = ST->isVolatile(); 3004 bool isNonTemporal = ST->isNonTemporal(); 3005 const MDNode *TBAAInfo = ST->getTBAAInfo(); 3006 SDValue ValOp = GetWidenedVector(ST->getValue()); 3007 SDLoc dl(ST); 3008 3009 EVT StVT = ST->getMemoryVT(); 3010 EVT ValVT = ValOp.getValueType(); 3011 3012 // It must be true that we the widen vector type is bigger than where 3013 // we need to store. 3014 assert(StVT.isVector() && ValOp.getValueType().isVector()); 3015 assert(StVT.bitsLT(ValOp.getValueType())); 3016 3017 // For truncating stores, we can not play the tricks of chopping legal 3018 // vector types and bit cast it to the right type. Instead, we unroll 3019 // the store. 3020 EVT StEltVT = StVT.getVectorElementType(); 3021 EVT ValEltVT = ValVT.getVectorElementType(); 3022 unsigned Increment = ValEltVT.getSizeInBits() / 8; 3023 unsigned NumElts = StVT.getVectorNumElements(); 3024 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3025 DAG.getConstant(0, TLI.getVectorIdxTy())); 3026 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 3027 ST->getPointerInfo(), StEltVT, 3028 isVolatile, isNonTemporal, Align, 3029 TBAAInfo)); 3030 unsigned Offset = Increment; 3031 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 3032 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 3033 BasePtr, DAG.getConstant(Offset, 3034 BasePtr.getValueType())); 3035 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3036 DAG.getConstant(0, TLI.getVectorIdxTy())); 3037 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 3038 ST->getPointerInfo().getWithOffset(Offset), 3039 StEltVT, isVolatile, isNonTemporal, 3040 MinAlign(Align, Offset), TBAAInfo)); 3041 } 3042} 3043 3044/// Modifies a vector input (widen or narrows) to a vector of NVT. The 3045/// input vector must have the same element type as NVT. 3046SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 3047 // Note that InOp might have been widened so it might already have 3048 // the right width or it might need be narrowed. 3049 EVT InVT = InOp.getValueType(); 3050 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 3051 "input and widen element type must match"); 3052 SDLoc dl(InOp); 3053 3054 // Check if InOp already has the right width. 3055 if (InVT == NVT) 3056 return InOp; 3057 3058 unsigned InNumElts = InVT.getVectorNumElements(); 3059 unsigned WidenNumElts = NVT.getVectorNumElements(); 3060 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 3061 unsigned NumConcat = WidenNumElts / InNumElts; 3062 SmallVector<SDValue, 16> Ops(NumConcat); 3063 SDValue UndefVal = DAG.getUNDEF(InVT); 3064 Ops[0] = InOp; 3065 for (unsigned i = 1; i != NumConcat; ++i) 3066 Ops[i] = UndefVal; 3067 3068 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 3069 } 3070 3071 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 3072 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 3073 DAG.getConstant(0, TLI.getVectorIdxTy())); 3074 3075 // Fall back to extract and build. 3076 SmallVector<SDValue, 16> Ops(WidenNumElts); 3077 EVT EltVT = NVT.getVectorElementType(); 3078 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 3079 unsigned Idx; 3080 for (Idx = 0; Idx < MinNumElts; ++Idx) 3081 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3082 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 3083 3084 SDValue UndefVal = DAG.getUNDEF(EltVT); 3085 for ( ; Idx < WidenNumElts; ++Idx) 3086 Ops[Idx] = UndefVal; 3087 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Ops); 3088} 3089