InstCombineVectorOps.cpp revision 7f1f4089a142e4f0081ac25699bf682a68247a35
1//===- InstCombineVectorOps.cpp -------------------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements instcombine for ExtractElement, InsertElement and 11// ShuffleVector. 12// 13//===----------------------------------------------------------------------===// 14 15#include "InstCombine.h" 16using namespace llvm; 17 18/// CheapToScalarize - Return true if the value is cheaper to scalarize than it 19/// is to leave as a vector operation. 20static bool CheapToScalarize(Value *V, bool isConstant) { 21 if (isa<ConstantAggregateZero>(V)) 22 return true; 23 if (ConstantVector *C = dyn_cast<ConstantVector>(V)) { 24 if (isConstant) return true; 25 // If all elts are the same, we can extract. 26 Constant *Op0 = C->getOperand(0); 27 for (unsigned i = 1; i < C->getNumOperands(); ++i) 28 if (C->getOperand(i) != Op0) 29 return false; 30 return true; 31 } 32 Instruction *I = dyn_cast<Instruction>(V); 33 if (!I) return false; 34 35 // Insert element gets simplified to the inserted element or is deleted if 36 // this is constant idx extract element and its a constant idx insertelt. 37 if (I->getOpcode() == Instruction::InsertElement && isConstant && 38 isa<ConstantInt>(I->getOperand(2))) 39 return true; 40 if (I->getOpcode() == Instruction::Load && I->hasOneUse()) 41 return true; 42 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) 43 if (BO->hasOneUse() && 44 (CheapToScalarize(BO->getOperand(0), isConstant) || 45 CheapToScalarize(BO->getOperand(1), isConstant))) 46 return true; 47 if (CmpInst *CI = dyn_cast<CmpInst>(I)) 48 if (CI->hasOneUse() && 49 (CheapToScalarize(CI->getOperand(0), isConstant) || 50 CheapToScalarize(CI->getOperand(1), isConstant))) 51 return true; 52 53 return false; 54} 55 56/// Read and decode a shufflevector mask. 57/// 58/// It turns undef elements into values that are larger than the number of 59/// elements in the input. 60static std::vector<unsigned> getShuffleMask(const ShuffleVectorInst *SVI) { 61 unsigned NElts = SVI->getType()->getNumElements(); 62 if (isa<ConstantAggregateZero>(SVI->getOperand(2))) 63 return std::vector<unsigned>(NElts, 0); 64 if (isa<UndefValue>(SVI->getOperand(2))) 65 return std::vector<unsigned>(NElts, 2*NElts); 66 67 std::vector<unsigned> Result; 68 const ConstantVector *CP = cast<ConstantVector>(SVI->getOperand(2)); 69 for (User::const_op_iterator i = CP->op_begin(), e = CP->op_end(); i!=e; ++i) 70 if (isa<UndefValue>(*i)) 71 Result.push_back(NElts*2); // undef -> 8 72 else 73 Result.push_back(cast<ConstantInt>(*i)->getZExtValue()); 74 return Result; 75} 76 77/// FindScalarElement - Given a vector and an element number, see if the scalar 78/// value is already around as a register, for example if it were inserted then 79/// extracted from the vector. 80static Value *FindScalarElement(Value *V, unsigned EltNo) { 81 assert(V->getType()->isVectorTy() && "Not looking at a vector?"); 82 const VectorType *PTy = cast<VectorType>(V->getType()); 83 unsigned Width = PTy->getNumElements(); 84 if (EltNo >= Width) // Out of range access. 85 return UndefValue::get(PTy->getElementType()); 86 87 if (isa<UndefValue>(V)) 88 return UndefValue::get(PTy->getElementType()); 89 if (isa<ConstantAggregateZero>(V)) 90 return Constant::getNullValue(PTy->getElementType()); 91 if (ConstantVector *CP = dyn_cast<ConstantVector>(V)) 92 return CP->getOperand(EltNo); 93 94 if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) { 95 // If this is an insert to a variable element, we don't know what it is. 96 if (!isa<ConstantInt>(III->getOperand(2))) 97 return 0; 98 unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue(); 99 100 // If this is an insert to the element we are looking for, return the 101 // inserted value. 102 if (EltNo == IIElt) 103 return III->getOperand(1); 104 105 // Otherwise, the insertelement doesn't modify the value, recurse on its 106 // vector input. 107 return FindScalarElement(III->getOperand(0), EltNo); 108 } 109 110 if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V)) { 111 unsigned LHSWidth = 112 cast<VectorType>(SVI->getOperand(0)->getType())->getNumElements(); 113 unsigned InEl = getShuffleMask(SVI)[EltNo]; 114 if (InEl < LHSWidth) 115 return FindScalarElement(SVI->getOperand(0), InEl); 116 else if (InEl < LHSWidth*2) 117 return FindScalarElement(SVI->getOperand(1), InEl - LHSWidth); 118 else 119 return UndefValue::get(PTy->getElementType()); 120 } 121 122 // Otherwise, we don't know. 123 return 0; 124} 125 126Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) { 127 // If vector val is undef, replace extract with scalar undef. 128 if (isa<UndefValue>(EI.getOperand(0))) 129 return ReplaceInstUsesWith(EI, UndefValue::get(EI.getType())); 130 131 // If vector val is constant 0, replace extract with scalar 0. 132 if (isa<ConstantAggregateZero>(EI.getOperand(0))) 133 return ReplaceInstUsesWith(EI, Constant::getNullValue(EI.getType())); 134 135 if (ConstantVector *C = dyn_cast<ConstantVector>(EI.getOperand(0))) { 136 // If vector val is constant with all elements the same, replace EI with 137 // that element. When the elements are not identical, we cannot replace yet 138 // (we do that below, but only when the index is constant). 139 Constant *op0 = C->getOperand(0); 140 for (unsigned i = 1; i != C->getNumOperands(); ++i) 141 if (C->getOperand(i) != op0) { 142 op0 = 0; 143 break; 144 } 145 if (op0) 146 return ReplaceInstUsesWith(EI, op0); 147 } 148 149 // If extracting a specified index from the vector, see if we can recursively 150 // find a previously computed scalar that was inserted into the vector. 151 if (ConstantInt *IdxC = dyn_cast<ConstantInt>(EI.getOperand(1))) { 152 unsigned IndexVal = IdxC->getZExtValue(); 153 unsigned VectorWidth = EI.getVectorOperandType()->getNumElements(); 154 155 // If this is extracting an invalid index, turn this into undef, to avoid 156 // crashing the code below. 157 if (IndexVal >= VectorWidth) 158 return ReplaceInstUsesWith(EI, UndefValue::get(EI.getType())); 159 160 // This instruction only demands the single element from the input vector. 161 // If the input vector has a single use, simplify it based on this use 162 // property. 163 if (EI.getOperand(0)->hasOneUse() && VectorWidth != 1) { 164 APInt UndefElts(VectorWidth, 0); 165 APInt DemandedMask(VectorWidth, 0); 166 DemandedMask.set(IndexVal); 167 if (Value *V = SimplifyDemandedVectorElts(EI.getOperand(0), 168 DemandedMask, UndefElts)) { 169 EI.setOperand(0, V); 170 return &EI; 171 } 172 } 173 174 if (Value *Elt = FindScalarElement(EI.getOperand(0), IndexVal)) 175 return ReplaceInstUsesWith(EI, Elt); 176 177 // If the this extractelement is directly using a bitcast from a vector of 178 // the same number of elements, see if we can find the source element from 179 // it. In this case, we will end up needing to bitcast the scalars. 180 if (BitCastInst *BCI = dyn_cast<BitCastInst>(EI.getOperand(0))) { 181 if (const VectorType *VT = 182 dyn_cast<VectorType>(BCI->getOperand(0)->getType())) 183 if (VT->getNumElements() == VectorWidth) 184 if (Value *Elt = FindScalarElement(BCI->getOperand(0), IndexVal)) 185 return new BitCastInst(Elt, EI.getType()); 186 } 187 } 188 189 if (Instruction *I = dyn_cast<Instruction>(EI.getOperand(0))) { 190 // Push extractelement into predecessor operation if legal and 191 // profitable to do so 192 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) { 193 if (I->hasOneUse() && 194 CheapToScalarize(BO, isa<ConstantInt>(EI.getOperand(1)))) { 195 Value *newEI0 = 196 Builder->CreateExtractElement(BO->getOperand(0), EI.getOperand(1), 197 EI.getName()+".lhs"); 198 Value *newEI1 = 199 Builder->CreateExtractElement(BO->getOperand(1), EI.getOperand(1), 200 EI.getName()+".rhs"); 201 return BinaryOperator::Create(BO->getOpcode(), newEI0, newEI1); 202 } 203 } else if (InsertElementInst *IE = dyn_cast<InsertElementInst>(I)) { 204 // Extracting the inserted element? 205 if (IE->getOperand(2) == EI.getOperand(1)) 206 return ReplaceInstUsesWith(EI, IE->getOperand(1)); 207 // If the inserted and extracted elements are constants, they must not 208 // be the same value, extract from the pre-inserted value instead. 209 if (isa<Constant>(IE->getOperand(2)) && isa<Constant>(EI.getOperand(1))) { 210 Worklist.AddValue(EI.getOperand(0)); 211 EI.setOperand(0, IE->getOperand(0)); 212 return &EI; 213 } 214 } else if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I)) { 215 // If this is extracting an element from a shufflevector, figure out where 216 // it came from and extract from the appropriate input element instead. 217 if (ConstantInt *Elt = dyn_cast<ConstantInt>(EI.getOperand(1))) { 218 unsigned SrcIdx = getShuffleMask(SVI)[Elt->getZExtValue()]; 219 Value *Src; 220 unsigned LHSWidth = 221 cast<VectorType>(SVI->getOperand(0)->getType())->getNumElements(); 222 223 if (SrcIdx < LHSWidth) 224 Src = SVI->getOperand(0); 225 else if (SrcIdx < LHSWidth*2) { 226 SrcIdx -= LHSWidth; 227 Src = SVI->getOperand(1); 228 } else { 229 return ReplaceInstUsesWith(EI, UndefValue::get(EI.getType())); 230 } 231 return ExtractElementInst::Create(Src, 232 ConstantInt::get(Type::getInt32Ty(EI.getContext()), 233 SrcIdx, false)); 234 } 235 } 236 // FIXME: Canonicalize extractelement(bitcast) -> bitcast(extractelement) 237 } 238 return 0; 239} 240 241/// CollectSingleShuffleElements - If V is a shuffle of values that ONLY returns 242/// elements from either LHS or RHS, return the shuffle mask and true. 243/// Otherwise, return false. 244static bool CollectSingleShuffleElements(Value *V, Value *LHS, Value *RHS, 245 std::vector<Constant*> &Mask) { 246 assert(V->getType() == LHS->getType() && V->getType() == RHS->getType() && 247 "Invalid CollectSingleShuffleElements"); 248 unsigned NumElts = cast<VectorType>(V->getType())->getNumElements(); 249 250 if (isa<UndefValue>(V)) { 251 Mask.assign(NumElts, UndefValue::get(Type::getInt32Ty(V->getContext()))); 252 return true; 253 } 254 255 if (V == LHS) { 256 for (unsigned i = 0; i != NumElts; ++i) 257 Mask.push_back(ConstantInt::get(Type::getInt32Ty(V->getContext()), i)); 258 return true; 259 } 260 261 if (V == RHS) { 262 for (unsigned i = 0; i != NumElts; ++i) 263 Mask.push_back(ConstantInt::get(Type::getInt32Ty(V->getContext()), 264 i+NumElts)); 265 return true; 266 } 267 268 if (InsertElementInst *IEI = dyn_cast<InsertElementInst>(V)) { 269 // If this is an insert of an extract from some other vector, include it. 270 Value *VecOp = IEI->getOperand(0); 271 Value *ScalarOp = IEI->getOperand(1); 272 Value *IdxOp = IEI->getOperand(2); 273 274 if (!isa<ConstantInt>(IdxOp)) 275 return false; 276 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue(); 277 278 if (isa<UndefValue>(ScalarOp)) { // inserting undef into vector. 279 // Okay, we can handle this if the vector we are insertinting into is 280 // transitively ok. 281 if (CollectSingleShuffleElements(VecOp, LHS, RHS, Mask)) { 282 // If so, update the mask to reflect the inserted undef. 283 Mask[InsertedIdx] = UndefValue::get(Type::getInt32Ty(V->getContext())); 284 return true; 285 } 286 } else if (ExtractElementInst *EI = dyn_cast<ExtractElementInst>(ScalarOp)){ 287 if (isa<ConstantInt>(EI->getOperand(1)) && 288 EI->getOperand(0)->getType() == V->getType()) { 289 unsigned ExtractedIdx = 290 cast<ConstantInt>(EI->getOperand(1))->getZExtValue(); 291 292 // This must be extracting from either LHS or RHS. 293 if (EI->getOperand(0) == LHS || EI->getOperand(0) == RHS) { 294 // Okay, we can handle this if the vector we are insertinting into is 295 // transitively ok. 296 if (CollectSingleShuffleElements(VecOp, LHS, RHS, Mask)) { 297 // If so, update the mask to reflect the inserted value. 298 if (EI->getOperand(0) == LHS) { 299 Mask[InsertedIdx % NumElts] = 300 ConstantInt::get(Type::getInt32Ty(V->getContext()), 301 ExtractedIdx); 302 } else { 303 assert(EI->getOperand(0) == RHS); 304 Mask[InsertedIdx % NumElts] = 305 ConstantInt::get(Type::getInt32Ty(V->getContext()), 306 ExtractedIdx+NumElts); 307 308 } 309 return true; 310 } 311 } 312 } 313 } 314 } 315 // TODO: Handle shufflevector here! 316 317 return false; 318} 319 320/// CollectShuffleElements - We are building a shuffle of V, using RHS as the 321/// RHS of the shuffle instruction, if it is not null. Return a shuffle mask 322/// that computes V and the LHS value of the shuffle. 323static Value *CollectShuffleElements(Value *V, std::vector<Constant*> &Mask, 324 Value *&RHS) { 325 assert(V->getType()->isVectorTy() && 326 (RHS == 0 || V->getType() == RHS->getType()) && 327 "Invalid shuffle!"); 328 unsigned NumElts = cast<VectorType>(V->getType())->getNumElements(); 329 330 if (isa<UndefValue>(V)) { 331 Mask.assign(NumElts, UndefValue::get(Type::getInt32Ty(V->getContext()))); 332 return V; 333 } else if (isa<ConstantAggregateZero>(V)) { 334 Mask.assign(NumElts, ConstantInt::get(Type::getInt32Ty(V->getContext()),0)); 335 return V; 336 } else if (InsertElementInst *IEI = dyn_cast<InsertElementInst>(V)) { 337 // If this is an insert of an extract from some other vector, include it. 338 Value *VecOp = IEI->getOperand(0); 339 Value *ScalarOp = IEI->getOperand(1); 340 Value *IdxOp = IEI->getOperand(2); 341 342 if (ExtractElementInst *EI = dyn_cast<ExtractElementInst>(ScalarOp)) { 343 if (isa<ConstantInt>(EI->getOperand(1)) && isa<ConstantInt>(IdxOp) && 344 EI->getOperand(0)->getType() == V->getType()) { 345 unsigned ExtractedIdx = 346 cast<ConstantInt>(EI->getOperand(1))->getZExtValue(); 347 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue(); 348 349 // Either the extracted from or inserted into vector must be RHSVec, 350 // otherwise we'd end up with a shuffle of three inputs. 351 if (EI->getOperand(0) == RHS || RHS == 0) { 352 RHS = EI->getOperand(0); 353 Value *V = CollectShuffleElements(VecOp, Mask, RHS); 354 Mask[InsertedIdx % NumElts] = 355 ConstantInt::get(Type::getInt32Ty(V->getContext()), 356 NumElts+ExtractedIdx); 357 return V; 358 } 359 360 if (VecOp == RHS) { 361 Value *V = CollectShuffleElements(EI->getOperand(0), Mask, RHS); 362 // Everything but the extracted element is replaced with the RHS. 363 for (unsigned i = 0; i != NumElts; ++i) { 364 if (i != InsertedIdx) 365 Mask[i] = ConstantInt::get(Type::getInt32Ty(V->getContext()), 366 NumElts+i); 367 } 368 return V; 369 } 370 371 // If this insertelement is a chain that comes from exactly these two 372 // vectors, return the vector and the effective shuffle. 373 if (CollectSingleShuffleElements(IEI, EI->getOperand(0), RHS, Mask)) 374 return EI->getOperand(0); 375 } 376 } 377 } 378 // TODO: Handle shufflevector here! 379 380 // Otherwise, can't do anything fancy. Return an identity vector. 381 for (unsigned i = 0; i != NumElts; ++i) 382 Mask.push_back(ConstantInt::get(Type::getInt32Ty(V->getContext()), i)); 383 return V; 384} 385 386Instruction *InstCombiner::visitInsertElementInst(InsertElementInst &IE) { 387 Value *VecOp = IE.getOperand(0); 388 Value *ScalarOp = IE.getOperand(1); 389 Value *IdxOp = IE.getOperand(2); 390 391 // Inserting an undef or into an undefined place, remove this. 392 if (isa<UndefValue>(ScalarOp) || isa<UndefValue>(IdxOp)) 393 ReplaceInstUsesWith(IE, VecOp); 394 395 // If the inserted element was extracted from some other vector, and if the 396 // indexes are constant, try to turn this into a shufflevector operation. 397 if (ExtractElementInst *EI = dyn_cast<ExtractElementInst>(ScalarOp)) { 398 if (isa<ConstantInt>(EI->getOperand(1)) && isa<ConstantInt>(IdxOp) && 399 EI->getOperand(0)->getType() == IE.getType()) { 400 unsigned NumVectorElts = IE.getType()->getNumElements(); 401 unsigned ExtractedIdx = 402 cast<ConstantInt>(EI->getOperand(1))->getZExtValue(); 403 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue(); 404 405 if (ExtractedIdx >= NumVectorElts) // Out of range extract. 406 return ReplaceInstUsesWith(IE, VecOp); 407 408 if (InsertedIdx >= NumVectorElts) // Out of range insert. 409 return ReplaceInstUsesWith(IE, UndefValue::get(IE.getType())); 410 411 // If we are extracting a value from a vector, then inserting it right 412 // back into the same place, just use the input vector. 413 if (EI->getOperand(0) == VecOp && ExtractedIdx == InsertedIdx) 414 return ReplaceInstUsesWith(IE, VecOp); 415 416 // If this insertelement isn't used by some other insertelement, turn it 417 // (and any insertelements it points to), into one big shuffle. 418 if (!IE.hasOneUse() || !isa<InsertElementInst>(IE.use_back())) { 419 std::vector<Constant*> Mask; 420 Value *RHS = 0; 421 Value *LHS = CollectShuffleElements(&IE, Mask, RHS); 422 if (RHS == 0) RHS = UndefValue::get(LHS->getType()); 423 // We now have a shuffle of LHS, RHS, Mask. 424 return new ShuffleVectorInst(LHS, RHS, 425 ConstantVector::get(Mask)); 426 } 427 } 428 } 429 430 unsigned VWidth = cast<VectorType>(VecOp->getType())->getNumElements(); 431 APInt UndefElts(VWidth, 0); 432 APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth)); 433 if (SimplifyDemandedVectorElts(&IE, AllOnesEltMask, UndefElts)) 434 return &IE; 435 436 return 0; 437} 438 439 440Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) { 441 Value *LHS = SVI.getOperand(0); 442 Value *RHS = SVI.getOperand(1); 443 std::vector<unsigned> Mask = getShuffleMask(&SVI); 444 445 bool MadeChange = false; 446 447 // Undefined shuffle mask -> undefined value. 448 if (isa<UndefValue>(SVI.getOperand(2))) 449 return ReplaceInstUsesWith(SVI, UndefValue::get(SVI.getType())); 450 451 unsigned VWidth = Mask.size(); 452 unsigned LHSWidth = cast<VectorType>(LHS->getType())->getNumElements(); 453 454 APInt UndefElts(VWidth, 0); 455 APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth)); 456 if (SimplifyDemandedVectorElts(&SVI, AllOnesEltMask, UndefElts)) { 457 LHS = SVI.getOperand(0); 458 RHS = SVI.getOperand(1); 459 MadeChange = true; 460 } 461 462 // Canonicalize shuffle(x ,x,mask) -> shuffle(x, undef,mask') 463 // Canonicalize shuffle(undef,x,mask) -> shuffle(x, undef,mask'). 464 if (LHS == RHS || isa<UndefValue>(LHS)) { 465 if (isa<UndefValue>(LHS) && LHS == RHS) 466 return ReplaceInstUsesWith(SVI, UndefValue::get(SVI.getType())); 467 468 // Remap any references to RHS to use LHS. 469 std::vector<Constant*> Elts; 470 for (unsigned i = 0, e = LHSWidth; i != VWidth; ++i) { 471 if (Mask[i] >= 2*e) 472 Elts.push_back(UndefValue::get(Type::getInt32Ty(SVI.getContext()))); 473 else { 474 if ((Mask[i] >= e && isa<UndefValue>(RHS)) || 475 (Mask[i] < e && isa<UndefValue>(LHS))) { 476 Mask[i] = 2*e; // Turn into undef. 477 Elts.push_back(UndefValue::get(Type::getInt32Ty(SVI.getContext()))); 478 } else { 479 Mask[i] = Mask[i] % e; // Force to LHS. 480 Elts.push_back(ConstantInt::get(Type::getInt32Ty(SVI.getContext()), 481 Mask[i])); 482 } 483 } 484 } 485 SVI.setOperand(0, SVI.getOperand(1)); 486 SVI.setOperand(1, UndefValue::get(RHS->getType())); 487 SVI.setOperand(2, ConstantVector::get(Elts)); 488 LHS = SVI.getOperand(0); 489 RHS = SVI.getOperand(1); 490 MadeChange = true; 491 } 492 493 // Analyze the shuffle, are the LHS or RHS and identity shuffles? 494 if (VWidth == LHSWidth) { 495 bool isLHSID = true, isRHSID = true; 496 497 for (unsigned i = 0, e = Mask.size(); i != e; ++i) { 498 if (Mask[i] >= e*2) continue; // Ignore undef values. 499 // Is this an identity shuffle of the LHS value? 500 isLHSID &= (Mask[i] == i); 501 502 // Is this an identity shuffle of the RHS value? 503 isRHSID &= (Mask[i]-e == i); 504 } 505 506 // Eliminate identity shuffles. 507 if (isLHSID) return ReplaceInstUsesWith(SVI, LHS); 508 if (isRHSID) return ReplaceInstUsesWith(SVI, RHS); 509 } 510 511 // Check for a handful of important shuffle(shuffle()) combinations. 512 ShuffleVectorInst *LSVI = dyn_cast<ShuffleVectorInst>(LHS); 513 if (!LSVI) 514 return MadeChange ? &SVI : 0; 515 516 LHS = LSVI->getOperand(0); 517 std::vector<unsigned> LHSMask = getShuffleMask(LSVI); 518 unsigned LHSInNElts = cast<VectorType>(LHS->getType())->getNumElements(); 519 520 // If the LHS is an identity shuffle, or if SVI + LHS form a full unpack 521 // operation, merge the LHS and SVI shuffles. This allows llvm to emit 522 // efficient code for matrix transposes written with generic vector ops. 523 bool isLHSLoExtract = true, isLHSHiExtract = true; 524 bool isUnpackLo = isPowerOf2_32(VWidth); 525 bool isUnpackHi = isPowerOf2_32(VWidth); 526 for (unsigned i = 0, e = LHSMask.size(); i != e; ++i) { 527 if (LHSMask[i] >= LHSInNElts*2) continue; // Ignore undef values; 528 isLHSLoExtract &= (LHSMask[i] == i); 529 isLHSHiExtract &= (LHSMask[i] == i+(LHSInNElts/2)); 530 isUnpackLo &= (LHSMask[i] == (i/2)); 531 isUnpackHi &= (LHSMask[i] == (i/2) + (e/2)); 532 } 533 for (unsigned i = 0, e = Mask.size(); i != e && (isUnpackLo || isUnpackHi); 534 i += 2) { 535 isUnpackLo &= (Mask[i] == i) && (Mask[i+1] == (i/2)+e); 536 isUnpackHi &= (Mask[i] == i) && (Mask[i+1] == (i/2)+e+(e/2)); 537 } 538 if ((isLHSLoExtract || isLHSHiExtract || isUnpackLo || isUnpackHi) && 539 (isa<UndefValue>(RHS) || (LHSWidth == LHSInNElts))) { 540 std::vector<Constant*> Elts; 541 for (unsigned i = 0, e = VWidth; i != e; ++i) { 542 if (Mask[i] >= 2*LHSWidth) 543 Elts.push_back(UndefValue::get(Type::getInt32Ty(SVI.getContext()))); 544 else if (Mask[i] >= e) 545 Elts.push_back(ConstantInt::get(Type::getInt32Ty(SVI.getContext()), 546 Mask[i])); 547 else 548 Elts.push_back(ConstantInt::get(Type::getInt32Ty(SVI.getContext()), 549 LHSMask[Mask[i]])); 550 } 551 if (isa<UndefValue>(RHS)) 552 RHS = UndefValue::get(LHS->getType()); 553 return new ShuffleVectorInst(LHS, RHS, ConstantVector::get(Elts)); 554 } 555 556 // If rhs is shuffle + identity, propagate. 557 if (ShuffleVectorInst *RSVI = dyn_cast<ShuffleVectorInst>(RHS)) { 558 std::vector<unsigned> RHSMask = getShuffleMask(RSVI); 559 unsigned RHSInNElts = 560 cast<VectorType>(RSVI->getOperand(0)->getType())->getNumElements(); 561 562 // If rhs is identity, propagate 563 bool isRHSLoExtract = true, isRHSHiExtract = true; 564 for (unsigned i = 0, e = RHSMask.size(); i != e; ++i) { 565 if (RHSMask[i] >= RHSInNElts*2) continue; // Ignore undef values; 566 isRHSLoExtract &= (RHSMask[i] == i); 567 isRHSHiExtract &= (RHSMask[i] == i+(RHSInNElts/2)); 568 } 569 if ((isRHSLoExtract || isRHSHiExtract) && (LHSWidth == RHSInNElts)) { 570 std::vector<Constant*> Elts; 571 for (unsigned i = 0, e = VWidth; i != e; ++i) { 572 if (Mask[i] >= 2*LHSWidth) 573 Elts.push_back(UndefValue::get(Type::getInt32Ty(SVI.getContext()))); 574 else if (Mask[i] < LHSWidth) 575 Elts.push_back(ConstantInt::get(Type::getInt32Ty(SVI.getContext()), 576 Mask[i])); 577 else 578 Elts.push_back(ConstantInt::get(Type::getInt32Ty(SVI.getContext()), 579 RHSMask[Mask[i]-LHSWidth]+LHSWidth)); 580 } 581 SVI.setOperand(1, RSVI->getOperand(0)); 582 SVI.setOperand(2, ConstantVector::get(Elts)); 583 return &SVI; 584 } 585 } 586 587 // Be extremely conservative when merging shufflevector instructions. It is 588 // difficult for the code generator to recognize a merged shuffle, which 589 // usually leads to worse code from merging a shuffle. 590 if (!isa<UndefValue>(RHS)) 591 return MadeChange ? &SVI : 0; 592 593 // If the merged shuffle mask is one of the two input shuffle masks, which 594 // just removes one instruction. This should handle splat(splat) -> splat. 595 if (LHSMask.size() == Mask.size()) { 596 std::vector<unsigned> NewMask; 597 for (unsigned i = 0, e = Mask.size(); i != e; ++i) 598 if (Mask[i] >= e) 599 NewMask.push_back(2*e); 600 else 601 NewMask.push_back(LHSMask[Mask[i]]); 602 603 // If the result mask is equal to the src shuffle or this shuffle mask, 604 // do the replacement. 605 if (NewMask == LHSMask || NewMask == Mask) { 606 std::vector<Constant*> Elts; 607 for (unsigned i = 0, e = NewMask.size(); i != e; ++i) { 608 if (NewMask[i] >= LHSInNElts*2) { 609 Elts.push_back(UndefValue::get(Type::getInt32Ty(SVI.getContext()))); 610 } else { 611 Elts.push_back(ConstantInt::get(Type::getInt32Ty(SVI.getContext()), 612 NewMask[i])); 613 } 614 } 615 return new ShuffleVectorInst(LHS, LSVI->getOperand(1), 616 ConstantVector::get(Elts)); 617 } 618 } 619 return MadeChange ? &SVI : 0; 620} 621