136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===--- Scalarizer.cpp - Scalarize vector operations ---------------------===// 236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// The LLVM Compiler Infrastructure 436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This file is distributed under the University of Illinois Open Source 636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// License. See LICENSE.TXT for details. 736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 1036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This pass converts vector operations into scalar operations, in order 1136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// to expose optimization opportunities on the individual scalar operations. 1236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// It is mainly intended for targets that do not have vector units, but it 1336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// may also be useful for revectorizing code to different vector widths. 1436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 1536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 1636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/ADT/STLExtras.h" 1836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/IRBuilder.h" 1936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/InstVisitor.h" 2036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Pass.h" 2136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Support/CommandLine.h" 2236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Scalar.h" 2336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/BasicBlockUtils.h" 2436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesusing namespace llvm; 2636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 27dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "scalarizer" 28dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesnamespace { 3036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Used to store the scattered form of a vector. 3136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinestypedef SmallVector<Value *, 8> ValueVector; 3236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Used to map a vector Value to its scattered form. We use std::map 3436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// because we want iterators to persist across insertion and because the 3536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// values are relatively large. 3636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinestypedef std::map<Value *, ValueVector> ScatterMap; 3736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Lists Instructions that have been replaced with scalar implementations, 3936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// along with a pointer to their scattered forms. 4036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinestypedef SmallVector<std::pair<Instruction *, ValueVector *>, 16> GatherList; 4136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Provides a very limited vector-like interface for lazily accessing one 4336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// component of a scattered vector or vector pointer. 4436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass Scatterer { 4536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 4636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer() {} 4736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Scatter V into Size components. If new instructions are needed, 4936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // insert them before BBI in BB. If Cache is nonnull, use it to cache 5036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the results. 5136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v, 52dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ValueVector *cachePtr = nullptr); 5336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Return component I, creating a new Value for it if necessary. 5536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *operator[](unsigned I); 5636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 5736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Return the number of components. 5836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned size() const { return Size; } 5936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 6036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesprivate: 6136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB; 6236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator BBI; 6336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V; 6436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector *CachePtr; 6536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PointerType *PtrTy; 6636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Tmp; 6736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Size; 6836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 6936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 7036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// FCmpSpliiter(FCI)(Builder, X, Y, Name) uses Builder to create an FCmp 7136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// called Name that compares X and Y in the same way as FCI. 7236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstruct FCmpSplitter { 7336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FCmpSplitter(FCmpInst &fci) : FCI(fci) {} 7436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1, 7536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Twine &Name) const { 7636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Builder.CreateFCmp(FCI.getPredicate(), Op0, Op1, Name); 7736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 7836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FCmpInst &FCI; 7936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 8036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// ICmpSpliiter(ICI)(Builder, X, Y, Name) uses Builder to create an ICmp 8236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// called Name that compares X and Y in the same way as ICI. 8336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstruct ICmpSplitter { 8436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ICmpSplitter(ICmpInst &ici) : ICI(ici) {} 8536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1, 8636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Twine &Name) const { 8736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Builder.CreateICmp(ICI.getPredicate(), Op0, Op1, Name); 8836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 8936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ICmpInst &ICI; 9036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 9136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 9236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// BinarySpliiter(BO)(Builder, X, Y, Name) uses Builder to create 9336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// a binary operator like BO called Name with operands X and Y. 9436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstruct BinarySplitter { 9536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BinarySplitter(BinaryOperator &bo) : BO(bo) {} 9636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1, 9736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Twine &Name) const { 9836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Builder.CreateBinOp(BO.getOpcode(), Op0, Op1, Name); 9936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 10036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BinaryOperator &BO; 10136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 10236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 10336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Information about a load or store that we're scalarizing. 10436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstruct VectorLayout { 105dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines VectorLayout() : VecTy(nullptr), ElemTy(nullptr), VecAlign(0), ElemSize(0) {} 10636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 10736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Return the alignment of element I. 10836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t getElemAlign(unsigned I) { 10936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MinAlign(VecAlign, I * ElemSize); 11036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 11136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 11236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The type of the vector. 11336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *VecTy; 11436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 11536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The type of each element. 11636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *ElemTy; 11736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 11836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The alignment of the vector. 11936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t VecAlign; 12036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The size of each element. 12236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t ElemSize; 12336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 12436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass Scalarizer : public FunctionPass, 12636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public InstVisitor<Scalarizer, bool> { 12736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 12836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static char ID; 12936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 13036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scalarizer() : 13136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines FunctionPass(ID) { 13236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines initializeScalarizerPass(*PassRegistry::getPassRegistry()); 13336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 13436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 13536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool doInitialization(Module &M) override; 13636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool runOnFunction(Function &F) override; 13736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 13836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // InstVisitor methods. They return true if the instruction was scalarized, 13936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // false if nothing changed. 14036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitInstruction(Instruction &) { return false; } 14136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitSelectInst(SelectInst &SI); 14236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitICmpInst(ICmpInst &); 14336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitFCmpInst(FCmpInst &); 14436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitBinaryOperator(BinaryOperator &); 14536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitGetElementPtrInst(GetElementPtrInst &); 14636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitCastInst(CastInst &); 14736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitBitCastInst(BitCastInst &); 14836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitShuffleVectorInst(ShuffleVectorInst &); 14936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitPHINode(PHINode &); 15036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitLoadInst(LoadInst &); 15136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool visitStoreInst(StoreInst &); 15236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 15336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesprivate: 15436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer scatter(Instruction *, Value *); 15536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void gather(Instruction *, const ValueVector &); 15636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool canTransferMetadata(unsigned Kind); 15736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void transferMetadata(Instruction *, const ValueVector &); 15836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool getVectorLayout(Type *, unsigned, VectorLayout &); 15936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool finish(); 16036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 16136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines template<typename T> bool splitBinary(Instruction &, const T &); 16236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 16336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ScatterMap Scattered; 16436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GatherList Gathered; 16536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned ParallelLoopAccessMDKind; 16636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const DataLayout *DL; 16736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 16836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 16936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hineschar Scalarizer::ID = 0; 17036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} // end anonymous namespace 17136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 17236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This is disabled by default because having separate loads and stores makes 17336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// it more likely that the -combiner-alias-analysis limits will be reached. 17436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<bool> ScalarizeLoadStore 17536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ("scalarize-load-store", cl::Hidden, cl::init(false), 17636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Allow the scalarizer pass to scalarize loads and store")); 17736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 17836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesINITIALIZE_PASS(Scalarizer, "scalarizer", "Scalarize vector operations", 17936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines false, false) 18036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 18136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesScatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v, 18236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector *cachePtr) 18336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : BB(bb), BBI(bbi), V(v), CachePtr(cachePtr) { 18436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *Ty = V->getType(); 18536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PtrTy = dyn_cast<PointerType>(Ty); 18636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PtrTy) 18736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Ty = PtrTy->getElementType(); 18836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Size = Ty->getVectorNumElements(); 18936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!CachePtr) 190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Tmp.resize(Size, nullptr); 19136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else if (CachePtr->empty()) 192dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CachePtr->resize(Size, nullptr); 19336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 19436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Size == CachePtr->size() && "Inconsistent vector sizes"); 19536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 19636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 19736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return component I, creating a new Value for it if necessary. 19836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesValue *Scatterer::operator[](unsigned I) { 19936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector &CV = (CachePtr ? *CachePtr : Tmp); 20036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Try to reuse a previous value. 20136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CV[I]) 20236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return CV[I]; 20336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(BB, BBI); 20436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PtrTy) { 20536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!CV[0]) { 20636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *Ty = 20736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PointerType::get(PtrTy->getElementType()->getVectorElementType(), 20836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PtrTy->getAddressSpace()); 20936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CV[0] = Builder.CreateBitCast(V, Ty, V->getName() + ".i0"); 21036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 21136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (I != 0) 21236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CV[I] = Builder.CreateConstGEP1_32(CV[0], I, 21336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V->getName() + ".i" + Twine(I)); 21436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 21536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Search through a chain of InsertElementInsts looking for element I. 21636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Record other elements in the cache. The new V is still suitable 21736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // for all uncached indices. 21836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (;;) { 21936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertElementInst *Insert = dyn_cast<InsertElementInst>(V); 22036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Insert) 22136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 22236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt *Idx = dyn_cast<ConstantInt>(Insert->getOperand(2)); 22336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Idx) 22436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 22536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned J = Idx->getZExtValue(); 22636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CV[J] = Insert->getOperand(1); 22736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Insert->getOperand(0); 22836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (I == J) 22936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return CV[J]; 23036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 23136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I), 23236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V->getName() + ".i" + Twine(I)); 23336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 23436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return CV[I]; 23536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 23636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::doInitialization(Module &M) { 23836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ParallelLoopAccessMDKind = 23936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines M.getContext().getMDKindID("llvm.mem.parallel_loop_access"); 24036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 24136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 24236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 24336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::runOnFunction(Function &F) { 24436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DataLayoutPass *DLP = getAnalysisIfAvailable<DataLayoutPass>(); 245dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DL = DLP ? &DLP->getDataLayout() : nullptr; 24636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) { 24736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = BBI; 24836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) { 24936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *I = II; 25036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Done = visit(I); 25136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++II; 25236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Done && I->getType()->isVoidTy()) 25336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I->eraseFromParent(); 25436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 25536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 25636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return finish(); 25736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 25836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 25936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return a scattered form of V that can be accessed by Point. V must be a 26036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// vector or a pointer to a vector. 26136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesScatterer Scalarizer::scatter(Instruction *Point, Value *V) { 26236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Argument *VArg = dyn_cast<Argument>(V)) { 26336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Put the scattered form of arguments in the entry block, 26436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // so that it can be used everywhere. 26536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Function *F = VArg->getParent(); 26636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = &F->getEntryBlock(); 26736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Scatterer(BB, BB->begin(), V, &Scattered[V]); 26836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 26936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Instruction *VOp = dyn_cast<Instruction>(V)) { 27036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Put the scattered form of an instruction directly after the 27136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // instruction. 27236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = VOp->getParent(); 27336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Scatterer(BB, std::next(BasicBlock::iterator(VOp)), 27436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V, &Scattered[V]); 27536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 27636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // In the fallback case, just put the scattered before Point and 27736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // keep the result local to Point. 27836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Scatterer(Point->getParent(), Point, V); 27936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 28036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 28136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Replace Op with the gathered form of the components in CV. Defer the 28236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// deletion of Op and creation of the gathered form to the end of the pass, 28336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// so that we can avoid creating the gathered form if all uses of Op are 28436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// replaced with uses of CV. 28536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid Scalarizer::gather(Instruction *Op, const ValueVector &CV) { 28636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Since we're not deleting Op yet, stub out its operands, so that it 28736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // doesn't make anything live unnecessarily. 28836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0, E = Op->getNumOperands(); I != E; ++I) 28936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op->setOperand(I, UndefValue::get(Op->getOperand(I)->getType())); 29036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 29136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines transferMetadata(Op, CV); 29236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 29336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we already have a scattered form of Op (created from ExtractElements 29436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // of Op itself), replace them with the new form. 29536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector &SV = Scattered[Op]; 29636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SV.empty()) { 29736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0, E = SV.size(); I != E; ++I) { 29836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Old = cast<Instruction>(SV[I]); 29936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CV[I]->takeName(Old); 30036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Old->replaceAllUsesWith(CV[I]); 30136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Old->eraseFromParent(); 30236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 30336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 30436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SV = CV; 30536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Gathered.push_back(GatherList::value_type(Op, &SV)); 30636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 30736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 30836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Return true if it is safe to transfer the given metadata tag from 30936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// vector to scalar instructions. 31036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::canTransferMetadata(unsigned Tag) { 31136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return (Tag == LLVMContext::MD_tbaa 31236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines || Tag == LLVMContext::MD_fpmath 31336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines || Tag == LLVMContext::MD_tbaa_struct 31436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines || Tag == LLVMContext::MD_invariant_load 31536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines || Tag == ParallelLoopAccessMDKind); 31636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 31736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 31836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Transfer metadata from Op to the instructions in CV if it is known 31936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// to be safe to do so. 32036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid Scalarizer::transferMetadata(Instruction *Op, const ValueVector &CV) { 32136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; 32236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op->getAllMetadataOtherThanDebugLoc(MDs); 32336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0, E = CV.size(); I != E; ++I) { 32436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Instruction *New = dyn_cast<Instruction>(CV[I])) { 32536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallVectorImpl<std::pair<unsigned, MDNode *> >::iterator 32636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MI = MDs.begin(), ME = MDs.end(); MI != ME; ++MI) 32736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (canTransferMetadata(MI->first)) 32836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines New->setMetadata(MI->first, MI->second); 32936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines New->setDebugLoc(Op->getDebugLoc()); 33036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 33136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 33236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 33336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 33436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Try to fill in Layout from Ty, returning true on success. Alignment is 33536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// the alignment of the vector, or 0 if the ABI default should be used. 33636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::getVectorLayout(Type *Ty, unsigned Alignment, 33736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorLayout &Layout) { 33836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DL) 33936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 34036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 34136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure we're dealing with a vector. 34236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Layout.VecTy = dyn_cast<VectorType>(Ty); 34336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Layout.VecTy) 34436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 34536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 34636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check that we're dealing with full-byte elements. 34736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Layout.ElemTy = Layout.VecTy->getElementType(); 34836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (DL->getTypeSizeInBits(Layout.ElemTy) != 34936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DL->getTypeStoreSizeInBits(Layout.ElemTy)) 35036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 35136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 35236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Alignment) 35336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Layout.VecAlign = Alignment; 35436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 35536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Layout.VecAlign = DL->getABITypeAlignment(Layout.VecTy); 35636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Layout.ElemSize = DL->getTypeStoreSize(Layout.ElemTy); 35736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 35836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 35936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 36036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Scalarize two-operand instruction I, using Split(Builder, X, Y, Name) 36136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// to create an instruction like I with operands X and Y and name Name. 36236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinestemplate<typename Splitter> 36336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::splitBinary(Instruction &I, const Splitter &Split) { 36436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *VT = dyn_cast<VectorType>(I.getType()); 36536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!VT) 36636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 36736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 36836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = VT->getNumElements(); 36936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(I.getParent(), &I); 37036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op0 = scatter(&I, I.getOperand(0)); 37136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op1 = scatter(&I, I.getOperand(1)); 37236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op0.size() == NumElems && "Mismatched binary operation"); 37336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op1.size() == NumElems && "Mismatched binary operation"); 37436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 37536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(NumElems); 37636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned Elem = 0; Elem < NumElems; ++Elem) 37736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[Elem] = Split(Builder, Op0[Elem], Op1[Elem], 37836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I.getName() + ".i" + Twine(Elem)); 37936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&I, Res); 38036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 38136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 38236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 38336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitSelectInst(SelectInst &SI) { 38436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *VT = dyn_cast<VectorType>(SI.getType()); 38536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!VT) 38636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 38736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 38836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = VT->getNumElements(); 38936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(SI.getParent(), &SI); 39036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op1 = scatter(&SI, SI.getOperand(1)); 39136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op2 = scatter(&SI, SI.getOperand(2)); 39236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op1.size() == NumElems && "Mismatched select"); 39336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op2.size() == NumElems && "Mismatched select"); 39436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 39536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(NumElems); 39636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 39736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SI.getOperand(0)->getType()->isVectorTy()) { 39836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op0 = scatter(&SI, SI.getOperand(0)); 39936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op0.size() == NumElems && "Mismatched select"); 40036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) 40136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Builder.CreateSelect(Op0[I], Op1[I], Op2[I], 40236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI.getName() + ".i" + Twine(I)); 40336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 40436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Op0 = SI.getOperand(0); 40536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) 40636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Builder.CreateSelect(Op0, Op1[I], Op2[I], 40736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI.getName() + ".i" + Twine(I)); 40836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 40936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&SI, Res); 41036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 41136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 41236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 41336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitICmpInst(ICmpInst &ICI) { 41436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return splitBinary(ICI, ICmpSplitter(ICI)); 41536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 41636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 41736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitFCmpInst(FCmpInst &FCI) { 41836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return splitBinary(FCI, FCmpSplitter(FCI)); 41936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 42036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 42136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitBinaryOperator(BinaryOperator &BO) { 42236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return splitBinary(BO, BinarySplitter(BO)); 42336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 42436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 42536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitGetElementPtrInst(GetElementPtrInst &GEPI) { 42636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *VT = dyn_cast<VectorType>(GEPI.getType()); 42736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!VT) 42836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 42936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 43036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(GEPI.getParent(), &GEPI); 43136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = VT->getNumElements(); 43236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumIndices = GEPI.getNumIndices(); 43336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 43436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Base = scatter(&GEPI, GEPI.getOperand(0)); 43536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 43636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Scatterer, 8> Ops; 43736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Ops.resize(NumIndices); 43836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumIndices; ++I) 43936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Ops[I] = scatter(&GEPI, GEPI.getOperand(I + 1)); 44036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 44136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 44236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(NumElems); 44336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) { 44436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Value *, 8> Indices; 44536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Indices.resize(NumIndices); 44636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned J = 0; J < NumIndices; ++J) 44736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Indices[J] = Ops[J][I]; 44836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Builder.CreateGEP(Base[I], Indices, 44936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GEPI.getName() + ".i" + Twine(I)); 45036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (GEPI.isInBounds()) 45136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I])) 45236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NewGEPI->setIsInBounds(); 45336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 45436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&GEPI, Res); 45536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 45636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 45736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 45836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitCastInst(CastInst &CI) { 45936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *VT = dyn_cast<VectorType>(CI.getDestTy()); 46036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!VT) 46136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 46236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 46336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = VT->getNumElements(); 46436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(CI.getParent(), &CI); 46536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op0 = scatter(&CI, CI.getOperand(0)); 46636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(Op0.size() == NumElems && "Mismatched cast"); 46736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 46836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(NumElems); 46936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) 47036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Builder.CreateCast(CI.getOpcode(), Op0[I], VT->getElementType(), 47136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI.getName() + ".i" + Twine(I)); 47236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&CI, Res); 47336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 47436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 47536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 47636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitBitCastInst(BitCastInst &BCI) { 47736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *DstVT = dyn_cast<VectorType>(BCI.getDestTy()); 47836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *SrcVT = dyn_cast<VectorType>(BCI.getSrcTy()); 47936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DstVT || !SrcVT) 48036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 48136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 48236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned DstNumElems = DstVT->getNumElements(); 48336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned SrcNumElems = SrcVT->getNumElements(); 48436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(BCI.getParent(), &BCI); 48536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op0 = scatter(&BCI, BCI.getOperand(0)); 48636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 48736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(DstNumElems); 48836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 48936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (DstNumElems == SrcNumElems) { 49036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < DstNumElems; ++I) 49136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Builder.CreateBitCast(Op0[I], DstVT->getElementType(), 49236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BCI.getName() + ".i" + Twine(I)); 49336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (DstNumElems > SrcNumElems) { 49436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // <M x t1> -> <N*M x t2>. Convert each t1 to <N x t2> and copy the 49536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // individual elements to the destination. 49636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned FanOut = DstNumElems / SrcNumElems; 49736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *MidTy = VectorType::get(DstVT->getElementType(), FanOut); 49836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned ResI = 0; 49936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) { 50036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = Op0[Op0I]; 50136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *VI; 50236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Look through any existing bitcasts before converting to <N x t2>. 50336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // In the best case, the resulting conversion might be a no-op. 50436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while ((VI = dyn_cast<Instruction>(V)) && 50536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VI->getOpcode() == Instruction::BitCast) 50636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = VI->getOperand(0); 50736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateBitCast(V, MidTy, V->getName() + ".cast"); 50836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Mid = scatter(&BCI, V); 50936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned MidI = 0; MidI < FanOut; ++MidI) 51036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[ResI++] = Mid[MidI]; 51136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 51236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 51336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // <N*M x t1> -> <M x t2>. Convert each group of <N x t1> into a t2. 51436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned FanIn = SrcNumElems / DstNumElems; 51536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *MidTy = VectorType::get(SrcVT->getElementType(), FanIn); 51636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Op0I = 0; 51736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned ResI = 0; ResI < DstNumElems; ++ResI) { 51836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = UndefValue::get(MidTy); 51936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned MidI = 0; MidI < FanIn; ++MidI) 52036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.getInt32(MidI), 52136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BCI.getName() + ".i" + Twine(ResI) 52236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines + ".upto" + Twine(MidI)); 52336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[ResI] = Builder.CreateBitCast(V, DstVT->getElementType(), 52436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BCI.getName() + ".i" + Twine(ResI)); 52536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 52636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 52736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&BCI, Res); 52836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 52936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 53036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 53136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitShuffleVectorInst(ShuffleVectorInst &SVI) { 53236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *VT = dyn_cast<VectorType>(SVI.getType()); 53336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!VT) 53436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 53536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 53636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = VT->getNumElements(); 53736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op0 = scatter(&SVI, SVI.getOperand(0)); 53836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op1 = scatter(&SVI, SVI.getOperand(1)); 53936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 54036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(NumElems); 54136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 54236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) { 54336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int Selector = SVI.getMaskValue(I); 54436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Selector < 0) 54536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = UndefValue::get(VT->getElementType()); 54636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else if (unsigned(Selector) < Op0.size()) 54736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Op0[Selector]; 54836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 54936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Op1[Selector - Op0.size()]; 55036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 55136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&SVI, Res); 55236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 55336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 55436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 55536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitPHINode(PHINode &PHI) { 55636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorType *VT = dyn_cast<VectorType>(PHI.getType()); 55736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!VT) 55836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 55936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 56036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = VT->getNumElements(); 56136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(PHI.getParent(), &PHI); 56236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 56336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(NumElems); 56436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 56536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumOps = PHI.getNumOperands(); 56636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) 56736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Builder.CreatePHI(VT->getElementType(), NumOps, 56836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PHI.getName() + ".i" + Twine(I)); 56936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 57036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumOps; ++I) { 57136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I)); 57236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *IncomingBlock = PHI.getIncomingBlock(I); 57336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned J = 0; J < NumElems; ++J) 57436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock); 57536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 57636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&PHI, Res); 57736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 57836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 57936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 58036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitLoadInst(LoadInst &LI) { 58136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ScalarizeLoadStore) 58236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 58336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!LI.isSimple()) 58436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 58536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 58636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorLayout Layout; 58736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!getVectorLayout(LI.getType(), LI.getAlignment(), Layout)) 58836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 58936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 59036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = Layout.VecTy->getNumElements(); 59136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(LI.getParent(), &LI); 59236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Ptr = scatter(&LI, LI.getPointerOperand()); 59336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Res; 59436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res.resize(NumElems); 59536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 59636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) 59736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res[I] = Builder.CreateAlignedLoad(Ptr[I], Layout.getElemAlign(I), 59836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LI.getName() + ".i" + Twine(I)); 59936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gather(&LI, Res); 60036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 60136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 60236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 60336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::visitStoreInst(StoreInst &SI) { 60436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ScalarizeLoadStore) 60536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 60636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SI.isSimple()) 60736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 60836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 60936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VectorLayout Layout; 61036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *FullValue = SI.getValueOperand(); 61136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!getVectorLayout(FullValue->getType(), SI.getAlignment(), Layout)) 61236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 61336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 61436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumElems = Layout.VecTy->getNumElements(); 61536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(SI.getParent(), &SI); 61636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Ptr = scatter(&SI, SI.getPointerOperand()); 61736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scatterer Val = scatter(&SI, FullValue); 61836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 61936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector Stores; 62036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Stores.resize(NumElems); 62136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < NumElems; ++I) { 62236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Align = Layout.getElemAlign(I); 62336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Stores[I] = Builder.CreateAlignedStore(Val[I], Ptr[I], Align); 62436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 62536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines transferMetadata(&SI, Stores); 62636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 62736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 62836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 62936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Delete the instructions that we scalarized. If a full vector result 63036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// is still needed, recreate it using InsertElements. 63136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool Scalarizer::finish() { 63236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Gathered.empty()) 63336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 63436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (GatherList::iterator GMI = Gathered.begin(), GME = Gathered.end(); 63536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GMI != GME; ++GMI) { 63636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Op = GMI->first; 63736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueVector &CV = *GMI->second; 63836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Op->use_empty()) { 63936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The value is still needed, so recreate it using a series of 64036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // InsertElements. 64136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *Ty = Op->getType(); 64236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Res = UndefValue::get(Ty); 64336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = Op->getParent(); 64436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Count = Ty->getVectorNumElements(); 64536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(BB, Op); 64636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<PHINode>(Op)) 64736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); 64836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0; I < Count; ++I) 64936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I), 65036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op->getName() + ".upto" + Twine(I)); 65136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Res->takeName(Op); 65236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op->replaceAllUsesWith(Res); 65336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 65436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Op->eraseFromParent(); 65536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 65636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Gathered.clear(); 65736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scattered.clear(); 65836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 65936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 66036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 66136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesFunctionPass *llvm::createScalarizerPass() { 66236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return new Scalarizer(); 66336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 664