1c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines//===- LoadCombine.cpp - Combine Adjacent Loads ---------------------------===// 2c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// 3c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// The LLVM Compiler Infrastructure 4c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// 5c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// This file is distributed under the University of Illinois Open Source 6c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// License. See LICENSE.TXT for details. 7c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines// 8c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines//===----------------------------------------------------------------------===// 9c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// \file 10c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// This transformation combines adjacent loads. 11c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// 12c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines//===----------------------------------------------------------------------===// 13c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 14c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/Transforms/Scalar.h" 15c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/ADT/DenseMap.h" 16c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/ADT/Statistic.h" 1737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Analysis/AliasAnalysis.h" 1837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Analysis/AliasSetTracker.h" 19cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar#include "llvm/Analysis/GlobalsModRef.h" 20c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/Analysis/TargetFolder.h" 21c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/IR/DataLayout.h" 22c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/IR/Function.h" 23c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/IR/IRBuilder.h" 244c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar#include "llvm/IR/Instructions.h" 254c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar#include "llvm/IR/Module.h" 264c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar#include "llvm/Pass.h" 27c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/Support/Debug.h" 28c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/Support/MathExtras.h" 29c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#include "llvm/Support/raw_ostream.h" 30c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 31c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesusing namespace llvm; 32c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 33c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines#define DEBUG_TYPE "load-combine" 34c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 35c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSTATISTIC(NumLoadsAnalyzed, "Number of loads analyzed for combining"); 36c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesSTATISTIC(NumLoadsCombined, "Number of loads combined"); 37c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 38c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesnamespace { 39c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesstruct PointerOffsetPair { 40c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Value *Pointer; 41c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines uint64_t Offset; 42c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines}; 43c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 44c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesstruct LoadPOPPair { 450c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar LoadPOPPair() = default; 46c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadPOPPair(LoadInst *L, PointerOffsetPair P, unsigned O) 47c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines : Load(L), POP(P), InsertOrder(O) {} 48c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadInst *Load; 49c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines PointerOffsetPair POP; 50c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines /// \brief The new load needs to be created before the first load in IR order. 51c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines unsigned InsertOrder; 52c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines}; 53c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 54c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesclass LoadCombine : public BasicBlockPass { 55c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LLVMContext *C; 5637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AliasAnalysis *AA; 57c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 58c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinespublic: 594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar LoadCombine() : BasicBlockPass(ID), C(nullptr), AA(nullptr) { 60cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar initializeLoadCombinePass(*PassRegistry::getPassRegistry()); 61c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 6237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 6337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines using llvm::Pass::doInitialization; 64c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool doInitialization(Function &) override; 65c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool runOnBasicBlock(BasicBlock &BB) override; 66c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines void getAnalysisUsage(AnalysisUsage &AU) const override; 67c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 68c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines const char *getPassName() const override { return "LoadCombine"; } 69c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines static char ID; 70c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 71c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines typedef IRBuilder<true, TargetFolder> BuilderTy; 72c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 73c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesprivate: 74c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines BuilderTy *Builder; 75c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 76c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines PointerOffsetPair getPointerOffsetPair(LoadInst &); 77c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool combineLoads(DenseMap<const Value *, SmallVector<LoadPOPPair, 8>> &); 78c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool aggregateLoads(SmallVectorImpl<LoadPOPPair> &); 79c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool combineLoads(SmallVectorImpl<LoadPOPPair> &); 80c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines}; 81c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 82c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 83c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesbool LoadCombine::doInitialization(Function &F) { 84c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines DEBUG(dbgs() << "LoadCombine function: " << F.getName() << "\n"); 85c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines C = &F.getContext(); 86c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return true; 87c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 88c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 89c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesPointerOffsetPair LoadCombine::getPointerOffsetPair(LoadInst &LI) { 90c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines PointerOffsetPair POP; 91c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines POP.Pointer = LI.getPointerOperand(); 92c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines POP.Offset = 0; 93c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines while (isa<BitCastInst>(POP.Pointer) || isa<GetElementPtrInst>(POP.Pointer)) { 94c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (auto *GEP = dyn_cast<GetElementPtrInst>(POP.Pointer)) { 954c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar auto &DL = LI.getModule()->getDataLayout(); 964c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned BitWidth = DL.getPointerTypeSizeInBits(GEP->getType()); 97c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines APInt Offset(BitWidth, 0); 984c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (GEP->accumulateConstantOffset(DL, Offset)) 99c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines POP.Offset += Offset.getZExtValue(); 100c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines else 101c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Can't handle GEPs with variable indices. 102c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return POP; 103c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines POP.Pointer = GEP->getPointerOperand(); 104c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } else if (auto *BC = dyn_cast<BitCastInst>(POP.Pointer)) 105c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines POP.Pointer = BC->getOperand(0); 106c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 107c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return POP; 108c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 109c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 110c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesbool LoadCombine::combineLoads( 111c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines DenseMap<const Value *, SmallVector<LoadPOPPair, 8>> &LoadMap) { 112c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool Combined = false; 113c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto &Loads : LoadMap) { 114c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (Loads.second.size() < 2) 115c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 116c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines std::sort(Loads.second.begin(), Loads.second.end(), 117c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines [](const LoadPOPPair &A, const LoadPOPPair &B) { 118c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return A.POP.Offset < B.POP.Offset; 119c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines }); 120c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (aggregateLoads(Loads.second)) 121c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Combined = true; 122c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 123c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Combined; 124c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 125c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 126c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// \brief Try to aggregate loads from a sorted list of loads to be combined. 127c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// 128c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// It is guaranteed that no writes occur between any of the loads. All loads 129c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// have the same base pointer. There are at least two loads. 130c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesbool LoadCombine::aggregateLoads(SmallVectorImpl<LoadPOPPair> &Loads) { 131c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines assert(Loads.size() >= 2 && "Insufficient loads!"); 132c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadInst *BaseLoad = nullptr; 133c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines SmallVector<LoadPOPPair, 8> AggregateLoads; 134c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool Combined = false; 135c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines uint64_t PrevOffset = -1ull; 136c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines uint64_t PrevSize = 0; 137c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto &L : Loads) { 138c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (PrevOffset == -1ull) { 139c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines BaseLoad = L.Load; 140c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines PrevOffset = L.POP.Offset; 1414c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar PrevSize = L.Load->getModule()->getDataLayout().getTypeStoreSize( 1424c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar L.Load->getType()); 143c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines AggregateLoads.push_back(L); 144c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 145c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 146c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (L.Load->getAlignment() > BaseLoad->getAlignment()) 147c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 148c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (L.POP.Offset > PrevOffset + PrevSize) { 149c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // No other load will be combinable 150c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (combineLoads(AggregateLoads)) 151c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Combined = true; 152c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines AggregateLoads.clear(); 153c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines PrevOffset = -1; 154c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 155c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 156c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (L.POP.Offset != PrevOffset + PrevSize) 157c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // This load is offset less than the size of the last load. 158c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // FIXME: We may want to handle this case. 159c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 160c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines PrevOffset = L.POP.Offset; 1614c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar PrevSize = L.Load->getModule()->getDataLayout().getTypeStoreSize( 1624c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar L.Load->getType()); 163c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines AggregateLoads.push_back(L); 164c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 165c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (combineLoads(AggregateLoads)) 166c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Combined = true; 167c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Combined; 168c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 169c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 170c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines/// \brief Given a list of combinable load. Combine the maximum number of them. 171c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesbool LoadCombine::combineLoads(SmallVectorImpl<LoadPOPPair> &Loads) { 172c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Remove loads from the end while the size is not a power of 2. 173c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines unsigned TotalSize = 0; 174c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (const auto &L : Loads) 175c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines TotalSize += L.Load->getType()->getPrimitiveSizeInBits(); 176c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines while (TotalSize != 0 && !isPowerOf2_32(TotalSize)) 177c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines TotalSize -= Loads.pop_back_val().Load->getType()->getPrimitiveSizeInBits(); 178c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (Loads.size() < 2) 179c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return false; 180c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 181c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines DEBUG({ 182c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines dbgs() << "***** Combining Loads ******\n"; 183c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (const auto &L : Loads) { 184c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines dbgs() << L.POP.Offset << ": " << *L.Load << "\n"; 185c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 186c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines }); 187c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 188c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Find first load. This is where we put the new load. 189c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadPOPPair FirstLP; 190c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines FirstLP.InsertOrder = -1u; 191c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (const auto &L : Loads) 192c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (L.InsertOrder < FirstLP.InsertOrder) 193c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines FirstLP = L; 194c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 195c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines unsigned AddressSpace = 196c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines FirstLP.POP.Pointer->getType()->getPointerAddressSpace(); 197c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 198c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Builder->SetInsertPoint(FirstLP.Load); 199c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Value *Ptr = Builder->CreateConstGEP1_64( 200c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Builder->CreatePointerCast(Loads[0].POP.Pointer, 201c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Builder->getInt8PtrTy(AddressSpace)), 202c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Loads[0].POP.Offset); 203c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadInst *NewLoad = new LoadInst( 204c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Builder->CreatePointerCast( 205c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Ptr, PointerType::get(IntegerType::get(Ptr->getContext(), TotalSize), 206c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Ptr->getType()->getPointerAddressSpace())), 207c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Twine(Loads[0].Load->getName()) + ".combined", false, 208c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Loads[0].Load->getAlignment(), FirstLP.Load); 209c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 210c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (const auto &L : Loads) { 211c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Builder->SetInsertPoint(L.Load); 212c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Value *V = Builder->CreateExtractInteger( 2134c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar L.Load->getModule()->getDataLayout(), NewLoad, 2144c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar cast<IntegerType>(L.Load->getType()), 215c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines L.POP.Offset - Loads[0].POP.Offset, "combine.extract"); 216c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines L.Load->replaceAllUsesWith(V); 217c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 218c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 219c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines NumLoadsCombined = NumLoadsCombined + Loads.size(); 220c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return true; 221c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 222c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 223c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesbool LoadCombine::runOnBasicBlock(BasicBlock &BB) { 2244c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (skipOptnoneFunction(BB)) 225c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return false; 226c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 227cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 22837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 2294c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar IRBuilder<true, TargetFolder> TheBuilder( 2304c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar BB.getContext(), TargetFolder(BB.getModule()->getDataLayout())); 231c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Builder = &TheBuilder; 232c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 233c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines DenseMap<const Value *, SmallVector<LoadPOPPair, 8>> LoadMap; 23437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AliasSetTracker AST(*AA); 235c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 236c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool Combined = false; 237c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines unsigned Index = 0; 238c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto &I : BB) { 23937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (I.mayThrow() || (I.mayWriteToMemory() && AST.containsUnknown(&I))) { 240c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (combineLoads(LoadMap)) 241c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Combined = true; 242c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadMap.clear(); 24337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AST.clear(); 244c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 245c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 246c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadInst *LI = dyn_cast<LoadInst>(&I); 247c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (!LI) 248c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 249c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines ++NumLoadsAnalyzed; 250c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (!LI->isSimple() || !LI->getType()->isIntegerTy()) 251c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 252c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines auto POP = getPointerOffsetPair(*LI); 253c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (!POP.Pointer) 254c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines continue; 255c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LoadMap[POP.Pointer].push_back(LoadPOPPair(LI, POP, Index++)); 25637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AST.add(LI); 257c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 258c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (combineLoads(LoadMap)) 259c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Combined = true; 260c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return Combined; 261c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 262c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 263c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hinesvoid LoadCombine::getAnalysisUsage(AnalysisUsage &AU) const { 264c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines AU.setPreservesCFG(); 26537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 266cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar AU.addRequired<AAResultsWrapperPass>(); 267cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga Nainar AU.addPreserved<GlobalsAAWrapperPass>(); 268c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 269c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 270c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hineschar LoadCombine::ID = 0; 271c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 272c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesBasicBlockPass *llvm::createLoadCombinePass() { 273c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return new LoadCombine(); 274c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines} 275c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 27637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesINITIALIZE_PASS_BEGIN(LoadCombine, "load-combine", "Combine Adjacent Loads", 27737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines false, false) 278cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga NainarINITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 279cddc3e03e4ec99c0268c03a126195173e519ed58Pirama Arumuga NainarINITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass) 28037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesINITIALIZE_PASS_END(LoadCombine, "load-combine", "Combine Adjacent Loads", 28137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines false, false) 28237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 283