LoopStrengthReduce.cpp revision 75ae20366fd1b480f4cc38400bb075c43c9f4f7f
12d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// The LLVM Compiler Infrastructure 4eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 9eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 10cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// This transformation analyzes and transforms the induction variables (and 11cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// computations derived from them) into forms suitable for efficient execution 12cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// on the target. 13cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// 14eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// This pass performs a strength reduction on array references inside loops that 15cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// have as one or more of their components the loop induction variable, it 16cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// rewrites expressions to take advantage of scaled-index addressing modes 17cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// available on the target, and it performs a variety of other optimizations 18cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// related to loop induction variables. 19eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 20572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// Terminology note: this code has a lot of handling for "post-increment" or 21572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// "post-inc" users. This is not talking about post-increment addressing modes; 22572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it is instead talking about code like this: 23572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 24572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i = phi [ 0, %entry ], [ %i.next, %latch ] 25572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// ... 26572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i.next = add %i, 1 27572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %c = icmp eq %i.next, %n 28572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 29572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however 30572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it's useful to think about these as the same register, with some uses using 31572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the register before the add and some using // it after. In this 32572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// example, the icmp is a post-increment user, since it uses %i.next, which is 33572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the induction variable after the increment. The other common 34572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// case of post-increment users is users outside the loop. 35572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 36572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: More sophistication in the way Formulae are generated and filtered. 37572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 38572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Handle multiple loops at a time. 39572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 40572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Should TargetLowering::AddrMode::BaseGV be changed to a ConstantExpr 41572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// instead of a GlobalValue? 42572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 43572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When truncation is free, truncate ICmp users' operands to make it a 44572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// smaller encoding (on x86 at least). 45572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 46572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When a negated register is used by an add (such as in a list of 47572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// multiple base registers, or as the increment expression in an addrec), 48572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// we may not actually need both reg and (-1 * reg) in registers; the 49572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// negation can be implemented by using a sub instead of an add. The 50572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// lack of support for taking this into consideration when making 51572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// register pressure decisions is partly worked around by the "Special" 52572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// use kind. 53572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 54eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 55eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 56be3e5212e23edc9210f774fac18d801de252e906Chris Lattner#define DEBUG_TYPE "loop-reduce" 57eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Scalar.h" 58eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Constants.h" 59eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Instructions.h" 60e5b01bea7b9b7dce7c24484d2d915b0c118d4d07Dan Gohman#include "llvm/IntrinsicInst.h" 612f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/DerivedTypes.h" 6281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman#include "llvm/Analysis/IVUsers.h" 63572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/Analysis/Dominators.h" 640f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel#include "llvm/Analysis/LoopPass.h" 65169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Analysis/ScalarEvolutionExpander.h" 669fc5cdf77c812aaa80419036de27576d45894d0dChris Lattner#include "llvm/Assembly/Writer.h" 67e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner#include "llvm/Transforms/Utils/BasicBlockUtils.h" 68eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Utils/Local.h" 69572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SmallBitVector.h" 70572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SetVector.h" 71572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/DenseSet.h" 72169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/Debug.h" 7380ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick#include "llvm/Support/CommandLine.h" 74afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman#include "llvm/Support/ValueHandle.h" 75460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar#include "llvm/Support/raw_ostream.h" 76d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng#include "llvm/Target/TargetLowering.h" 77cfb1d4235fe3291028341e6acf4139723b4749e3Jeff Cohen#include <algorithm> 78eaa13851a7fe604363577350c5cf65c257c4d41aNate Begemanusing namespace llvm; 79eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 800861f5793a1834f02b522fb86fb037cd592c134fBenjamin Kramerstatic cl::opt<bool> EnableNested( 810c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick "enable-lsr-nested", cl::Hidden, cl::desc("Enable LSR on nested loops")); 820c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick 830861f5793a1834f02b522fb86fb037cd592c134fBenjamin Kramerstatic cl::opt<bool> EnableRetry( 840861f5793a1834f02b522fb86fb037cd592c134fBenjamin Kramer "enable-lsr-retry", cl::Hidden, cl::desc("Enable LSR retry")); 85a02bfced06b4cc700e50bc497cc42667653f091aAndrew Trick 86a02bfced06b4cc700e50bc497cc42667653f091aAndrew Trick// Temporary flag to cleanup congruent phis after LSR phi expansion. 87a02bfced06b4cc700e50bc497cc42667653f091aAndrew Trick// It's currently disabled until we can determine whether it's truly useful or 88a02bfced06b4cc700e50bc497cc42667653f091aAndrew Trick// not. The flag should be removed after the v3.0 release. 8924f670f1bae0281b08276229041dfc952a810dbfAndrew Trick// This is now needed for ivchains. 900861f5793a1834f02b522fb86fb037cd592c134fBenjamin Kramerstatic cl::opt<bool> EnablePhiElim( 9124f670f1bae0281b08276229041dfc952a810dbfAndrew Trick "enable-lsr-phielim", cl::Hidden, cl::init(true), 9224f670f1bae0281b08276229041dfc952a810dbfAndrew Trick cl::desc("Enable LSR phi elimination")); 9380ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 9422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick#ifndef NDEBUG 9522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick// Stress test IV chain generation. 9622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic cl::opt<bool> StressIVChain( 9722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick "stress-ivchain", cl::Hidden, cl::init(false), 9822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick cl::desc("Stress test LSR IV chains")); 9922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick#else 10022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic bool StressIVChain = false; 10122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick#endif 10222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegSortData - This class holds data which is used to order reuse candidates. 106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegSortData { 107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UsedByIndices - This represents the set of LSRUse indices which reference 109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// a particular register. 110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallBitVector UsedByIndices; 111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData() {} 113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::print(raw_ostream &OS) const { 121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "[NumUses=" << UsedByIndices.count() << ']'; 122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 123dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::dump() const { 125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 127dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 1287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmannamespace { 1297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegUseTracker - Map register candidates to information about how they are 131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// used. 132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegUseTracker { 133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, RegSortData> RegUsesTy; 134d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 13590bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy RegUsesMap; 136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 16> RegSequence; 137d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegister(const SCEV *Reg, size_t LUIdx); 140b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void DropRegister(const SCEV *Reg, size_t LUIdx); 141c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void SwapAndDropUse(size_t LUIdx, size_t LastLUIdx); 142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const; 144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &getUsedByIndices(const SCEV *Reg) const; 146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 147572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void clear(); 148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::iterator iterator; 150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator; 151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator begin() { return RegSequence.begin(); } 152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator end() { return RegSequence.end(); } 153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator begin() const { return RegSequence.begin(); } 154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator end() const { return RegSequence.end(); } 155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 157572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 160572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) { 161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegUsesTy::iterator, bool> Pair = 16290bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.insert(std::make_pair(Reg, RegSortData())); 163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData &RSD = Pair.first->second; 164572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.push_back(Reg); 166572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1)); 167572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.set(LUIdx); 168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 170b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid 171b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan GohmanRegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) { 172b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUsesTy::iterator It = RegUsesMap.find(Reg); 173b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(It != RegUsesMap.end()); 174b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegSortData &RSD = It->second; 175b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(RSD.UsedByIndices.size() > LUIdx); 176b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RSD.UsedByIndices.reset(LUIdx); 177b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 178b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 179a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanvoid 180c6897706b7c3796ac24535c9ea1449c0411f8c7aDan GohmanRegUseTracker::SwapAndDropUse(size_t LUIdx, size_t LastLUIdx) { 181c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman assert(LUIdx <= LastLUIdx); 182c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 183c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. The data structure is not optimized for this purpose; 184c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // we must iterate through it and update each of the bit vectors. 185a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (RegUsesTy::iterator I = RegUsesMap.begin(), E = RegUsesMap.end(); 186c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman I != E; ++I) { 187c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman SmallBitVector &UsedByIndices = I->second.UsedByIndices; 188c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman if (LUIdx < UsedByIndices.size()) 189c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices[LUIdx] = 190c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : 0; 191c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx)); 192c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman } 193a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 194a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 196572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { 19746fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 19846fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman if (I == RegUsesMap.end()) 19946fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman return false; 20046fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman const SmallBitVector &UsedByIndices = I->second.UsedByIndices; 201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int i = UsedByIndices.find_first(); 202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (i == -1) return false; 203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((size_t)i != LUIdx) return true; 204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return UsedByIndices.find_next(i) != -1; 205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanconst SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { 20890bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 20990bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman assert(I != RegUsesMap.end() && "Unknown register!"); 210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return I->second.UsedByIndices; 211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegUseTracker::clear() { 21490bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.clear(); 215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.clear(); 216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Formula - This class holds information that describes a formula for 221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// computing satisfying a use. It may include broken-out immediates and scaled 222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// registers. 223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct Formula { 224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AM - This is used to represent complex addressing, as well as other kinds 225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// of interesting uses. 226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// BaseRegs - The list of "base" registers for this use. When this is 229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// non-empty, AM.HasBaseReg should be set to true. 230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> BaseRegs; 231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// ScaledReg - The 'scaled' register for this use. This should be non-null 233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// when AM.Scale is not zero. 234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledReg; 235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 236cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// UnfoldedOffset - An additional constant offset which added near the 237cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// use. This requires a temporary register, but the offset itself can 238cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// live in an add immediate field rather than a register. 239cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman int64_t UnfoldedOffset; 240cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 241cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Formula() : ScaledReg(0), UnfoldedOffset(0) {} 242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 243dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE); 244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned getNumRegs() const; 246db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *getType() const; 247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2485ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman void DeleteBaseReg(const SCEV *&S); 2495ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool referencesReg(const SCEV *S) const; 251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool hasRegsUsedByUsesOtherThan(size_t LUIdx, 252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const; 253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2603f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// DoInitialMatch - Recursion helper for InitialMatch. 261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic void DoInitialMatch(const SCEV *S, Loop *L, 262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Good, 263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Bad, 264dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman ScalarEvolution &SE) { 265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect expressions which properly dominate the loop header. 266dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(S, L->getHeader())) { 267572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(S); 268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 270d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at add operands. 272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) 275dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(*I, L, Good, Bad, SE); 276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 278169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at addrec operands. 280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) 281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!AR->getStart()->isZero()) { 282dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(AR->getStart(), L, Good, Bad, SE); 283deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), 284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE), 2853228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags() 2863228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick AR->getLoop(), SCEV::FlagAnyWrap), 287dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman L, Good, Bad, SE); 288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 2902d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle a multiplication by -1 (negation) if it didn't fold. 292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) 293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mul->getOperand(0)->isAllOnesValue()) { 294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end()); 295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewMul = SE.getMulExpr(Ops); 296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyGood; 298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyBad; 299dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(NewMul, L, MyGood, MyBad, SE); 300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( 301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getEffectiveSCEVType(NewMul->getType()))); 302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(), 303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyGood.end(); I != E; ++I) 304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(SE.getMulExpr(NegOne, *I)); 305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(), 306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyBad.end(); I != E; ++I) 307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(SE.getMulExpr(NegOne, *I)); 308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 3097979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 310eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we can't do anything interesting. Just stuff the whole thing into a 312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // register and hope for the best. 313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(S); 314a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 315844731a7f1909f55935e3514c9e713a62d67662eDan Gohman 316572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InitialMatch - Incorporate loop-variant parts of S into this Formula, 317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// attempting to keep all loop-invariant and loop-computable values in a 318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// single base register. 319dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohmanvoid Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { 320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Good; 321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Bad; 322dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(S, L, Good, Bad, SE); 323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Good.empty()) { 324e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Good); 325e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 326e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Bad.empty()) { 330e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Bad); 331e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 332e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 336eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getNumRegs - Return the total number of register operands used by this 338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// formula. This does not include register uses implied by non-constant 339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrec strides. 340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanunsigned Formula::getNumRegs() const { 341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !!ScaledReg + BaseRegs.size(); 342a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 34356a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getType - Return the type of this formula, if it has one, or null 345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// otherwise. This type is meaningless except for the bit size. 346db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris LattnerType *Formula::getType() const { 347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !BaseRegs.empty() ? BaseRegs.front()->getType() : 348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledReg ? ScaledReg->getType() : 349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV ? AM.BaseGV->getType() : 350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 0; 351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 35256a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 3535ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteBaseReg - Delete the given base reg from the BaseRegs list. 3545ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohmanvoid Formula::DeleteBaseReg(const SCEV *&S) { 3555ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&S != &BaseRegs.back()) 3565ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(S, BaseRegs.back()); 3575ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman BaseRegs.pop_back(); 3585ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 3595ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// referencesReg - Test if this formula references the given register. 361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::referencesReg(const SCEV *S) const { 362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return S == ScaledReg || 363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(BaseRegs.begin(), BaseRegs.end(), S) != BaseRegs.end(); 364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 365eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// hasRegsUsedByUsesOtherThan - Test whether this formula uses registers 367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// which are used by uses other than the use with the given index. 368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx, 369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const { 370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx)) 372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) 375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(*I, LUIdx)) 376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::print(raw_ostream &OS) const { 381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) { 383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 384572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, AM.BaseGV, /*PrintType=*/false); 385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.BaseOffs; 389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) { 392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "reg(" << **I << ')'; 394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 395c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (AM.HasBaseReg && BaseRegs.empty()) { 396c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 397c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: HasBaseReg**"; 398c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } else if (!AM.HasBaseReg && !BaseRegs.empty()) { 399c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 400c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: !HasBaseReg**"; 401c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } 402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0) { 403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.Scale << "*reg("; 405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 406572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *ScaledReg; 407572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "<unknown>"; 409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ')'; 410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 411cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (UnfoldedOffset != 0) { 412cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (!First) OS << " + "; else First = false; 413cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman OS << "imm(" << UnfoldedOffset << ')'; 414cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 415572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 416572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::dump() const { 418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 421aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddRecSExtable - Return true if the given addrec can be sign-extended 422aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 423aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { 424db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 425ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1); 426aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy)); 427aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 428aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 429aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddSExtable - Return true if the given add can be sign-extended 430aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 431aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) { 432db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 433ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1); 434aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy)); 435aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 436aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 437473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman/// isMulSExtable - Return true if the given mul can be sign-extended 438aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 439473e63512acb3751da7dffbb10e0452cb581b265Dan Gohmanstatic bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) { 440db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 441473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman IntegerType::get(SE.getContext(), 442473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman SE.getTypeSizeInBits(M->getType()) * M->getNumOperands()); 443473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy)); 444aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 445aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 446f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// getExactSDiv - Return an expression for LHS /s RHS, if it can be determined 447f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// and if the remainder is known to be zero, or null otherwise. If 448f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified 449f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// to Y, ignoring that the multiplication may overflow, which is useful when 450f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// the result will be used in a context where the most significant bits are 451f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// ignored. 452f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohmanstatic const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, 453f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman ScalarEvolution &SE, 454f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman bool IgnoreSignificantBits = false) { 455572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle the trivial case, which works for any SCEV type. 456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LHS == RHS) 457deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman return SE.getConstant(LHS->getType(), 1); 458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 459d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle a few RHS special cases. 460d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS); 461d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RC) { 462d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 463d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do 464d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // some folding. 465d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA.isAllOnesValue()) 466d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getMulExpr(LHS, RC); 467d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s 1 as x. 468d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA == 1) 469d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return LHS; 470d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman } 471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 472572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a division of a constant by a constant. 473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) { 474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!RC) 475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 476d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &LA = C->getValue()->getValue(); 477d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 478d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (LA.srem(RA) != 0) 479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 480d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getConstant(LA.sdiv(RA)); 481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 483aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over addrec operands, if the addrec doesn't overflow. 484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) { 485aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddRecSExtable(AR, SE)) { 486f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, 487f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 488aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Step) return 0; 489694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, 490694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman IgnoreSignificantBits); 491694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman if (!Start) return 0; 4923228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FlagNW is independent of the start value, step direction, and is 4933228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // preserved with smaller magnitude steps. 4943228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 4953228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); 496aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 4972ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 500aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over add operands, if the add doesn't overflow. 501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) { 502aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddSExtable(Add, SE)) { 503aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman SmallVector<const SCEV *, 8> Ops; 504aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 505aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman I != E; ++I) { 506f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Op = getExactSDiv(*I, RHS, SE, 507f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 508aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Op) return 0; 509aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman Ops.push_back(Op); 510aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 511aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return SE.getAddExpr(Ops); 512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5132ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a multiply operand that we can pull RHS out of. 5172ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) { 518aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) { 519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Found = false; 521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVMulExpr::op_iterator I = Mul->op_begin(), E = Mul->op_end(); 522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 5234766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman const SCEV *S = *I; 524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Found) 5254766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman if (const SCEV *Q = getExactSDiv(S, RHS, SE, 526f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits)) { 5274766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman S = Q; 528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Found = true; 529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5304766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman Ops.push_back(S); 531a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Found ? SE.getMulExpr(Ops) : 0; 533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5342ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 5352ea09e05466613a22e1211f52c30cd01af563983Dan Gohman } 536a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise we don't know. 538572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractImmediate - If S involves the addition of a constant integer value, 542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that integer value, and mutate S to point to a new SCEV with that 543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { 545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { 546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getValue()->getValue().getMinSignedBits() <= 64) { 547deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(C->getType(), 0); 548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return C->getValue()->getSExtValue(); 549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 553e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 554e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 559e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 5603228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5613228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5623228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 563572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 56421e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman } 565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 566572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractSymbol - If S involves the addition of a GlobalValue address, 569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that symbol, and mutate S to point to a new SCEV with that 570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { 572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 573572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) { 574deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(GV->getType(), 0); 575572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return GV; 576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 578572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 579572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.back(), SE); 580e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 581e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.front(), SE); 586e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 5873228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5883228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5893228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 590572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 591572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 593169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 594169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 5957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// isAddressUse - Returns true if the specified instruction is using the 5967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// specified value as an address. 5977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmanstatic bool isAddressUse(Instruction *Inst, Value *OperandVal) { 5987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman bool isAddress = isa<LoadInst>(Inst); 5997979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 6007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (SI->getOperand(1) == OperandVal) 6017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 6027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 6037979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 6047979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 6057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 6067979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 6077979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::prefetch: 6087979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 6097979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 6107979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 6117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 612ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif if (II->getArgOperand(0) == OperandVal) 6137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 6147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 6157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6167979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6177979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return isAddress; 618169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 619169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 6207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// getAccessType - Return the type of the memory being accessed. 621db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattnerstatic Type *getAccessType(const Instruction *Inst) { 622db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = Inst->getType(); 6237979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) 6247979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AccessTy = SI->getOperand(0)->getType(); 6257979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 6267979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 6277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 6287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 6297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 6307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 6317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 6327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 6337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 634ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif AccessTy = II->getArgOperand(0)->getType(); 6357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 6367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // All pointers have the same requirements, so canonicalize them to an 640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // arbitrary pointer type to minimize variation. 641db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner if (PointerType *PTy = dyn_cast<PointerType>(AccessTy)) 642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = PointerType::get(IntegerType::get(PTy->getContext(), 1), 643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PTy->getAddressSpace()); 644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 6457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return AccessTy; 646a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 64781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 6488a5d792944582de8e63e96440dbd2cde754351adAndrew Trick/// isExistingPhi - Return true if this AddRec is already a phi in its loop. 6498a5d792944582de8e63e96440dbd2cde754351adAndrew Trickstatic bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { 6508a5d792944582de8e63e96440dbd2cde754351adAndrew Trick for (BasicBlock::iterator I = AR->getLoop()->getHeader()->begin(); 6518a5d792944582de8e63e96440dbd2cde754351adAndrew Trick PHINode *PN = dyn_cast<PHINode>(I); ++I) { 6528a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (SE.isSCEVable(PN->getType()) && 6538a5d792944582de8e63e96440dbd2cde754351adAndrew Trick (SE.getEffectiveSCEVType(PN->getType()) == 6548a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SE.getEffectiveSCEVType(AR->getType())) && 6558a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SE.getSCEV(PN) == AR) 6568a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return true; 6578a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 6588a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return false; 6598a5d792944582de8e63e96440dbd2cde754351adAndrew Trick} 6608a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 66164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// Check if expanding this expression is likely to incur significant cost. This 66264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// is tricky because SCEV doesn't track which expressions are actually computed 66364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// by the current IR. 66464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// 66564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// We currently allow expansion of IV increments that involve adds, 66664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// multiplication by constants, and AddRecs from existing phis. 66764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// 66864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// TODO: Allow UDivExpr if we can find an existing IV increment that is an 66964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// obvious multiple of the UDivExpr. 67064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trickstatic bool isHighCostExpansion(const SCEV *S, 67164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick SmallPtrSet<const SCEV*, 8> &Processed, 67264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ScalarEvolution &SE) { 67364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Zero/One operand expressions 67464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick switch (S->getSCEVType()) { 67564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scUnknown: 67664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scConstant: 67764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 67864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scTruncate: 67964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(cast<SCEVTruncateExpr>(S)->getOperand(), 68064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Processed, SE); 68164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scZeroExtend: 68264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(cast<SCEVZeroExtendExpr>(S)->getOperand(), 68364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Processed, SE); 68464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scSignExtend: 68564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(cast<SCEVSignExtendExpr>(S)->getOperand(), 68664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Processed, SE); 68764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 68864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 68964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (!Processed.insert(S)) 69064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 69164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 69264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 69364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 69464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick I != E; ++I) { 69564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isHighCostExpansion(*I, Processed, SE)) 69664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return true; 69764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 69864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 69964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 70064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 70164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 70264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (Mul->getNumOperands() == 2) { 70364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Multiplication by a constant is ok 70464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isa<SCEVConstant>(Mul->getOperand(0))) 70564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(Mul->getOperand(1), Processed, SE); 70664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 70764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // If we have the value of one operand, check if an existing 70864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // multiplication already generates this expression. 70964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) { 71064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Value *UVal = U->getValue(); 71164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (Value::use_iterator UI = UVal->use_begin(), UE = UVal->use_end(); 71264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick UI != UE; ++UI) { 71364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Instruction *User = cast<Instruction>(*UI); 71464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (User->getOpcode() == Instruction::Mul 71564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick && SE.isSCEVable(User->getType())) { 71664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return SE.getSCEV(User) == Mul; 71764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 71864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 71964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 72364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 72464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isExistingPhi(AR, SE)) 72564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 72664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 72864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Fow now, consider any other type of expression (div/mul/min/max) high cost. 72964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return true; 73064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick} 73164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DeleteTriviallyDeadInstructions - If any of the instructions is the 733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// specified set are trivially dead, delete them and see if this makes any of 734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// their operands subsequently dead. 735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool 736572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanDeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) { 737572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 7382f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!DeadInsts.empty()) { 740f097b59e0e2231d431a6c660c65fa5cae22d9a44Gabor Greif Instruction *I = dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()); 74181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (I == 0 || !isInstructionTriviallyDead(I)) 743572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 744221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 745572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) 746572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *U = dyn_cast<Instruction>(*OI)) { 747572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *OI = 0; 748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (U->use_empty()) 749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(U); 750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 751221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I->eraseFromParent(); 753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 7552f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Changed; 7577979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 7582f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 7602f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Cost - This class is used to measure and compare candidate formulae. 762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass Cost { 763572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TODO: Some of these could be merged. Also, a lexical ordering 764572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// isn't always optimal. 765572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumRegs; 766572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned AddRecCost; 767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumIVMuls; 768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumBaseAdds; 769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned ImmCost; 770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned SetupCost; 771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 773572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost() 774572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : NumRegs(0), AddRecCost(0), NumIVMuls(0), NumBaseAdds(0), ImmCost(0), 775572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost(0) {} 776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool operator<(const Cost &Other) const; 778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 779572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Loose(); 780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 7817d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick#ifndef NDEBUG 7827d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick // Once any of the metrics loses, they must all remain losers. 7837d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick bool isValid() { 7847d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return ((NumRegs | AddRecCost | NumIVMuls | NumBaseAdds 7857d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick | ImmCost | SetupCost) != ~0u) 7867d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick || ((NumRegs & AddRecCost & NumIVMuls & NumBaseAdds 7877d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick & ImmCost & SetupCost) == ~0u); 7887d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 7897d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick#endif 7907d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick 7917d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick bool isLoser() { 7927d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick assert(isValid() && "invalid cost"); 7937d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return NumRegs == ~0u; 7947d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 7957d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick 796572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateFormula(const Formula &F, 797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 8018a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 8028a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs = 0); 803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 808572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateRegister(const SCEV *Reg, 809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 8129214b82c5446791b280821bbd892dca633130f80Dan Gohman void RatePrimaryRegister(const SCEV *Reg, 8139214b82c5446791b280821bbd892dca633130f80Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 8149214b82c5446791b280821bbd892dca633130f80Dan Gohman const Loop *L, 8158a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 8168a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs); 817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 8180e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 8207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 821572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RateRegister - Tally up interesting quantities from the given register. 822572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateRegister(const SCEV *Reg, 823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 8269214b82c5446791b280821bbd892dca633130f80Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) { 8279214b82c5446791b280821bbd892dca633130f80Dan Gohman if (AR->getLoop() == L) 8289214b82c5446791b280821bbd892dca633130f80Dan Gohman AddRecCost += 1; /// TODO: This should be a function of the stride. 8299214b82c5446791b280821bbd892dca633130f80Dan Gohman 8300c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // If this is an addrec for another loop, don't second-guess its addrec phi 8310c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // nodes. LSR isn't currently smart enough to reason about more than one 8320c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // loop at a time. LSR has either already run on inner loops, will not run 8330c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // on other loops, and cannot be expected to change sibling loops. If the 8340c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // AddRec exists, consider it's register free and leave it alone. Otherwise, 8350c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // do not consider this formula at all. 8360c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick else if (!EnableNested || L->contains(AR->getLoop()) || 8379214b82c5446791b280821bbd892dca633130f80Dan Gohman (!AR->getLoop()->contains(L) && 8389214b82c5446791b280821bbd892dca633130f80Dan Gohman DT.dominates(L->getHeader(), AR->getLoop()->getHeader()))) { 8398a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (isExistingPhi(AR, SE)) 8408a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return; 8418a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 8428a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // For !EnableNested, never rewrite IVs in other loops. 8430c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick if (!EnableNested) { 8440c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick Loose(); 8450c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick return; 8460c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick } 8479214b82c5446791b280821bbd892dca633130f80Dan Gohman // If this isn't one of the addrecs that the loop already has, it 8489214b82c5446791b280821bbd892dca633130f80Dan Gohman // would require a costly new phi and add. TODO: This isn't 8499214b82c5446791b280821bbd892dca633130f80Dan Gohman // precisely modeled right now. 8509214b82c5446791b280821bbd892dca633130f80Dan Gohman ++NumBaseAdds; 8517d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (!Regs.count(AR->getStart())) { 852572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RateRegister(AR->getStart(), Regs, L, SE, DT); 8537d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 8547d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 8557d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 8569214b82c5446791b280821bbd892dca633130f80Dan Gohman } 8577979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 8589214b82c5446791b280821bbd892dca633130f80Dan Gohman // Add the step value register, if it needs one. 8599214b82c5446791b280821bbd892dca633130f80Dan Gohman // TODO: The non-affine case isn't precisely modeled here. 86025b689e067697d3b49ae123120703fada030350fAndrew Trick if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) { 86125b689e067697d3b49ae123120703fada030350fAndrew Trick if (!Regs.count(AR->getOperand(1))) { 862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RateRegister(AR->getOperand(1), Regs, L, SE, DT); 86325b689e067697d3b49ae123120703fada030350fAndrew Trick if (isLoser()) 86425b689e067697d3b49ae123120703fada030350fAndrew Trick return; 86525b689e067697d3b49ae123120703fada030350fAndrew Trick } 86625b689e067697d3b49ae123120703fada030350fAndrew Trick } 867a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 8689214b82c5446791b280821bbd892dca633130f80Dan Gohman ++NumRegs; 8699214b82c5446791b280821bbd892dca633130f80Dan Gohman 8709214b82c5446791b280821bbd892dca633130f80Dan Gohman // Rough heuristic; favor registers which don't require extra setup 8719214b82c5446791b280821bbd892dca633130f80Dan Gohman // instructions in the preheader. 8729214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!isa<SCEVUnknown>(Reg) && 8739214b82c5446791b280821bbd892dca633130f80Dan Gohman !isa<SCEVConstant>(Reg) && 8749214b82c5446791b280821bbd892dca633130f80Dan Gohman !(isa<SCEVAddRecExpr>(Reg) && 8759214b82c5446791b280821bbd892dca633130f80Dan Gohman (isa<SCEVUnknown>(cast<SCEVAddRecExpr>(Reg)->getStart()) || 8769214b82c5446791b280821bbd892dca633130f80Dan Gohman isa<SCEVConstant>(cast<SCEVAddRecExpr>(Reg)->getStart())))) 8779214b82c5446791b280821bbd892dca633130f80Dan Gohman ++SetupCost; 87823c3fde39aa35334e74c26273a8973a872871e34Dan Gohman 87923c3fde39aa35334e74c26273a8973a872871e34Dan Gohman NumIVMuls += isa<SCEVMulExpr>(Reg) && 88017ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman SE.hasComputableLoopEvolution(Reg, L); 8819214b82c5446791b280821bbd892dca633130f80Dan Gohman} 8829214b82c5446791b280821bbd892dca633130f80Dan Gohman 8839214b82c5446791b280821bbd892dca633130f80Dan Gohman/// RatePrimaryRegister - Record this register in the set. If we haven't seen it 8848a5d792944582de8e63e96440dbd2cde754351adAndrew Trick/// before, rate it. Optional LoserRegs provides a way to declare any formula 8858a5d792944582de8e63e96440dbd2cde754351adAndrew Trick/// that refers to one of those regs an instant loser. 8869214b82c5446791b280821bbd892dca633130f80Dan Gohmanvoid Cost::RatePrimaryRegister(const SCEV *Reg, 8877fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 8887fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman const Loop *L, 8898a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 8908a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs) { 8918a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (LoserRegs && LoserRegs->count(Reg)) { 8928a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Loose(); 8938a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return; 8948a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 8958a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (Regs.insert(Reg)) { 8969214b82c5446791b280821bbd892dca633130f80Dan Gohman RateRegister(Reg, Regs, L, SE, DT); 8978a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (isLoser()) 8988a5d792944582de8e63e96440dbd2cde754351adAndrew Trick LoserRegs->insert(Reg); 8998a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 9007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 9017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateFormula(const Formula &F, 903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 905572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 906572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 9078a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 9088a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs) { 909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the registers. 910572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEV *ScaledReg = F.ScaledReg) { 911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(ScaledReg)) { 912572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 9147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 9158a5d792944582de8e63e96440dbd2cde754351adAndrew Trick RatePrimaryRegister(ScaledReg, Regs, L, SE, DT, LoserRegs); 9167d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 9177d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 9183821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner } 919572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 922572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(BaseReg)) { 923572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 924572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 9257979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 9268a5d792944582de8e63e96440dbd2cde754351adAndrew Trick RatePrimaryRegister(BaseReg, Regs, L, SE, DT, LoserRegs); 9277d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 9287d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 931cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Determine how many (unfolded) adds we'll need inside the loop. 932cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman size_t NumBaseParts = F.BaseRegs.size() + (F.UnfoldedOffset != 0); 933cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (NumBaseParts > 1) 934cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NumBaseAdds += NumBaseParts - 1; 935572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the non-zero immediates. 937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)*I + F.AM.BaseOffs; 940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.BaseGV) 941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += 64; // Handle symbolic values conservatively. 942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This should probably be the pointer size. 943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Offset != 0) 944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += APInt(64, Offset, true).getMinSignedBits(); 945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 9467d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick assert(isValid() && "invalid cost"); 947169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 948169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 9497a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner/// Loose - Set this cost to a losing value. 950572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::Loose() { 951572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumRegs = ~0u; 952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AddRecCost = ~0u; 953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumIVMuls = ~0u; 954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumBaseAdds = ~0u; 955572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost = ~0u; 956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost = ~0u; 957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 95856a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 959572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// operator< - Choose the lower cost. 960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Cost::operator<(const Cost &Other) const { 961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumRegs != Other.NumRegs) 962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumRegs < Other.NumRegs; 963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != Other.AddRecCost) 964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return AddRecCost < Other.AddRecCost; 965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != Other.NumIVMuls) 966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumIVMuls < Other.NumIVMuls; 967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != Other.NumBaseAdds) 968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumBaseAdds < Other.NumBaseAdds; 969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != Other.ImmCost) 970572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return ImmCost < Other.ImmCost; 971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != Other.SetupCost) 972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return SetupCost < Other.SetupCost; 973572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 9757979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 976572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::print(raw_ostream &OS) const { 977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << NumRegs << " reg" << (NumRegs == 1 ? "" : "s"); 978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != 0) 979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", with addrec cost " << AddRecCost; 980572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != 0) 981572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumIVMuls << " IV mul" << (NumIVMuls == 1 ? "" : "s"); 982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != 0) 983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumBaseAdds << " base add" 984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << (NumBaseAdds == 1 ? "" : "s"); 985572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != 0) 986572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << ImmCost << " imm cost"; 987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != 0) 988572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << SetupCost << " setup cost"; 989572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 99044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::dump() const { 992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 99344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner} 99444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 99644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 997572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRFixup - An operand value in an instruction which is to be replaced 998572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// with some equivalent, possibly strength-reduced, replacement. 999572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct LSRFixup { 1000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UserInst - The instruction which will be updated. 1001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *UserInst; 100256a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1003572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// OperandValToReplace - The operand of the instruction which will 1004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// be replaced. The operand may be used more than once; every instance 1005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// will be replaced. 1006572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OperandValToReplace; 100756a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1008448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman /// PostIncLoops - If this user is to use the post-incremented value of an 1009572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable, this variable is non-null and holds the loop 1010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// associated with the induction variable. 1011448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet PostIncLoops; 101226d91f16464db56283087176a73981048331dd2dChris Lattner 1013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LUIdx - The index of the LSRUse describing the expression which 1014572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// this fixup needs, minus an offset (below). 1015572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 101626d91f16464db56283087176a73981048331dd2dChris Lattner 1017572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Offset - A constant offset to be added to the LSRUse expression. 1018572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// This allows multiple fixups to share the same LSRUse with different 1019572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// offsets, for example in an unrolled loop. 1020572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset; 1021572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1022448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman bool isUseFullyOutsideLoop(const Loop *L) const; 1023448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 1024572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup(); 1025572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1032572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRFixup::LSRFixup() 1033ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman : UserInst(0), OperandValToReplace(0), LUIdx(~size_t(0)), Offset(0) {} 1034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1035448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// isUseFullyOutsideLoop - Test whether this fixup always uses its 1036448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// value outside of the given loop. 1037448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohmanbool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const { 1038448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // PHI nodes use their value in their incoming blocks. 1039448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) { 1040448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 1041448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (PN->getIncomingValue(i) == OperandValToReplace && 1042448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman L->contains(PN->getIncomingBlock(i))) 1043448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return false; 1044448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return true; 1045448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 1046448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 1047448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return !L->contains(UserInst); 1048448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman} 1049448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 1050572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::print(raw_ostream &OS) const { 1051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "UserInst="; 1052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Store is common and interesting enough to be worth special-casing. 1053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) { 1054572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "store "; 1055572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, Store->getOperand(0), /*PrintType=*/false); 1056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (UserInst->getType()->isVoidTy()) 1057572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << UserInst->getOpcodeName(); 1058572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1059572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, UserInst, /*PrintType=*/false); 1060572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1061572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", OperandValToReplace="; 1062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, OperandValToReplace, /*PrintType=*/false); 1063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1064448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (PostIncLoopSet::const_iterator I = PostIncLoops.begin(), 1065448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman E = PostIncLoops.end(); I != E; ++I) { 1066572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", PostIncLoop="; 1067448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman WriteAsOperand(OS, (*I)->getHeader(), /*PrintType=*/false); 1068934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1069a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1070572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LUIdx != ~size_t(0)) 1071572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", LUIdx=" << LUIdx; 1072572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1073572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) 1074572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offset=" << Offset; 10757979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 10767979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1077572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::dump() const { 1078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 10797979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 10807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 1082a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1083572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// UniquifierDenseMapInfo - A DenseMapInfo implementation for holding 1084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DenseMaps and DenseSets of sorted SmallVectors of const SCEV*. 1085572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct UniquifierDenseMapInfo { 1086572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getEmptyKey() { 1087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 1088572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-1)); 1089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 1090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1091934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getTombstoneKey() { 1093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 1094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-2)); 1095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 1096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1097572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static unsigned getHashValue(const SmallVector<const SCEV *, 2> &V) { 1099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Result = 0; 1100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = V.begin(), 1101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = V.end(); I != E; ++I) 1102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Result ^= DenseMapInfo<const SCEV *>::getHashValue(*I); 11037979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Result; 1104a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1105a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static bool isEqual(const SmallVector<const SCEV *, 2> &LHS, 1107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVector<const SCEV *, 2> &RHS) { 1108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return LHS == RHS; 1109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse - This class holds the state that LSR keeps for each use in 1113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// IVUsers, as well as uses invented by LSR itself. It includes information 1114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// about what kinds of things can be folded into the user, information about 1115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the user itself, and information about how the use may be satisfied. 1116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Represent multiple users of the same expression in common? 1117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRUse { 1118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<SmallVector<const SCEV *, 2>, UniquifierDenseMapInfo> Uniquifier; 1119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 1121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// KindType - An enum for a kind of use, indicating what types of 1122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// scaled and immediate operands it might support. 1123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman enum KindType { 1124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Basic, ///< A normal use, with no folding. 1125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Special, ///< A special case of basic, allowing -1 scales. 1126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Address, ///< An address use; folding according to TargetLowering 1127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpZero ///< An equality icmp with both operands folded into one. 1128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Add a generic icmp too? 1129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 11307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman KindType Kind; 1132db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy; 11337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<int64_t, 8> Offsets; 1135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset; 1136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MaxOffset; 1137203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 1138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AllFixupsOutsideLoop - This records whether all of the fixups using this 1139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LSRUse are outside of the loop, in which case some special-case heuristics 1140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// may be used. 1141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool AllFixupsOutsideLoop; 11427979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1143a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// WidestFixupType - This records the widest use type for any fixup using 1144a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// this LSRUse. FindUseWithSimilarFormula can't consider uses with different 1145a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// max fixup widths to be equivalent, because the narrower one may be relying 1146a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// on the implicit truncation to truncate away bogus bits. 1147db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WidestFixupType; 1148a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 1149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Formulae - A list of ways to build a value that can satisfy this user. 1150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// After the list is populated, one of these is selected heuristically and 1151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// used to formulate a replacement for OperandValToReplace in UserInst. 1152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Formula, 12> Formulae; 1153589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 1154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Regs - The set of register candidates used by all formulae in this LSRUse. 1155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Regs; 1156934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1157db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse(KindType K, Type *T) : Kind(K), AccessTy(T), 1158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MinOffset(INT64_MAX), 1159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MaxOffset(INT64_MIN), 1160a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman AllFixupsOutsideLoop(true), 1161a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman WidestFixupType(0) {} 116256a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1163a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool HasFormulaWithSameRegs(const Formula &F) const; 1164454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman bool InsertFormula(const Formula &F); 1165d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman void DeleteFormula(Formula &F); 1166b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses); 1167572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1172b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman} 1173b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 1174a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// HasFormula - Test whether this use as a formula which has the same 1175a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// registers as the given formula. 1176a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanbool LSRUse::HasFormulaWithSameRegs(const Formula &F) const { 1177a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1178a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1179a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1180a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman std::sort(Key.begin(), Key.end()); 1181a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return Uniquifier.count(Key); 1182a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 1183a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 1185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 1186454d26dc43207ec537d843229db6f5e6a302e23dDan Gohmanbool LSRUse::InsertFormula(const Formula &F) { 1187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::sort(Key.begin(), Key.end()); 1191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Uniquifier.insert(Key).second) 1193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Using a register to hold the value of 0 is not profitable. 1196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert((!F.ScaledReg || !F.ScaledReg->isZero()) && 1197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Zero allocated in a scaled register!"); 1198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 1199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = 1200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) 1201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!(*I)->isZero() && "Zero allocated in a base register!"); 1202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 1203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Add the formula to the list. 1205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formulae.push_back(F); 120656a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record registers now being used by this use. 1208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1213d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman/// DeleteFormula - Remove the given formula from this use's list. 1214d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohmanvoid LSRUse::DeleteFormula(Formula &F) { 12155ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&F != &Formulae.back()) 12165ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(F, Formulae.back()); 1217d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman Formulae.pop_back(); 1218d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman} 1219d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman 1220b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman/// RecomputeRegs - Recompute the Regs field, and update RegUses. 1221b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) { 1222b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 1223b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman SmallPtrSet<const SCEV *, 4> OldRegs = Regs; 1224b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.clear(); 1225402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = Formulae.begin(), 1226402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Formulae.end(); I != E; ++I) { 1227402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 1228b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (F.ScaledReg) Regs.insert(F.ScaledReg); 1229b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1230b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman } 1231b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1232b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Update the RegTracker. 1233b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (SmallPtrSet<const SCEV *, 4>::iterator I = OldRegs.begin(), 1234b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman E = OldRegs.end(); I != E; ++I) 1235b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (!Regs.count(*I)) 1236b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUses.DropRegister(*I, LUIdx); 1237b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 1238b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::print(raw_ostream &OS) const { 1240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR Use: Kind="; 1241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Basic: OS << "Basic"; break; 1243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Special: OS << "Special"; break; 1244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case ICmpZero: OS << "ICmpZero"; break; 1245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Address: 1246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "Address of "; 12471df9859c40492511b8aa4321eb76496005d3b75bDuncan Sands if (AccessTy->isPointerTy()) 1248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "pointer"; // the full pointer type could be really verbose 12497979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else 1250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *AccessTy; 12517979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 12521bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offsets={"; 1254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 1255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 1256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *I; 1257ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes if (llvm::next(I) != E) 1258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ','; 1259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '}'; 1261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AllFixupsOutsideLoop) 1263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", all-fixups-outside-loop"; 1264a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 1265a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (WidestFixupType) 1266a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman OS << ", widest fixup type: " << *WidestFixupType; 12677979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 1268d6b62a572210aff965a55626cf36a68821838844Evan Cheng 1269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::dump() const { 1270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 1271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 127256a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// isLegalUse - Test whether the use described by AM is "legal", meaning it can 1274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// be completely folded into the user instruction at isel time. This includes 1275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// address-mode folding and special icmp tricks. 1276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(const TargetLowering::AddrMode &AM, 1277db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Address: 1281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can 1282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // completely fold this address. 1283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) return TLI->isLegalAddressingMode(AM, AccessTy); 1284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise, just guess that reg+reg addressing is legal. 1286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.BaseOffs == 0 && AM.Scale <= 1; 1287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::ICmpZero: 1289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // There's not even a target hook for querying whether it would be legal to 1290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // fold a GV into an ICmp. 1291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) 1292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1293579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner 1294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only has two operands; don't allow more than two non-trivial parts. 1295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.HasBaseReg && AM.BaseOffs != 0) 1296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 12971bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by 1299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // putting the scaled register in the other operand of the icmp. 1300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.Scale != -1) 13017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 130281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can fold an 1304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // integer immediate on an icmp. 1305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 1306dae36ba802f12966e4fc44d99097a55ff0b7d87bEli Friedman if (TLI) return TLI->isLegalICmpImmediate(-(uint64_t)AM.BaseOffs); 1307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 13107979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return true; 131181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Basic: 1313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Only handle single-register values. 1314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.Scale == 0 && AM.BaseOffs == 0; 131581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1316572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Special: 1317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Only handle -1 scales, or no scale. 1318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return AM.Scale == 0 || AM.Scale == -1; 1319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 13214d6ccb5f68cd7c6418a209f1fa4dbade569e4493David Blaikie llvm_unreachable("Invalid LSRUse Kind!"); 1322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 132381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(TargetLowering::AddrMode AM, 1325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1326db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MinOffset) > AM.BaseOffs) != 1330572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MinOffset > 0)) 1331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MinOffset; 1333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(AM, Kind, AccessTy, TLI)) { 1334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs - MinOffset; 1335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MaxOffset) > AM.BaseOffs) != 1337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MaxOffset > 0)) 13387979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MaxOffset; 1340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 13417979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 13435f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 13445f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(int64_t BaseOffs, 1346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV, 1347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1348db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1349454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman const TargetLowering *TLI) { 1350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1361a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Canonicalize a scale of 1 to a base register if the formula doesn't 1362a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // already have a base register. 1363a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!AM.HasBaseReg && AM.Scale == 1) { 1364a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.Scale = 0; 1365a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.HasBaseReg = true; 1366a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1367a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 1369a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1370a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(const SCEV *S, 1372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1374db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI, 1376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE) { 1377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (S->isZero()) return true; 1379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t BaseOffs = ExtractImmediate(S, SE); 1383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV = ExtractSymbol(S, SE); 1384572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If there's anything else involved, it's not foldable. 1386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!S->isZero()) return false; 1387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, MinOffset, MaxOffset, Kind, AccessTy, TLI); 1400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 140156a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1402b6211710acdf558b3b45c2d198e74aa602496893Dan Gohmannamespace { 1403b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 14041e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// UseMapDenseMapInfo - A DenseMapInfo implementation for holding 14051e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// DenseMaps and DenseSets of pairs of const SCEV* and LSRUse::Kind. 14061e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohmanstruct UseMapDenseMapInfo { 14071e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getEmptyKey() { 14081e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-1), LSRUse::Basic); 14091e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14101e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14111e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getTombstoneKey() { 14121e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-2), LSRUse::Basic); 14131e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14141e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14151e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static unsigned 14161e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman getHashValue(const std::pair<const SCEV *, LSRUse::KindType> &V) { 14171e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman unsigned Result = DenseMapInfo<const SCEV *>::getHashValue(V.first); 14181e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman Result ^= DenseMapInfo<unsigned>::getHashValue(unsigned(V.second)); 14191e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return Result; 14201e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14211e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14221e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static bool isEqual(const std::pair<const SCEV *, LSRUse::KindType> &LHS, 14231e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman const std::pair<const SCEV *, LSRUse::KindType> &RHS) { 14241e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return LHS == RHS; 14251e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14261e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman}; 14271e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14286c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// IVInc - An individual increment in a Chain of IV increments. 14296c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Relate an IV user to an expression that computes the IV it uses from the IV 14306c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// used by the previous link in the Chain. 14316c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 14326c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// For the head of a chain, IncExpr holds the absolute SCEV expression for the 14336c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// original IVOperand. The head of the chain's IVOperand is only valid during 14346c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// chain collection, before LSR replaces IV users. During chain generation, 14356c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// IncExpr can be used to find the new IVOperand that computes the same 14366c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// expression. 14376c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstruct IVInc { 14386c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *UserInst; 14396c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Value* IVOperand; 14406c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick const SCEV *IncExpr; 14416c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 14426c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick IVInc(Instruction *U, Value *O, const SCEV *E): 14436c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick UserInst(U), IVOperand(O), IncExpr(E) {} 14446c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick}; 14456c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 14466c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick// IVChain - The list of IV increments in program order. 14476c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick// We typically add the head of a chain without finding subsequent links. 14486c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Tricktypedef SmallVector<IVInc,1> IVChain; 14496c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 14506c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// ChainUsers - Helper for CollectChains to track multiple IV increment uses. 14516c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Distinguish between FarUsers that definitely cross IV increments and 14526c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// NearUsers that may be used between IV increments. 14536c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstruct ChainUsers { 14546c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*, 4> FarUsers; 14556c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*, 4> NearUsers; 14566c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick}; 14576c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 1458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRInstance - This class holds state for the main loop strength reduction 1459572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// logic. 1460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRInstance { 1461572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers &IU; 1462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE; 1463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DominatorTree &DT; 1464e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LoopInfo &LI; 1465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 1466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loop *const L; 1467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed; 1468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// IVIncInsertPos - This is the insert position that the current loop's 1470572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable increment should be placed. In simple loops, this is 1471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// the latch block's terminator. But in more complicated cases, this is a 1472572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// position which will dominate all the in-loop post-increment users. 1473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *IVIncInsertPos; 1474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Factors - Interesting factors between use strides. 1476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<int64_t, 8> Factors; 1477572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Types - Interesting use types, to facilitate truncation reuse. 1479db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner SmallSetVector<Type *, 4> Types; 1480572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Fixups - The list of operands which are to be replaced. 1482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRFixup, 16> Fixups; 1483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Uses - The list of interesting uses. 1485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRUse, 16> Uses; 1486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1487572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// RegUses - Track which uses use which register candidates. 1488572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUseTracker RegUses; 1489572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 14906c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Limit the number of chains to avoid quadratic behavior. We don't expect to 14916c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // have more than a few IV increment chains in a loop. Missing a Chain falls 14926c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // back to normal LSR behavior for those uses. 14936c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick static const unsigned MaxChains = 8; 14946c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 14956c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick /// IVChainVec - IV users can form a chain of IV increments. 14966c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVector<IVChain, MaxChains> IVChainVec; 14976c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 149822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick /// IVIncSet - IV users that belong to profitable IVChains. 149922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SmallPtrSet<Use*, MaxChains> IVIncSet; 150022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 1501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void OptimizeShadowIV(); 1502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse); 1503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse); 1504c6519f916b5922de81c53547fd21364994195a70Dan Gohman void OptimizeLoopTermCond(); 1505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 15066c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick void ChainInstruction(Instruction *UserInst, Instruction *IVOper, 15076c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVectorImpl<ChainUsers> &ChainUsersVec); 150822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick void FinalizeChain(IVChain &Chain); 15096c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick void CollectChains(); 151022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick void GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter, 151122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SmallVectorImpl<WeakVH> &DeadInsts); 15126c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 1513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectInterestingTypesAndFactors(); 1514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectFixupsAndInitialFormulae(); 1515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &getNewFixup() { 1517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Fixups.push_back(LSRFixup()); 1518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Fixups.back(); 1519a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 152032e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng 1521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Support for sharing of LSRUses between LSRFixups. 15221e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman typedef DenseMap<std::pair<const SCEV *, LSRUse::KindType>, 15231e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman size_t, 15241e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMapDenseMapInfo> UseMapTy; 1525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UseMapTy UseMap; 1526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1527191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 1528db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy); 1529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> getUse(const SCEV *&Expr, 1531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, 1532db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy); 1533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1534c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void DeleteUse(LSRUse &LU, size_t LUIdx); 15355ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 1536191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU); 1537a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1538454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegisters(const Formula &F, size_t LUIdx); 1541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F); 1542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectLoopInvariantFixupsAndFormulae(); 1544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base, 1546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth = 0); 1547572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base); 1548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base); 1553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCrossUseConstantOffsets(); 1554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateAllReuseFormulae(); 1555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void FilterOutUndesirableDedicatedRegisters(); 1557d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 1558d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t EstimateSearchSpaceComplexity() const; 15594aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByDetectingSupersets(); 15604aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByCollapsingUnrolledCode(); 15614f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 15624aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByPickingWinnerRegs(); 1563572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void NarrowSearchSpaceUsingHeuristics(); 1564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 1566572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 1567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 1568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 1569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 1570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const; 1571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Solve(SmallVectorImpl<const Formula *> &Solution) const; 1572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1573e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock::iterator 1574e5f76877aee6f33964de105893f0ef338661ecadDan Gohman HoistInsertPosition(BasicBlock::iterator IP, 1575e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) const; 1576b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick BasicBlock::iterator 1577b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick AdjustInsertPositionForExpand(BasicBlock::iterator IP, 1578b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick const LSRFixup &LF, 1579b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick const LSRUse &LU, 1580b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick SCEVExpander &Rewriter) const; 1581d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 1582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *Expand(const LSRFixup &LF, 1583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1584454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman BasicBlock::iterator IP, 1585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1586454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const; 15873a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman void RewriteForPHI(PHINode *PN, const LSRFixup &LF, 15883a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 15893a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 15903a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 15913a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const; 1592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Rewrite(const LSRFixup &LF, 1593572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 1596572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const; 1597572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 1598572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P); 1599572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1600d56ef8d709c3dd5735debc23e082722718a7d58aAndrew Trickpublic: 1601572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRInstance(const TargetLowering *tli, Loop *l, Pass *P); 1602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool getChanged() const { return Changed; } 1604572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1605572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_factors_and_types(raw_ostream &OS) const; 1606572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_fixups(raw_ostream &OS) const; 1607572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_uses(raw_ostream &OS) const; 1608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 16117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1612a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1613c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1614572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// OptimizeShadowIV - If IV is used in a int-to-float cast 16153f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// inside the loop then try to eliminate the cast operation. 1616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::OptimizeShadowIV() { 1617572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 1618572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 1619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 1620c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); 1622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != E; /* empty */) { 1623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers::const_iterator CandidateUI = UI; 1624572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ++UI; 1625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *ShadowUse = CandidateUI->getUser(); 1626db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *DestTy = NULL; 1627c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick bool IsSigned = false; 1628c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* If shadow use is a int->float cast then insert a second IV 1630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman to eliminate this cast. 1631c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i) 1633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo((double)i); 1634c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman is transformed into 1636c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1637572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman double d = 0.0; 1638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i, ++d) 1639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo(d); 1640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman */ 1641c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) { 1642c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick IsSigned = false; 1643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = UCast->getDestTy(); 1644c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick } 1645c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) { 1646c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick IsSigned = true; 1647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = SCast->getDestTy(); 1648c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick } 1649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DestTy) continue; 16505792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1651572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) { 1652572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If target does not support DestTy natively then do not apply 1653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // this transformation. 1654572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman EVT DVT = TLI->getValueType(DestTy); 1655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI->isTypeLegal(DVT)) continue; 16567e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 16577e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); 1659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!PH) continue; 1660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getNumIncomingValues() != 2) continue; 1661a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1662db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *SrcTy = PH->getType(); 1663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int Mantissa = DestTy->getFPMantissaWidth(); 1664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mantissa == -1) continue; 1665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa) 1666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 16672f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 1668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Entry, Latch; 1669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { 1670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 0; 1671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 1; 1672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 1673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 1; 1674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 0; 1675a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1676eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 1677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); 1678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Init) continue; 1679c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick Constant *NewInit = ConstantFP::get(DestTy, IsSigned ? 1680c205a094bd5019773c98adcfbdc21c07c9da1888Andrew Trick (double)Init->getSExtValue() : 1681c205a094bd5019773c98adcfbdc21c07c9da1888Andrew Trick (double)Init->getZExtValue()); 1682a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1683572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *Incr = 1684572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); 1685572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Incr) continue; 1686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOpcode() != Instruction::Add 1687572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman && Incr->getOpcode() != Instruction::Sub) 1688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1689aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 1690572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Initialize new IV, double d = 0.0 in above example. */ 1691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *C = NULL; 1692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOperand(0) == PH) 1693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(1)); 1694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Incr->getOperand(1) == PH) 1695572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(0)); 1696572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1697572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1698bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman 1699572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C) continue; 1700a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1701572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore negative constants, as the code below doesn't handle them 1702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // correctly. TODO: Remove this restriction. 1703572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C->getValue().isStrictlyPositive()) continue; 1704a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1705572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Add new PHINode. */ 17063ecfc861b4365f341c5c969b40e1afccde676e6fJay Foad PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH); 1707a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1708572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* create new increment. '++d' in above example. */ 1709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue()); 1710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *NewIncr = 1711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ? 1712572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction::FAdd : Instruction::FSub, 1713572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH, CFP, "IV.S.next.", Incr); 1714a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1715572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); 1716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); 17178b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 1718572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Remove cast operation */ 1719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->replaceAllUsesWith(NewPH); 1720572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->eraseFromParent(); 1721c6519f916b5922de81c53547fd21364994195a70Dan Gohman Changed = true; 1722572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 17237979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1724a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1725cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 17267979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// FindIVUserForCond - If Cond has an operand that is an expression of an IV, 17277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// set the IV user and stride information and return true, otherwise return 17287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// false. 1729ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanbool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) { 1730572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UI->getUser() == Cond) { 1732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // NOTE: we could handle setcc instructions with multiple uses here, but 1733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // InstCombine does it as well for simple uses, it's not clear that it 1734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // occurs enough in real life to handle. 1735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CondUse = UI; 1736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1737a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1739cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 1740cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 17417979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// OptimizeMax - Rewrite the loop's terminating condition if it uses 17427979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a max computation. 17437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17447979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This is a narrow solution to a specific, but acute, problem. For loops 17457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// like this: 17467979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17477979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 17487979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 17497979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 17507979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 17517979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17527979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the trip count isn't just 'n', because 'n' might not be positive. And 17537979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// unfortunately this can come up even for loops where the user didn't use 17547979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a C do-while loop. For example, seemingly well-behaved top-test loops 17557979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// will commonly be lowered like this: 17567979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman// 17577979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// if (n > 0) { 17587979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 17597979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 17607979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 17617979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 17627979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } 17637979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17647979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// and then it's possible for subsequent optimization to obscure the if 17657979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// test in such a way that indvars can't find it. 17667979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17677979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// When indvars can't find the if test in loops like this, it creates a 17687979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max expression, which allows it to give the loop a canonical 17697979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// induction variable: 17707979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17717979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 17727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max = n < 1 ? 1 : n; 17737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 17747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 17757979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i != max); 17767979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17777979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// Canonical induction variables are necessary because the loop passes 17787979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// are designed around them. The most obvious example of this is the 17797979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// LoopInfo analysis, which doesn't remember trip count values. It 17807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// expects to be able to rediscover the trip count each time it is 1781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needed, and it does this using a simple analysis that only succeeds if 17827979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the loop has a canonical induction variable. 17837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// However, when it comes time to generate code, the maximum operation 17857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// can be quite costly, especially if it's inside of an outer loop. 17867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This function solves this problem by detecting this type of loop and 17887979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting 17897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the instructions for the maximum computation. 17907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 1791572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { 17927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check that the loop matches the pattern we're looking for. 17937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() != CmpInst::ICMP_EQ && 17947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->getPredicate() != CmpInst::ICMP_NE) 17957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1796ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 17977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); 17987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!Sel || !Sel->hasOneUse()) return Cond; 1799ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 1800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 18017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 18027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1803deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); 1804ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Add one to the backedge-taken count to get the trip count. 18064065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); 18071d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (IterationCount != SE.getSCEV(Sel)) return Cond; 18081d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 18091d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Check for a max calculation that matches the pattern. There's no check 18101d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for ICMP_ULE here because the comparison would be with zero, which 18111d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // isn't interesting. 18121d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; 18131d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman const SCEVNAryExpr *Max = 0; 18141d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) { 18151d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLE; 18161d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 18171d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) { 18181d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLT; 18191d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 18201d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) { 18211d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_ULT; 18221d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = U; 18231d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else { 18241d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // No match; bail. 18257979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 18261d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } 1827ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // To handle a max with more than two operands, this optimization would 18297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // require additional checking and setup. 18307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Max->getNumOperands() != 2) 18317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 18327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 18337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxLHS = Max->getOperand(0); 18347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxRHS = Max->getOperand(1); 18351d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 18361d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // ScalarEvolution canonicalizes constants to the left. For < and >, look 18371d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for a comparison with 1. For <= and >=, a comparison with zero. 18381d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!MaxLHS || 18391d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One))) 18401d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 18411d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 18427979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the relevant induction variable for conformance to 18437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // the pattern. 1844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); 18457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 18467979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!AR || !AR->isAffine() || 18477979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AR->getStart() != One || 1848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE) != One) 18497979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1850ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18517979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman assert(AR->getLoop() == L && 18527979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman "Loop condition operand is an addrec in a different loop!"); 1853bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman 18547979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the right operand of the select, and remember it, as it will 18557979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // be used in the new comparison instruction. 18567979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Value *NewRHS = 0; 18571d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (ICmpInst::isTrueWhenEqual(Pred)) { 18581d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Look for n+1, and grab n. 18591d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1))) 18601d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 18611d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 18621d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 18631d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 18641d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2))) 18651d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 18661d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 18671d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 18681d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 18691d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!NewRHS) 18701d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 18711d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS) 18727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(1); 1873572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS) 18747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(2); 1875caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS)) 1876caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman NewRHS = SU->getValue(); 18771d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman else 1878caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman // Max doesn't match expected pattern. 1879caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman return Cond; 1880ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Determine the new comparison opcode. It may be signed or unsigned, 18827979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // and the original comparison may be either equality or inequality. 18837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() == CmpInst::ICMP_EQ) 18847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Pred = CmpInst::getInversePredicate(Pred); 18852781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman 18867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Ok, everything looks ok to change the condition into an SLT or SGE and 18877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // delete the max calculation. 18887979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman ICmpInst *NewCond = 18897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp"); 1890ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Delete the max calculation instructions. 18927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->replaceAllUsesWith(NewCond); 18937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman CondUse->setUser(NewCond); 18947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); 18957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->eraseFromParent(); 18967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Sel->eraseFromParent(); 18977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cmp->use_empty()) 18987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cmp->eraseFromParent(); 18997979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return NewCond; 1900ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman} 1901ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 190256a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach/// OptimizeLoopTermCond - Change loop terminating condition to use the 19032d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng/// postinc iv when possible. 1904c6519f916b5922de81c53547fd21364994195a70Dan Gohmanvoid 1905572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::OptimizeLoopTermCond() { 1906572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<Instruction *, 4> PostIncs; 1907572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 19085792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng BasicBlock *LatchBlock = L->getLoopLatch(); 1909076e085698c484354c9e131f1bd8fd001581397bEvan Cheng SmallVector<BasicBlock*, 8> ExitingBlocks; 1910076e085698c484354c9e131f1bd8fd001581397bEvan Cheng L->getExitingBlocks(ExitingBlocks); 191156a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1912076e085698c484354c9e131f1bd8fd001581397bEvan Cheng for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 1913076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BasicBlock *ExitingBlock = ExitingBlocks[i]; 1914010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 1915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Get the terminating condition for the loop if possible. If we 1916076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // can, we want to change it to use a post-incremented version of its 1917076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // induction variable, to allow coalescing the live ranges for the IV into 1918076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // one register value. 1919cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1920076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); 1921076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (!TermBr) 1922076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1923076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // FIXME: Overly conservative, termination condition could be an 'or' etc.. 1924076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition())) 1925076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1926076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 1927076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Search IVUsesByStride to find Cond's IVUse if there is one. 1928076e085698c484354c9e131f1bd8fd001581397bEvan Cheng IVStrideUse *CondUse = 0; 1929076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); 1930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!FindIVUserForCond(Cond, CondUse)) 1931076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1932076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 19337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // If the trip count is computed in terms of a max (due to ScalarEvolution 19347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // being unable to find a sufficient guard, for example), change the loop 19357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // comparison to use SLT or ULT instead of NE. 1936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // One consequence of doing this now is that it disrupts the count-down 1937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // optimization. That's not always a bad thing though, because in such 1938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // cases it may still be worthwhile to avoid a max. 1939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cond = OptimizeMax(Cond, CondUse); 1940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this exiting block dominates the latch block, it may also use 1942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the post-inc value if it won't be shared with other uses. 1943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for dominance. 1944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(ExitingBlock, LatchBlock)) 19457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman continue; 19467979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1947572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively avoid trying to use the post-inc value in non-latch 1948572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // exits if there may be pre-inc users in intervening blocks. 1949590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (LatchBlock != ExitingBlock) 1950572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1951572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Test if the use is reachable from the exiting block. This dominator 1952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // query is a conservative approximation of reachability. 1953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*UI != CondUse && 1954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) { 1955572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume there may be reuse if the quotient of their 1956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // strides could be a legal scale. 1957c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *A = IU.getStride(*CondUse, L); 1958c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *B = IU.getStride(*UI, L); 1959448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (!A || !B) continue; 1960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) != 1961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) { 1962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) > 1963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) 1964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman B = SE.getSignExtendExpr(B, A->getType()); 1965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman A = SE.getSignExtendExpr(A, B->getType()); 1967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *D = 1969f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) { 19709f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const ConstantInt *C = D->getValue(); 1971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Stride of one or negative one can have reuse with non-addresses. 19729f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->isOne() || C->isAllOnesValue()) 1973572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Avoid weird situations. 19759f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->getValue().getMinSignedBits() >= 64 || 19769f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman C->getValue().isMinSignedValue()) 1977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1978590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // Without TLI, assume that any stride might be valid, and so any 1979590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // use might be shared. 1980590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (!TLI) 1981590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman goto decline_post_inc; 1982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for possible scaled-address reuse. 1983db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = getAccessType(UI->getUser()); 1984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 19859f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman AM.Scale = C->getSExtValue(); 19862763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 1987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1988572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = -AM.Scale; 19892763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 1990572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 199463c9463c62fce8cbe02176dfa2d73f375a06f1f2David Greene DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: " 1995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << *Cond << '\n'); 199656a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1997076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // It's possible for the setcc instruction to be anywhere in the loop, and 1998076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // possible for it to have multiple users. If it is not immediately before 1999076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // the exiting block branch, move it. 2000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*++BasicBlock::iterator(Cond) != TermBr) { 2001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Cond->hasOneUse()) { 2002076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->moveBefore(TermBr); 2003076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } else { 2004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clone the terminating condition and insert into the loopend. 2005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OldCond = Cond; 2006076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond = cast<ICmpInst>(Cond->clone()); 2007076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->setName(L->getHeader()->getName() + ".termcond"); 2008076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ExitingBlock->getInstList().insert(TermBr, Cond); 2009076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 2010076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Clone the IVUse, as the old use still exists! 20114417e537b65c14b378aeca75b2773582dd102f63Andrew Trick CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace()); 2012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TermBr->replaceUsesOfWith(OldCond, Cond); 2013076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } 2014010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2015010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2016076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // If we get to here, we know that we can transform the setcc instruction to 2017076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // use the post-incremented version of the IV, allowing us to coalesce the 2018076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // live ranges for the IV correctly. 2019448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman CondUse->transformToPostInc(L); 2020076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Changed = true; 20215792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 2022572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PostIncs.insert(Cond); 2023572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman decline_post_inc:; 2024c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen } 2025c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine an insertion point for the loop induction variable increment. It 2027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // must dominate all the post-inc comparisons we just set up, and it must 2028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // dominate the loop latch edge. 2029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = L->getLoopLatch()->getTerminator(); 2030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<Instruction *, 4>::const_iterator I = PostIncs.begin(), 2031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = PostIncs.end(); I != E; ++I) { 2032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BasicBlock *BB = 2033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT.findNearestCommonDominator(IVIncInsertPos->getParent(), 2034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (*I)->getParent()); 2035572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BB == (*I)->getParent()) 2036572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = *I; 2037572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (BB != IVIncInsertPos->getParent()) 2038572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = BB->getTerminator(); 2039572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2040572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2041a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 20427a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner/// reconcileNewOffset - Determine if the given use can accommodate a fixup 204376c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// at the given offset and other details. If so, update the use and 204476c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// return true. 2045572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 2046191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan GohmanLSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 2047db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy) { 2048191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMinOffset = LU.MinOffset; 2049191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMaxOffset = LU.MaxOffset; 2050db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *NewAccessTy = AccessTy; 2051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched kind. It's tempting to collapse mismatched kinds to 2053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // something conservative, however this can pessimize in the case that one of 2054572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the uses will have all its uses outside the loop, for example. 2055572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != Kind) 2056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 2057572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume HasBaseReg is true for now. 2058191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset < LU.MinOffset) { 2059191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(LU.MaxOffset - NewOffset, 0, HasBaseReg, 2060454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 20617979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 2062191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMinOffset = NewOffset; 2063191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } else if (NewOffset > LU.MaxOffset) { 2064191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(NewOffset - LU.MinOffset, 0, HasBaseReg, 2065454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 20667979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 2067191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMaxOffset = NewOffset; 2068a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2069572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched access type, and fall back conservatively as needed. 207074e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // TODO: Be less conservative when the type is similar and can use the same 207174e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // addressing modes. 2072572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Kind == LSRUse::Address && AccessTy != LU.AccessTy) 2073191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewAccessTy = Type::getVoidTy(AccessTy->getContext()); 2074572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2075572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the use. 2076191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MinOffset = NewMinOffset; 2077191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MaxOffset = NewMaxOffset; 2078191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.AccessTy = NewAccessTy; 2079191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset != LU.Offsets.back()) 2080191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(NewOffset); 2081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 2082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2083a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getUse - Return an LSRUse index and an offset value for a fixup which 2085572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needs the given expression, with the given kind and optional access type. 20863f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// Either reuse an existing use or create a new one, as needed. 2087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstd::pair<size_t, int64_t> 2088572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::getUse(const SCEV *&Expr, 2089db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy) { 2090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Copy = Expr; 2091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = ExtractImmediate(Expr, SE); 2092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Basic uses can't accept any offset, for example. 2094454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!isAlwaysFoldable(Offset, 0, /*HasBaseReg=*/true, Kind, AccessTy, TLI)) { 2095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Expr = Copy; 2096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = 0; 2097a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2098a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<UseMapTy::iterator, bool> P = 21001e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMap.insert(std::make_pair(std::make_pair(Expr, Kind), 0)); 2101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!P.second) { 2102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // A use already existed with this base. 2103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = P.first->second; 2104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2105191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy)) 2106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Reuse this use. 2107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 2108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2109a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Create a new use. 2111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = Uses.size(); 2112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman P.first->second = LUIdx; 2113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses.push_back(LSRUse(Kind, AccessTy)); 2114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2115a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2116191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // We don't need to track redundant offsets, but we don't need to go out 2117191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // of our way here to avoid them. 2118191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LU.Offsets.empty() || Offset != LU.Offsets.back()) 2119191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(Offset); 2120191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 2121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MinOffset = Offset; 2122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MaxOffset = Offset; 2123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 2124a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 2125a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 21265ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteUse - Delete the given use from the Uses list. 2127c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohmanvoid LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) { 2128191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (&LU != &Uses.back()) 21295ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(LU, Uses.back()); 21305ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman Uses.pop_back(); 2131c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 2132c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. 2133c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman RegUses.SwapAndDropUse(LUIdx, Uses.size()); 21345ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 21355ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 2136a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// FindUseWithFormula - Look for a use distinct from OrigLU which is has 2137a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// a formula that has the same registers as the given formula. 2138a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRUse * 2139a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRInstance::FindUseWithSimilarFormula(const Formula &OrigF, 2140191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman const LSRUse &OrigLU) { 2141191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Search all uses for the formula. This could be more clever. 2142a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2143a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 21446a832715325f740a0e78acd194abad9104389075Dan Gohman // Check whether this use is close enough to OrigLU, to see whether it's 21456a832715325f740a0e78acd194abad9104389075Dan Gohman // worthwhile looking through its formulae. 21466a832715325f740a0e78acd194abad9104389075Dan Gohman // Ignore ICmpZero uses because they may contain formulae generated by 21476a832715325f740a0e78acd194abad9104389075Dan Gohman // GenerateICmpZeroScales, in which case adding fixup offsets may 21486a832715325f740a0e78acd194abad9104389075Dan Gohman // be invalid. 2149a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (&LU != &OrigLU && 2150a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind != LSRUse::ICmpZero && 2151a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy && 2152a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType == OrigLU.WidestFixupType && 2153a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.HasFormulaWithSameRegs(OrigF)) { 21546a832715325f740a0e78acd194abad9104389075Dan Gohman // Scan through this use's formulae. 2155402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 2156402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 2157402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 21586a832715325f740a0e78acd194abad9104389075Dan Gohman // Check to see if this formula has the same registers and symbols 21596a832715325f740a0e78acd194abad9104389075Dan Gohman // as OrigF. 2160a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.BaseRegs == OrigF.BaseRegs && 2161a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.ScaledReg == OrigF.ScaledReg && 2162a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.AM.BaseGV == OrigF.AM.BaseGV && 2163cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.AM.Scale == OrigF.AM.Scale && 2164cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset == OrigF.UnfoldedOffset) { 2165191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (F.AM.BaseOffs == 0) 2166a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return &LU; 21676a832715325f740a0e78acd194abad9104389075Dan Gohman // This is the formula where all the registers and symbols matched; 21686a832715325f740a0e78acd194abad9104389075Dan Gohman // there aren't going to be any others. Since we declined it, we 21696a832715325f740a0e78acd194abad9104389075Dan Gohman // can skip the rest of the formulae and procede to the next LSRUse. 2170a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 2171a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2172a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2173a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2174a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2175a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 21766a832715325f740a0e78acd194abad9104389075Dan Gohman // Nothing looked good. 2177a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return 0; 2178a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 2179a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectInterestingTypesAndFactors() { 2181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> Strides; 2182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 21831b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Collect interesting types and strides. 2184448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman SmallVector<const SCEV *, 4> Worklist; 2185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 2186c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *Expr = IU.getExpr(*UI); 2187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect interesting types. 2189448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Types.insert(SE.getEffectiveSCEVType(Expr->getType())); 2190448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 2191448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // Add strides for mentioned loops. 2192448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(Expr); 2193448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman do { 2194448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman const SCEV *S = Worklist.pop_back_val(); 2195448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 2196fa1948a40f14d98c1a31a2ec19035a2d5254e854Andrew Trick if (EnableNested || AR->getLoop() == L) 2197fa1948a40f14d98c1a31a2ec19035a2d5254e854Andrew Trick Strides.insert(AR->getStepRecurrence(SE)); 2198448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(AR->getStart()); 2199448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 2200403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(Add->op_begin(), Add->op_end()); 2201448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 2202448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } while (!Worklist.empty()); 22031b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman } 22041b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman 22051b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Compute interesting factors from the set of interesting strides. 22061b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator 22071b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman I = Strides.begin(), E = Strides.end(); I != E; ++I) 2208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = 2209ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes llvm::next(I); NewStrideIter != E; ++NewStrideIter) { 22101b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman const SCEV *OldStride = *I; 2211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewStride = *NewStrideIter; 2212a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) != 2214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) { 2215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) > 2216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) 2217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType()); 2218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType()); 2220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *Factor = 2222f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride, 2223f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVConstant *Factor = 2227454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride, 2228454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman NewStride, 2229f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2232a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2233a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2234586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If all uses use the same type, don't bother looking for truncation-based 2236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reuse. 2237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Types.size() == 1) 2238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 2239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_factors_and_types(dbgs())); 2241c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen} 2242c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 22436c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// findIVOperand - Helper for CollectChains that finds an IV operand (computed 22446c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// by an AddRec in this loop) within [OI,OE) or returns OE. If IVUsers mapped 22456c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Instructions to IVStrideUses, we could partially skip this. 22466c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstatic User::op_iterator 22476c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew TrickfindIVOperand(User::op_iterator OI, User::op_iterator OE, 22486c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Loop *L, ScalarEvolution &SE) { 22496c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for(; OI != OE; ++OI) { 22506c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (Instruction *Oper = dyn_cast<Instruction>(*OI)) { 22516c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!SE.isSCEVable(Oper->getType())) 22526c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 22536c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 22546c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (const SCEVAddRecExpr *AR = 22556c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Oper))) { 22566c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (AR->getLoop() == L) 22576c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick break; 22586c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 22596c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 22606c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 22616c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return OI; 22626c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 22636c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 22646c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// getWideOperand - IVChain logic must consistenctly peek base TruncInst 22656c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// operands, so wrap it in a convenient helper. 22666c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstatic Value *getWideOperand(Value *Oper) { 22676c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (TruncInst *Trunc = dyn_cast<TruncInst>(Oper)) 22686c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return Trunc->getOperand(0); 22696c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return Oper; 22706c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 22716c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 22726c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// isCompatibleIVType - Return true if we allow an IV chain to include both 22736c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// types. 22746c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstatic bool isCompatibleIVType(Value *LVal, Value *RVal) { 22756c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Type *LType = LVal->getType(); 22766c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Type *RType = RVal->getType(); 22776c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return (LType == RType) || (LType->isPointerTy() && RType->isPointerTy()); 22786c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 22796c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 228064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// getExprBase - Return an approximation of this SCEV expression's "base", or 228164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// NULL for any constant. Returning the expression itself is 228264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// conservative. Returning a deeper subexpression is more precise and valid as 228364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// long as it isn't less complex than another subexpression. For expressions 228464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// involving multiple unscaled values, we need to return the pointer-type 228564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// SCEVUnknown. This avoids forming chains across objects, such as: 228664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// PrevOper==a[i], IVOper==b[i], IVInc==b-a. 228764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// 228864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost 228964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// SCEVUnknown, we simply return the rightmost SCEV operand. 229064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trickstatic const SCEV *getExprBase(const SCEV *S) { 229164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick switch (S->getSCEVType()) { 229264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick default: // uncluding scUnknown. 229364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return S; 229464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scConstant: 229564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return 0; 229664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scTruncate: 229764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVTruncateExpr>(S)->getOperand()); 229864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scZeroExtend: 229964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVZeroExtendExpr>(S)->getOperand()); 230064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scSignExtend: 230164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVSignExtendExpr>(S)->getOperand()); 230264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scAddExpr: { 230364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Skip over scaled operands (scMulExpr) to follow add operands as long as 230464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // there's nothing more complex. 230564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // FIXME: not sure if we want to recognize negation. 230664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEVAddExpr *Add = cast<SCEVAddExpr>(S); 230764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (std::reverse_iterator<SCEVAddExpr::op_iterator> I(Add->op_end()), 230864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick E(Add->op_begin()); I != E; ++I) { 230964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *SubExpr = *I; 231064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (SubExpr->getSCEVType() == scAddExpr) 231164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(SubExpr); 231264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 231364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (SubExpr->getSCEVType() != scMulExpr) 231464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return SubExpr; 231564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 231664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return S; // all operands are scaled, be conservative. 231764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 231864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scAddRecExpr: 231964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVAddRecExpr>(S)->getStart()); 232064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 232164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick} 232264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 232322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Return true if the chain increment is profitable to expand into a loop 232422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// invariant value, which may require its own register. A profitable chain 232522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// increment will be an offset relative to the same base. We allow such offsets 232622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// to potentially be used as chain increment as long as it's not obviously 232722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// expensive to expand using real instructions. 232822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic const SCEV * 232922d20c218aeb14af388bff2346d6d4cc131e8449Andrew TrickgetProfitableChainIncrement(Value *NextIV, Value *PrevIV, 233022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const IVChain &Chain, Loop *L, 233122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ScalarEvolution &SE, const TargetLowering *TLI) { 233264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Prune the solution space aggressively by checking that both IV operands 233364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // are expressions that operate on the same unscaled SCEVUnknown. This 233464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // "base" will be canceled by the subsequent getMinusSCEV call. Checking first 233564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // avoids creating extra SCEV expressions. 233664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *OperExpr = SE.getSCEV(NextIV); 233764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *PrevExpr = SE.getSCEV(PrevIV); 233864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (getExprBase(OperExpr) != getExprBase(PrevExpr) && !StressIVChain) 233964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return 0; 234064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 234164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr); 234222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!SE.isLoopInvariant(IncExpr, L)) 234322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return 0; 234422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 234522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // We are not able to expand an increment unless it is loop invariant, 234622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // however, the following checks are purely for profitability. 234722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (StressIVChain) 234822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return IncExpr; 234922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 235064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Do not replace a constant offset from IV head with a nonconstant IV 235164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // increment. 235264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (!isa<SCEVConstant>(IncExpr)) { 235364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Chain[0].IVOperand)); 235464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr))) 235564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return 0; 235664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 235764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 235864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick SmallPtrSet<const SCEV*, 8> Processed; 235964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isHighCostExpansion(IncExpr, Processed, SE)) 236064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return 0; 236164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 236264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return IncExpr; 236322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 236422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 236522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Return true if the number of registers needed for the chain is estimated to 236622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// be less than the number required for the individual IV users. First prohibit 236722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// any IV users that keep the IV live across increments (the Users set should 236822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// be empty). Next count the number and type of increments in the chain. 236922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// 237022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Chaining IVs can lead to considerable code bloat if ISEL doesn't 237122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// effectively use postinc addressing modes. Only consider it profitable it the 237222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// increments can be computed in fewer registers when chained. 237322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// 237422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// TODO: Consider IVInc free if it's already used in another chains. 237522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic bool 237622d20c218aeb14af388bff2346d6d4cc131e8449Andrew TrickisProfitableChain(IVChain &Chain, SmallPtrSet<Instruction*, 4> &Users, 237722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ScalarEvolution &SE, const TargetLowering *TLI) { 237822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (StressIVChain) 237922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return true; 238022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 238164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (Chain.size() <= 2) 238264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 238364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 238464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (!Users.empty()) { 238564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick DEBUG(dbgs() << "Chain: " << *Chain[0].UserInst << " users:\n"; 238664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (SmallPtrSet<Instruction*, 4>::const_iterator I = Users.begin(), 238764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick E = Users.end(); I != E; ++I) { 238864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick dbgs() << " " << **I << "\n"; 238964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick }); 239064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 239164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 239264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick assert(!Chain.empty() && "empty IV chains are not allowed"); 239364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 239464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // The chain itself may require a register, so intialize cost to 1. 239564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick int cost = 1; 239664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 239764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // A complete chain likely eliminates the need for keeping the original IV in 239864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // a register. LSR does not currently know how to form a complete chain unless 239964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // the header phi already exists. 240064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isa<PHINode>(Chain.back().UserInst) 240164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick && SE.getSCEV(Chain.back().UserInst) == Chain[0].IncExpr) { 240264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick --cost; 240364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 240464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *LastIncExpr = 0; 240564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick unsigned NumConstIncrements = 0; 240664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick unsigned NumVarIncrements = 0; 240764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick unsigned NumReusedIncrements = 0; 240864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (IVChain::const_iterator I = llvm::next(Chain.begin()), E = Chain.end(); 240964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick I != E; ++I) { 241064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 241164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (I->IncExpr->isZero()) 241264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick continue; 241364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 241464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Incrementing by zero or some constant is neutral. We assume constants can 241564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // be folded into an addressing mode or an add's immediate operand. 241664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isa<SCEVConstant>(I->IncExpr)) { 241764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ++NumConstIncrements; 241864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick continue; 241964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 242064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 242164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (I->IncExpr == LastIncExpr) 242264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ++NumReusedIncrements; 242364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick else 242464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ++NumVarIncrements; 242564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 242664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick LastIncExpr = I->IncExpr; 242764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 242864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // An IV chain with a single increment is handled by LSR's postinc 242964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // uses. However, a chain with multiple increments requires keeping the IV's 243064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // value live longer than it needs to be if chained. 243164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (NumConstIncrements > 1) 243264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick --cost; 243364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 243464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Materializing increment expressions in the preheader that didn't exist in 243564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // the original code may cost a register. For example, sign-extended array 243664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // indices can produce ridiculous increments like this: 243764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64))) 243864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick cost += NumVarIncrements; 243964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 244064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Reusing variable increments likely saves a register to hold the multiple of 244164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // the stride. 244264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick cost -= NumReusedIncrements; 244364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 244464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick DEBUG(dbgs() << "Chain: " << *Chain[0].UserInst << " Cost: " << cost << "\n"); 244564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 244664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return cost < 0; 244722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 244822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 24496c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// ChainInstruction - Add this IV user to an existing chain or make it the head 24506c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// of a new chain. 24516c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickvoid LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper, 24526c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVectorImpl<ChainUsers> &ChainUsersVec) { 24536c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // When IVs are used as types of varying widths, they are generally converted 24546c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // to a wider type with some uses remaining narrow under a (free) trunc. 24556c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Value *NextIV = getWideOperand(IVOper); 24566c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 24576c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Visit all existing chains. Check if its IVOper can be computed as a 24586c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // profitable loop invariant increment from the last link in the Chain. 24596c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick unsigned ChainIdx = 0, NChains = IVChainVec.size(); 24606c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick const SCEV *LastIncExpr = 0; 24616c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (; ChainIdx < NChains; ++ChainIdx) { 24626c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Value *PrevIV = getWideOperand(IVChainVec[ChainIdx].back().IVOperand); 24636c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!isCompatibleIVType(PrevIV, NextIV)) 24646c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 24656c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 24666c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // A phi nodes terminates a chain. 24676c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (isa<PHINode>(UserInst) 24686c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick && isa<PHINode>(IVChainVec[ChainIdx].back().UserInst)) 24696c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 24706c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 247122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (const SCEV *IncExpr = 247222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick getProfitableChainIncrement(NextIV, PrevIV, IVChainVec[ChainIdx], 247322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick L, SE, TLI)) { 24746c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick LastIncExpr = IncExpr; 24756c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick break; 24766c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 24776c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 24786c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // If we haven't found a chain, create a new one, unless we hit the max. Don't 24796c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // bother for phi nodes, because they must be last in the chain. 24806c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (ChainIdx == NChains) { 24816c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (isa<PHINode>(UserInst)) 24826c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return; 248322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (NChains >= MaxChains && !StressIVChain) { 24846c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick DEBUG(dbgs() << "IV Chain Limit\n"); 24856c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return; 24866c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 24870041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick LastIncExpr = SE.getSCEV(NextIV); 24880041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick // IVUsers may have skipped over sign/zero extensions. We don't currently 24890041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick // attempt to form chains involving extensions unless they can be hoisted 24900041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick // into this loop's AddRec. 24910041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick if (!isa<SCEVAddRecExpr>(LastIncExpr)) 24920041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick return; 24936c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ++NChains; 24946c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick IVChainVec.resize(NChains); 24956c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec.resize(NChains); 24966c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick DEBUG(dbgs() << "IV Head: (" << *UserInst << ") IV=" << *LastIncExpr 24976c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick << "\n"); 24986c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 24996c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick else 25006c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick DEBUG(dbgs() << "IV Inc: (" << *UserInst << ") IV+" << *LastIncExpr 25016c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick << "\n"); 25026c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25036c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Add this IV user to the end of the chain. 25046c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick IVChainVec[ChainIdx].push_back(IVInc(UserInst, IVOper, LastIncExpr)); 25056c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25066c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers; 25076c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // This chain's NearUsers become FarUsers. 25086c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!LastIncExpr->isZero()) { 25096c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec[ChainIdx].FarUsers.insert(NearUsers.begin(), 25106c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick NearUsers.end()); 25116c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick NearUsers.clear(); 25126c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25136c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25146c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // All other uses of IVOperand become near uses of the chain. 25156c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // We currently ignore intermediate values within SCEV expressions, assuming 25166c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // they will eventually be used be the current chain, or can be computed 25176c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // from one of the chain increments. To be more precise we could 25186c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // transitively follow its user and only add leaf IV users to the set. 25196c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (Value::use_iterator UseIter = IVOper->use_begin(), 25206c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick UseEnd = IVOper->use_end(); UseIter != UseEnd; ++UseIter) { 25216c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *OtherUse = dyn_cast<Instruction>(*UseIter); 25226c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (SE.isSCEVable(OtherUse->getType()) 25236c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick && !isa<SCEVUnknown>(SE.getSCEV(OtherUse)) 25246c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick && IU.isIVUserOrOperand(OtherUse)) { 25256c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 25266c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25276c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (OtherUse && OtherUse != UserInst) 25286c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick NearUsers.insert(OtherUse); 25296c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25306c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25316c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Since this user is part of the chain, it's no longer considered a use 25326c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // of the chain. 25336c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec[ChainIdx].FarUsers.erase(UserInst); 25346c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 25356c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25366c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// CollectChains - Populate the vector of Chains. 25376c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25386c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// This decreases ILP at the architecture level. Targets with ample registers, 25396c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// multiple memory ports, and no register renaming probably don't want 25406c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// this. However, such targets should probably disable LSR altogether. 25416c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25426c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// The job of LSR is to make a reasonable choice of induction variables across 25436c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// the loop. Subsequent passes can easily "unchain" computation exposing more 25446c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// ILP *within the loop* if the target wants it. 25456c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25466c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Finding the best IV chain is potentially a scheduling problem. Since LSR 25476c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// will not reorder memory operations, it will recognize this as a chain, but 25486c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// will generate redundant IV increments. Ideally this would be corrected later 25496c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// by a smart scheduler: 25506c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// = A[i] 25516c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// = A[i+x] 25526c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// A[i] = 25536c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// A[i+x] = 25546c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25556c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// TODO: Walk the entire domtree within this loop, not just the path to the 25566c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// loop latch. This will discover chains on side paths, but requires 25576c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// maintaining multiple copies of the Chains state. 25586c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickvoid LSRInstance::CollectChains() { 25596c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVector<ChainUsers, 8> ChainUsersVec; 25606c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25616c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVector<BasicBlock *,8> LatchPath; 25626c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick BasicBlock *LoopHeader = L->getHeader(); 25636c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (DomTreeNode *Rung = DT.getNode(L->getLoopLatch()); 25646c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) { 25656c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick LatchPath.push_back(Rung->getBlock()); 25666c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25676c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick LatchPath.push_back(LoopHeader); 25686c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25696c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Walk the instruction stream from the loop header to the loop latch. 25706c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (SmallVectorImpl<BasicBlock *>::reverse_iterator 25716c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick BBIter = LatchPath.rbegin(), BBEnd = LatchPath.rend(); 25726c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick BBIter != BBEnd; ++BBIter) { 25736c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (BasicBlock::iterator I = (*BBIter)->begin(), E = (*BBIter)->end(); 25746c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick I != E; ++I) { 25756c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Skip instructions that weren't seen by IVUsers analysis. 25766c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (isa<PHINode>(I) || !IU.isIVUserOrOperand(I)) 25776c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 25786c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25796c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Ignore users that are part of a SCEV expression. This way we only 25806c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // consider leaf IV Users. This effectively rediscovers a portion of 25816c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // IVUsers analysis but in program order this time. 25826c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (SE.isSCEVable(I->getType()) && !isa<SCEVUnknown>(SE.getSCEV(I))) 25836c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 25846c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25856c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Remove this instruction from any NearUsers set it may be in. 25866c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (unsigned ChainIdx = 0, NChains = IVChainVec.size(); 25876c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainIdx < NChains; ++ChainIdx) { 25886c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec[ChainIdx].NearUsers.erase(I); 25896c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25906c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Search for operands that can be chained. 25916c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*, 4> UniqueOperands; 25926c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick User::op_iterator IVOpEnd = I->op_end(); 25936c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick User::op_iterator IVOpIter = findIVOperand(I->op_begin(), IVOpEnd, L, SE); 25946c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick while (IVOpIter != IVOpEnd) { 25956c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *IVOpInst = cast<Instruction>(*IVOpIter); 25966c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (UniqueOperands.insert(IVOpInst)) 25976c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainInstruction(I, IVOpInst, ChainUsersVec); 25986c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick IVOpIter = findIVOperand(llvm::next(IVOpIter), IVOpEnd, L, SE); 25996c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 26006c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } // Continue walking down the instructions. 26016c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } // Continue walking down the domtree. 26026c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Visit phi backedges to determine if the chain can generate the IV postinc. 26036c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (BasicBlock::iterator I = L->getHeader()->begin(); 26046c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick PHINode *PN = dyn_cast<PHINode>(I); ++I) { 26056c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!SE.isSCEVable(PN->getType())) 26066c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 26076c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 26086c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *IncV = 26096c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick dyn_cast<Instruction>(PN->getIncomingValueForBlock(L->getLoopLatch())); 26106c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (IncV) 26116c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainInstruction(PN, IncV, ChainUsersVec); 26126c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 261322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Remove any unprofitable chains. 261422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick unsigned ChainIdx = 0; 261522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (unsigned UsersIdx = 0, NChains = IVChainVec.size(); 261622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick UsersIdx < NChains; ++UsersIdx) { 261722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!isProfitableChain(IVChainVec[UsersIdx], 261822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ChainUsersVec[UsersIdx].FarUsers, SE, TLI)) 261922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 262022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Preserve the chain at UsesIdx. 262122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (ChainIdx != UsersIdx) 262222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVChainVec[ChainIdx] = IVChainVec[UsersIdx]; 262322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick FinalizeChain(IVChainVec[ChainIdx]); 262422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ++ChainIdx; 262522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 262622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVChainVec.resize(ChainIdx); 262722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 262822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 262922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickvoid LSRInstance::FinalizeChain(IVChain &Chain) { 263022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(!Chain.empty() && "empty IV chains are not allowed"); 263122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DEBUG(dbgs() << "Final Chain: " << *Chain[0].UserInst << "\n"); 263222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 263322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (IVChain::const_iterator I = llvm::next(Chain.begin()), E = Chain.end(); 263422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick I != E; ++I) { 263522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DEBUG(dbgs() << " Inc: " << *I->UserInst << "\n"); 263622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator UseI = 263722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick std::find(I->UserInst->op_begin(), I->UserInst->op_end(), I->IVOperand); 263822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(UseI != I->UserInst->op_end() && "cannot find IV operand"); 263922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVIncSet.insert(UseI); 264022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 264122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 264222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 264322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Return true if the IVInc can be folded into an addressing mode. 264422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst, 264522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *Operand, const TargetLowering *TLI) { 264622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(IncExpr); 264722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!IncConst || !isAddressUse(UserInst, Operand)) 264822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return false; 264922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 265022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IncConst->getValue()->getValue().getMinSignedBits() > 64) 265122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return false; 265222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 265322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick int64_t IncOffset = IncConst->getValue()->getSExtValue(); 265422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!isAlwaysFoldable(IncOffset, /*BaseGV=*/0, /*HaseBaseReg=*/false, 265522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LSRUse::Address, getAccessType(UserInst), TLI)) 265622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return false; 265722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 265822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return true; 265922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 266022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 266122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// GenerateIVChains - Generate an add or subtract for each IVInc in a chain to 266222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// materialize the IV user's operand from the previous IV user's operand. 266322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickvoid LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter, 266422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SmallVectorImpl<WeakVH> &DeadInsts) { 266522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Find the new IVOperand for the head of the chain. It may have been replaced 266622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // by LSR. 266722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const IVInc &Head = Chain[0]; 266822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator IVOpEnd = Head.UserInst->op_end(); 266922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator IVOpIter = findIVOperand(Head.UserInst->op_begin(), 267022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOpEnd, L, SE); 267122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IVSrc = 0; 267222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick while (IVOpIter != IVOpEnd) { 267322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVSrc = getWideOperand(*IVOpIter); 267422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 267522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // If this operand computes the expression that the chain needs, we may use 267622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // it. (Check this after setting IVSrc which is used below.) 267722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // 267822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Note that if Head.IncExpr is wider than IVSrc, then this phi is too 267922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // narrow for the chain, so we can no longer use it. We do allow using a 268022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // wider phi, assuming the LSR checked for free truncation. In that case we 268122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // should already have a truncate on this operand such that 268222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // getSCEV(IVSrc) == IncExpr. 268322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (SE.getSCEV(*IVOpIter) == Head.IncExpr 268422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick || SE.getSCEV(IVSrc) == Head.IncExpr) { 268522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick break; 268622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 268722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOpIter = findIVOperand(llvm::next(IVOpIter), IVOpEnd, L, SE); 268822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 268922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVOpIter == IVOpEnd) { 269022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Gracefully give up on this chain. 269122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n"); 269222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return; 269322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 269422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 269522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n"); 269622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *IVTy = IVSrc->getType(); 269722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *IntTy = SE.getEffectiveSCEVType(IVTy); 269822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEV *LeftOverExpr = 0; 269922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (IVChain::const_iterator IncI = llvm::next(Chain.begin()), 270022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IncE = Chain.end(); IncI != IncE; ++IncI) { 270122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 270222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Instruction *InsertPt = IncI->UserInst; 270322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (isa<PHINode>(InsertPt)) 270422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick InsertPt = L->getLoopLatch()->getTerminator(); 270522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 270622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // IVOper will replace the current IV User's operand. IVSrc is the IV 270722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // value currently held in a register. 270822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IVOper = IVSrc; 270922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!IncI->IncExpr->isZero()) { 271022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // IncExpr was the result of subtraction of two narrow values, so must 271122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // be signed. 271222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEV *IncExpr = SE.getNoopOrSignExtend(IncI->IncExpr, IntTy); 271322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LeftOverExpr = LeftOverExpr ? 271422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr; 271522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 271622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (LeftOverExpr && !LeftOverExpr->isZero()) { 271722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Expand the IV increment. 271822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Rewriter.clearPostInc(); 271922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IncV = Rewriter.expandCodeFor(LeftOverExpr, IntTy, InsertPt); 272022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc), 272122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SE.getUnknown(IncV)); 272222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt); 272322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 272422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // If an IV increment can't be folded, use it as the next IV value. 272522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!canFoldIVIncExpr(LeftOverExpr, IncI->UserInst, IncI->IVOperand, 272622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick TLI)) { 272722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(IVTy == IVOper->getType() && "inconsistent IV increment type"); 272822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVSrc = IVOper; 272922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LeftOverExpr = 0; 273022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 273122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 273222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *OperTy = IncI->IVOperand->getType(); 273322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVTy != OperTy) { 273422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) && 273522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick "cannot extend a chained IV"); 273622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IRBuilder<> Builder(InsertPt); 273722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain"); 273822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 273922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IncI->UserInst->replaceUsesOfWith(IncI->IVOperand, IVOper); 274022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DeadInsts.push_back(IncI->IVOperand); 274122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 274222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // If LSR created a new, wider phi, we may also replace its postinc. We only 274322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // do this if we also found a wide value for the head of the chain. 274422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (isa<PHINode>(Chain.back().UserInst)) { 274522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (BasicBlock::iterator I = L->getHeader()->begin(); 274622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick PHINode *Phi = dyn_cast<PHINode>(I); ++I) { 274722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!isCompatibleIVType(Phi, IVSrc)) 274822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 274922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Instruction *PostIncV = dyn_cast<Instruction>( 275022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Phi->getIncomingValueForBlock(L->getLoopLatch())); 275122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc))) 275222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 275322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IVOper = IVSrc; 275422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *PostIncTy = PostIncV->getType(); 275522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVTy != PostIncTy) { 275622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types"); 275722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IRBuilder<> Builder(L->getLoopLatch()->getTerminator()); 275822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc()); 275922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain"); 276022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 276122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Phi->replaceUsesOfWith(PostIncV, IVOper); 276222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DeadInsts.push_back(PostIncV); 276322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 276422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 27656c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 27666c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 2767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectFixupsAndInitialFormulae() { 2768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 276922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Instruction *UserInst = UI->getUser(); 277022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Skip IV users that are part of profitable IV Chains. 277122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator UseI = std::find(UserInst->op_begin(), UserInst->op_end(), 277222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick UI->getOperandValToReplace()); 277322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(UseI != UserInst->op_end() && "cannot find IV operand"); 277422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVIncSet.count(UseI)) 277522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 277622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 2777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record the uses. 2778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 277922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LF.UserInst = UserInst; 2780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI->getOperandValToReplace(); 2781448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.PostIncLoops = UI->getPostIncLoops(); 27820f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel 2783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind = LSRUse::Basic; 2784db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = 0; 2785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAddressUse(LF.UserInst, LF.OperandValToReplace)) { 2786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::Address; 2787572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = getAccessType(LF.UserInst); 2788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2789572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2790c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *S = IU.getExpr(*UI); 2791572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2792572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Equality (== and !=) ICmps are special. We can rewrite (i == N) as 2793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // (N - i == 0), and this allows (N - i) to be the expression that we work 2794572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with rather than just N or i, so we can consider the register 2795572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // requirements for both N and i at the same time. Limiting this code to 2796572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps is not a problem because all interesting loops use 2797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps, thanks to IndVarSimplify. 2798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpInst *CI = dyn_cast<ICmpInst>(LF.UserInst)) 2799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (CI->isEquality()) { 2800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Swap the operands if needed to put the OperandValToReplace on the 2801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // left, for consistency. 2802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *NV = CI->getOperand(1); 2803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NV == LF.OperandValToReplace) { 2804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, CI->getOperand(0)); 2805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(0, NV); 2806f182b23f8fed014c26da48d94890080c7d916a0cDan Gohman NV = CI->getOperand(1); 28079da1bf4845e670096b9bf9e62c40960af1697ea0Dan Gohman Changed = true; 2808572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // x == y --> x - y == 0 2811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *N = SE.getSCEV(NV); 281217ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (SE.isLoopInvariant(N, L)) { 2813673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman // S is normalized, so normalize N before folding it into S 2814673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman // to keep the result normalized. 2815673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman N = TransformForPostIncUse(Normalize, N, CI, 0, 2816673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman LF.PostIncLoops, SE, DT); 2817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::ICmpZero; 2818572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman S = SE.getMinusSCEV(N, S); 2819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2820572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2821572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // -1 and the negations of all interesting strides (except the negation 2822572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of -1) are now also interesting. 2823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Factors.size(); i != e; ++i) 2824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors[i] != -1) 2825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-(uint64_t)Factors[i]); 2826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-1); 2827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Set up the initial formula for this use. 2830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy); 2831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 2832572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 2833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 2834448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 2835a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 2836a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 2837a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 2838a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 2839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2840572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is the first use of this LSRUse, give it a formula. 2841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Formulae.empty()) { 2842454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman InsertInitialFormula(S, LU, LF.LUIdx); 2843572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), LF.LUIdx); 2844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2845572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_fixups(dbgs())); 2848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2849572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 285076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertInitialFormula - Insert a formula for the given expression into 285176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given use, separating out loop-variant portions from loop-invariant 285276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// and loop-computable portions. 2853572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2854454d26dc43207ec537d843229db6f5e6a302e23dDan GohmanLSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { 2855572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2856dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman F.InitialMatch(S, L, SE); 2857572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Initial formula already exists!"); (void)Inserted; 2859572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 286176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertSupplementalFormula - Insert a simple single-register formula for 286276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given expression into the given use. 2863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2864572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::InsertSupplementalFormula(const SCEV *S, 2865572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU, size_t LUIdx) { 2866572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2867572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(S); 2868572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.HasBaseReg = true; 2869572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Supplemental formula already exists!"); (void)Inserted; 2871572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2872572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2873572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CountRegisters - Note which registers are used by the given formula, 2874572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// updating RegUses. 2875572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) { 2876572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) 2877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(F.ScaledReg, LUIdx); 2878572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 2879572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) 2880572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(*I, LUIdx); 2881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2882572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 2884572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 2885572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) { 2886454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!LU.InsertFormula(F)) 28877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 288880b0f8c0628d848d36f04440687b61e3b2d3dff1Dan Gohman 2889572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(F, LUIdx); 2890572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 2891572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2892572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2893572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectLoopInvariantFixupsAndFormulae - Check for other uses of 2894572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-invariant values which we're tracking. These other uses will pin these 2895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// values in registers, making them less profitable for elimination. 2896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: This currently misses non-constant addrec step registers. 2897572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Should this give more weight to users inside the loop? 2898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2899572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::CollectLoopInvariantFixupsAndFormulae() { 2900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); 2901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 8> Inserted; 2902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!Worklist.empty()) { 2904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *S = Worklist.pop_back_val(); 2905572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2906572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S)) 2907403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(N->op_begin(), N->op_end()); 2908572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S)) 2909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(C->getOperand()); 2910572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) { 2911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getLHS()); 2912572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getRHS()); 2913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 2914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Inserted.insert(U)) continue; 2915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Value *V = U->getValue(); 2916a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 2917a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Look for instructions defined outside the loop. 2918572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (L->contains(Inst)) continue; 2919a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman } else if (isa<UndefValue>(V)) 2920a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Undef doesn't have a live range, so it doesn't matter. 2921a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman continue; 292260ad781c61815ca5b8dc2a45a102e1c8af65992fGabor Greif for (Value::const_use_iterator UI = V->use_begin(), UE = V->use_end(); 2923572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != UE; ++UI) { 2924572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Instruction *UserInst = dyn_cast<Instruction>(*UI); 2925572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore non-instructions. 2926572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!UserInst) 2927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions in other functions (as can happen with 2929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Constants). 2930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UserInst->getParent()->getParent() != L->getHeader()->getParent()) 2931572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2932572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions not dominated by the loop. 2933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const BasicBlock *UseBB = !isa<PHINode>(UserInst) ? 2934572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UserInst->getParent() : 2935572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman cast<PHINode>(UserInst)->getIncomingBlock( 2936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode::getIncomingValueNumForOperand(UI.getOperandNo())); 2937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(L->getHeader(), UseBB)) 2938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore uses which are part of other SCEV expressions, to avoid 2940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // analyzing them multiple times. 29414a2a68336638e99a475b09a9278399db6749618fDan Gohman if (SE.isSCEVable(UserInst->getType())) { 29424a2a68336638e99a475b09a9278399db6749618fDan Gohman const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); 29434a2a68336638e99a475b09a9278399db6749618fDan Gohman // If the user is a no-op, look through to its uses. 29444a2a68336638e99a475b09a9278399db6749618fDan Gohman if (!isa<SCEVUnknown>(UserS)) 29454a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 29464a2a68336638e99a475b09a9278399db6749618fDan Gohman if (UserS == U) { 29474a2a68336638e99a475b09a9278399db6749618fDan Gohman Worklist.push_back( 29484a2a68336638e99a475b09a9278399db6749618fDan Gohman SE.getUnknown(const_cast<Instruction *>(UserInst))); 29494a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 29504a2a68336638e99a475b09a9278399db6749618fDan Gohman } 29514a2a68336638e99a475b09a9278399db6749618fDan Gohman } 2952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore icmp instructions which are already being analyzed. 2953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) { 2954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned OtherIdx = !UI.getOperandNo(); 2955572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx)); 295617ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L)) 2957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2958572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2959572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 2961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst = const_cast<Instruction *>(UserInst); 2962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI.getUse(); 2963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, LSRUse::Basic, 0); 2964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 2965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 2966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 2967448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 2968a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 2969a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 2970a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 2971a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 2972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman InsertSupplementalFormula(U, LU, LF.LUIdx); 2973572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), Uses.size() - 1); 2974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 2975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2976572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2980572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectSubexprs - Split S into subexpressions which can be pulled out into 2981572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// separate registers. If C is non-null, multiply each subexpression by C. 2982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic void CollectSubexprs(const SCEV *S, const SCEVConstant *C, 2983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Ops, 29843e3f15bb09eaca4d259e531c9a1ecafb5710b57bDan Gohman const Loop *L, 2985572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE) { 2986572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 2987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break out add operands. 2988572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 2989572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) 29903e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(*I, C, Ops, L, SE); 2991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 2993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Split a non-zero base out of an addrec. 2994572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!AR->getStart()->isZero()) { 2995deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman CollectSubexprs(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), 2996572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE), 29973228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick AR->getLoop(), 29983228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick //FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 29993228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap), 30003e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman C, Ops, L, SE); 30013e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(AR->getStart(), C, Ops, L, SE); 3002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 3003572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break (C * (a + b + c)) into C*a + C*b + C*c. 3006572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mul->getNumOperands() == 2) 3007572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *Op0 = 3008572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<SCEVConstant>(Mul->getOperand(0))) { 3009572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectSubexprs(Mul->getOperand(1), 3010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0, 30113e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman Ops, L, SE); 3012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 3013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3014572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3015572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 30163e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // Otherwise use the value itself, optionally with a scale applied. 30173e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman Ops.push_back(C ? SE.getMulExpr(C, S) : S); 3018572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3019572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3020572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateReassociations - Split out subexpressions from adds and the bases of 3021572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrecs. 3022572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx, 3023572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base, 3024572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth) { 3025572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Arbitrarily cap recursion to protect compile time. 3026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Depth >= 3) return; 3027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 3029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = Base.BaseRegs[i]; 3030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 30313e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman SmallVector<const SCEV *, 8> AddOps; 30323e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(BaseReg, 0, AddOps, L, SE); 30333e3f15bb09eaca4d259e531c9a1ecafb5710b57bDan Gohman 3034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddOps.size() == 1) continue; 3035572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3036572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), 3037572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = AddOps.end(); J != JE; ++J) { 30383e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 30393e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // Loop-variant "unknown" values are uninteresting; we won't be able to 30403e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // do anything meaningful with them. 304117ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L)) 30423e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman continue; 30433e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 3044572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't pull a constant into a register if the constant could be folded 3045572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // into an immediate field. 3046572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAlwaysFoldable(*J, LU.MinOffset, LU.MaxOffset, 3047572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 3048572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 3049572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3050572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect all operands except *J. 3052403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman SmallVector<const SCEV *, 8> InnerAddOps 30534eaee28e347f33b3e23d2fb7275fc5222d5482fdDan Gohman (((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); 3054403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman InnerAddOps.append 3055ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes (llvm::next(J), ((const SmallVector<const SCEV *, 8> &)AddOps).end()); 3056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3057572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't leave just a constant behind in a register if the constant could 3058572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // be folded into an immediate field. 3059572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InnerAddOps.size() == 1 && 3060572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isAlwaysFoldable(InnerAddOps[0], LU.MinOffset, LU.MaxOffset, 3061572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 3062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 3063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3064572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3065fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); 3066fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman if (InnerSum->isZero()) 3067fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman continue; 3068572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3069cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3070cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Add the remaining pieces of the add back into the new formula. 3071cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum); 3072cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (TLI && InnerSumSC && 3073cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 && 3074cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman TLI->isLegalAddImmediate((uint64_t)F.UnfoldedOffset + 3075cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman InnerSumSC->getValue()->getZExtValue())) { 3076cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset + 3077cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman InnerSumSC->getValue()->getZExtValue(); 3078cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs.erase(F.BaseRegs.begin() + i); 3079cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } else 3080cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs[i] = InnerSum; 3081cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3082cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Add J as its own register, or an unfolded immediate. 3083cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J); 3084cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (TLI && SC && SE.getTypeSizeInBits(SC->getType()) <= 64 && 3085cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman TLI->isLegalAddImmediate((uint64_t)F.UnfoldedOffset + 3086cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SC->getValue()->getZExtValue())) 3087cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset + 3088cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SC->getValue()->getZExtValue(); 3089cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman else 3090cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs.push_back(*J); 3091cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InsertFormula(LU, LUIdx, F)) 3093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If that formula hadn't been seen before, recurse to find more like 3094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // it. 3095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth+1); 3096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3097572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCombinations - Generate a formula consisting of all of the 3101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-dominating registers added into a single register. 3102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx, 3103441a38993e89104c424e00c95cce63c7351f4fc3Dan Gohman Formula Base) { 31043f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // This method is only interesting on a plurality of registers. 3105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.BaseRegs.size() <= 1) return; 3106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.clear(); 3109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 3110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 3111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Base.BaseRegs.begin(), E = Base.BaseRegs.end(); I != E; ++I) { 3112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 3113dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(BaseReg, L->getHeader()) && 311417ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman !SE.hasComputableLoopEvolution(BaseReg, L)) 3115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(BaseReg); 3116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 3117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(BaseReg); 3118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Ops.size() > 1) { 3120ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman const SCEV *Sum = SE.getAddExpr(Ops); 3121ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // TODO: If Sum is zero, it probably means ScalarEvolution missed an 3122ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // opportunity to fold something. For now, just ignore such cases 31233f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // rather than proceed with zero in a register. 3124ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman if (!Sum->isZero()) { 3125ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman F.BaseRegs.push_back(Sum); 3126ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3127ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman } 3128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateSymbolicOffsets - Generate reuse formulae using symbolic offsets. 3132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, 3133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 3134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We can't add a symbolic offset if the address already contains one. 3135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 3136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3137572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 3138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 3139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *GV = ExtractSymbol(G, SE); 3140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || !GV) 3141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseGV = GV; 3144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 3145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 3146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3147572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 3148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets. 3153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, 3154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 3155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: For now, just add the min and max offset, because it usually isn't 3156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // worthwhile looking at everything inbetween. 3157c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman SmallVector<int64_t, 2> Worklist; 3158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MinOffset); 3159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MaxOffset != LU.MinOffset) 3160572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MaxOffset); 3161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 3163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 3164572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Worklist.begin(), 3166572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Worklist.end(); I != E; ++I) { 3167572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)Base.AM.BaseOffs - *I; 3169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(F.AM, LU.MinOffset - *I, LU.MaxOffset - *I, 3170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 3171c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // Add the offset to the base register. 31724065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G); 3173c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // If it cancelled out, drop the base register, otherwise update it. 3174c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman if (NewG->isZero()) { 3175c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman std::swap(F.BaseRegs[i], F.BaseRegs.back()); 3176c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs.pop_back(); 3177c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman } else 3178c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs[i] = NewG; 3179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3183572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(G, SE); 3185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || Imm == 0) 3186572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Imm; 3189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 3190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 3191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 3193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateICmpZeroScales - For ICmpZero, check to see if we can scale up 3198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the comparison. For example, x == y -> x*c == y*c. 3199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, 3200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 3201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != LSRUse::ICmpZero) return; 3202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 3204db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = Base.getType(); 3205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 3206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(IntTy) > 64) return; 3207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't do this if there is more than one offset. 3209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MinOffset != LU.MaxOffset) return; 3210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Base.AM.BaseGV && "ICmpZero use is not legal!"); 3212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 3214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 3215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 3216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 3217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that the multiplication doesn't overflow. 32192ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (Base.AM.BaseOffs == INT64_MIN && Factor == -1) 3220968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 32212ea09e05466613a22e1211f52c30cd01af563983Dan Gohman int64_t NewBaseOffs = (uint64_t)Base.AM.BaseOffs * Factor; 32222ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (NewBaseOffs / Factor != Base.AM.BaseOffs) 3223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the use offset doesn't overflow. 3226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = LU.MinOffset; 3227968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman if (Offset == INT64_MIN && Factor == -1) 3228968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 3229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = (uint64_t)Offset * Factor; 3230378c0b35a7b9884f1de2a9762825424fbf1813acDan Gohman if (Offset / Factor != LU.MinOffset) 3231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 32332ea09e05466613a22e1211f52c30cd01af563983Dan Gohman Formula F = Base; 32342ea09e05466613a22e1211f52c30cd01af563983Dan Gohman F.AM.BaseOffs = NewBaseOffs; 32352ea09e05466613a22e1211f52c30cd01af563983Dan Gohman 3236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that this scale is legal. 3237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, Offset, Offset, LU.Kind, LU.AccessTy, TLI)) 3238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compensate for the use having MinOffset built into it. 3241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Offset - LU.MinOffset; 3242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3243deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 3244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with each base register doesn't overflow. 3246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) { 3247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS); 3248f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i]) 3249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto next; 3250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the scaled register doesn't overflow. 3253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) { 3254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS); 3255f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg) 3256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3259cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Check that multiplying with the unfolded offset doesn't overflow. 3260cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (F.UnfoldedOffset != 0) { 32611b58d4536a561f28bf935dcb29b483c52a6bf4c6Dan Gohman if (F.UnfoldedOffset == INT64_MIN && Factor == -1) 32621b58d4536a561f28bf935dcb29b483c52a6bf4c6Dan Gohman continue; 3263cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset * Factor; 3264cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (F.UnfoldedOffset / Factor != Base.UnfoldedOffset) 3265cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman continue; 3266cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 3267cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we make it here and it's legal, add it. 3269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman next:; 3271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateScales - Generate stride factor reuse formulae by making use of 3275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// scaled-offset address modes, for example. 3276ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) { 3277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 3278db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = Base.getType(); 3279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 3280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this Formula already has a scaled register, we can't add another one. 3282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.Scale != 0) return; 3283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 3285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 3286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 3287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 3288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.Scale = Factor; 3290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.HasBaseReg = Base.BaseRegs.size() > 1; 3291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check whether this scale is going to be legal. 3292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 3293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 3294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // As a special-case, handle special out-of-loop Basic users specially. 3295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Reconsider this special case. 3296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::Basic && 3297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 3298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::Special, LU.AccessTy, TLI) && 3299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.AllFixupsOutsideLoop) 3300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind = LSRUse::Special; 3301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 3302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For an ICmpZero, negating a solitary base register won't lead to 3305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // new solutions. 3306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero && 3307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !Base.AM.HasBaseReg && Base.AM.BaseOffs == 0 && !Base.AM.BaseGV) 3308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For each addrec base reg, apply the scale, if possible. 3310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) 3311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = 3312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i])) { 3313deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 3314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FactorS->isZero()) 3315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3316572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Divide out the factor, ignoring high bits, since we'll be 3317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scaling the value back up in the end. 3318f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) { 3319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This could be optimized to avoid all the copying. 3320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = Quotient; 33225ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman F.DeleteBaseReg(F.BaseRegs[i]); 3323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateTruncates - Generate reuse formulae from different IV types. 3330ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) { 3331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This requires TargetLowering to tell us which truncates are free. 3332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI) return; 3333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't bother truncating symbolic values. 3335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 3336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 3338db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *DstTy = Base.getType(); 3339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DstTy) return; 3340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DstTy = SE.getEffectiveSCEVType(DstTy); 3341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3342db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner for (SmallSetVector<Type *, 4>::const_iterator 3343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 3344db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *SrcTy = *I; 3345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SrcTy != DstTy && TLI->isTruncateFree(SrcTy, DstTy)) { 3346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, *I); 3349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(), 3350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = F.BaseRegs.end(); J != JE; ++J) 3351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *J = SE.getAnyExtendExpr(*J, SrcTy); 3352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This assumes we've done basic processing on all uses and 3354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // have an idea what the register usage is. 3355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses)) 3356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 3364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 33656020d85c41987b0b7890d91bf66187aac6e8a3a1Dan Gohman/// WorkItem - Helper class for GenerateCrossUseConstantOffsets. It's used to 3366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// defer modifications so that the search phase doesn't have to worry about 3367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the data structures moving underneath it. 3368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct WorkItem { 3369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 3370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm; 3371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg; 3372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WorkItem(size_t LI, int64_t I, const SCEV *R) 3374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : LUIdx(LI), Imm(I), OrigReg(R) {} 3375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 3377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 3378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 3379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::print(raw_ostream &OS) const { 3383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx 3384572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << " , add offset " << Imm; 3385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::dump() const { 3388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 3389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCrossUseConstantOffsets - Look for registers which are a constant 3392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// distance apart and try to form reuse opportunities between them. 3393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCrossUseConstantOffsets() { 3394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Group the registers by their value without any added constant offset. 3395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef std::map<int64_t, const SCEV *> ImmMapTy; 3396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy; 3397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegMapTy Map; 3398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; 3399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Sequence; 3400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 3401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 3402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(Reg, SE); 3404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegMapTy::iterator, bool> Pair = 3405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.insert(std::make_pair(Reg, ImmMapTy())); 3406572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 3407572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.push_back(Reg); 3408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pair.first->second.insert(std::make_pair(Imm, *I)); 3409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(*I); 3410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now examine each set of registers with the same base value. Build up 3413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // a list of work to do and do the work in a separate step so that we're 3414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // not adding formulae and register counts while we're searching. 3415191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallVector<WorkItem, 32> WorkItems; 3416191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems; 3417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(), 3418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Sequence.end(); I != E; ++I) { 3419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const ImmMapTy &Imms = Map.find(Reg)->second; 3421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3422cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman // It's not worthwhile looking for reuse if there's only one offset. 3423cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (Imms.size() == 1) 3424cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman continue; 3425cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman 3426572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':'; 3427572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 3428572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 3429572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << J->first; 3430572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 3431572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Examine each offset. 3433572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 3434572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) { 3435572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = J->second; 3436572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3437572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t JImm = J->first; 3438572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg); 3439572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3440572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isa<SCEVConstant>(OrigReg) && 3441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg].count() == 1) { 3442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n'); 3443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively examine offsets between this orig reg a few selected 3447572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // other orig regs. 3448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator OtherImms[] = { 3449572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imms.begin(), prior(Imms.end()), 3450cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Imms.lower_bound((Imms.begin()->first + prior(Imms.end())->first) / 2) 3451572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 3452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = array_lengthof(OtherImms); i != e; ++i) { 3453572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator M = OtherImms[i]; 3454cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (M == J || M == JE) continue; 3455572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compute the difference between the two. 3457572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = (uint64_t)JImm - M->first; 3458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (int LUIdx = UsedByIndices.find_first(); LUIdx != -1; 3459191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LUIdx = UsedByIndices.find_next(LUIdx)) 3460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Make a memo of this use, offset, and register tuple. 3461191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (UniqueItems.insert(std::make_pair(LUIdx, Imm))) 3462191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg)); 3463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.clear(); 3468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.clear(); 3469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap.clear(); 3470191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman UniqueItems.clear(); 3471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3472572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now iterate through the worklist and add new formulae. 3473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<WorkItem>::const_iterator I = WorkItems.begin(), 3474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = WorkItems.end(); I != E; ++I) { 3475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const WorkItem &WI = *I; 3476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = WI.LUIdx; 3477572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = WI.Imm; 3479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = WI.OrigReg; 3480572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3481db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType()); 3482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); 3483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BitWidth = SE.getTypeSizeInBits(IntTy); 3484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 34853f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // TODO: Use a more targeted data structure. 3486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) { 34879f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const Formula &F = LU.Formulae[L]; 3488572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in the scaled register. 3489572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg == OrigReg) { 3490572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offs = (uint64_t)F.AM.BaseOffs + 3491572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imm * (uint64_t)F.AM.Scale; 3492572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't create 50 + reg(-50). 3493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.referencesReg(SE.getSCEV( 3494572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt::get(IntTy, -(uint64_t)Offs)))) 3495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3496572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 3497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = Offs; 3498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 3499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 3500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg); 3502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new scale is a constant in a register, and adding the constant 3504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // value to the immediate would produce a value closer to zero than the 3505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // immediate itself, then the formula isn't worthwhile. 3506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg)) 3507c73b24db5f6226ed44ebc44ce1c25bb357206623Chris Lattner if (C->getValue()->isNegative() != 3508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (NewF.AM.BaseOffs < 0) && 3509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (C->getValue()->getValue().abs() * APInt(BitWidth, F.AM.Scale)) 3510e05678132345eb8a632362dbd320ee7d36226e67Dan Gohman .ule(abs64(NewF.AM.BaseOffs))) 3511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // OK, looks good. 3514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 3515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in a base register. 3517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) { 3518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = F.BaseRegs[N]; 3519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseReg != OrigReg) 3520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 3522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = (uint64_t)NewF.AM.BaseOffs + Imm; 3523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 3524cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 3525cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (!TLI || 3526cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman !TLI->isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm)) 3527cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman continue; 3528cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NewF = F; 3529cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NewF.UnfoldedOffset = (uint64_t)NewF.UnfoldedOffset + Imm; 3530cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 3531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg); 3532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new formula has a constant in a register, and adding the 3534572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // constant value to the immediate would produce a value closer to 3535572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // zero than the immediate itself, then the formula isn't worthwhile. 3536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 3537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J = NewF.BaseRegs.begin(), JE = NewF.BaseRegs.end(); 3538572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 3539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*J)) 3540360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman if ((C->getValue()->getValue() + NewF.AM.BaseOffs).abs().slt( 3541360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman abs64(NewF.AM.BaseOffs)) && 3542360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman (C->getValue()->getValue() + 3543360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman NewF.AM.BaseOffs).countTrailingZeros() >= 3544360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman CountTrailingZeros_64(NewF.AM.BaseOffs)) 3545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto skip_formula; 3546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3547572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, looks good. 3548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 3549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 3550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman skip_formula:; 3551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateAllReuseFormulae - Generate formulae for each use. 3558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 3559572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::GenerateAllReuseFormulae() { 3560c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman // This is split into multiple loops so that hasRegsUsedByUsesOtherThan 3561572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // queries are more precise. 3562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3563572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae[i]); 3566572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCombinations(LU, LUIdx, LU.Formulae[i]); 3568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]); 3573572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3574572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]); 3575572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]); 3577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3578572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateScales(LU, LUIdx, LU.Formulae[i]); 3579c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman } 3580c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3581c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman LSRUse &LU = Uses[LUIdx]; 3582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateTruncates(LU, LUIdx, LU.Formulae[i]); 3584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCrossUseConstantOffsets(); 35873902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman 35883902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman DEBUG(dbgs() << "\n" 35893902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman "After generating reuse formulae:\n"; 35903902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman print_uses(dbgs())); 3591572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3593f63d70f218807d7522e8bbe2d9e887ff3ea87b24Dan Gohman/// If there are multiple formulae with the same set of registers used 3594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// by other uses, pick the best one and delete the others. 3595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::FilterOutUndesirableDedicatedRegisters() { 3596fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman DenseSet<const SCEV *> VisitedRegs; 3597fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman SmallPtrSet<const SCEV *, 16> Regs; 35988a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> LoserRegs; 3599572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 3600c6519f916b5922de81c53547fd21364994195a70Dan Gohman bool ChangedFormulae = false; 3601572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 3602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect the best formula for each unique set of shared registers. This 3604572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // is reset for each use. 3605572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<SmallVector<const SCEV *, 2>, size_t, UniquifierDenseMapInfo> 3606572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy; 3607572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy BestFormulae; 3608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3611ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n'); 3612572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3613b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 3614572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t FIdx = 0, NumForms = LU.Formulae.size(); 3615572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FIdx != NumForms; ++FIdx) { 3616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[FIdx]; 3617572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 36188a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // Some formulas are instant losers. For example, they may depend on 36198a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // nonexistent AddRecs from other loops. These need to be filtered 36208a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // immediately, otherwise heuristics could choose them over others leading 36218a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // to an unsatisfactory solution. Passing LoserRegs into RateFormula here 36228a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // avoids the need to recompute this information across formulae using the 36238a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // same bad AddRec. Passing LoserRegs is also essential unless we remove 36248a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // the corresponding bad register from the Regs set. 36258a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Cost CostF; 36268a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Regs.clear(); 36278a5d792944582de8e63e96440dbd2cde754351adAndrew Trick CostF.RateFormula(F, Regs, VisitedRegs, L, LU.Offsets, SE, DT, 36288a5d792944582de8e63e96440dbd2cde754351adAndrew Trick &LoserRegs); 36298a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (CostF.isLoser()) { 36308a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // During initial formula generation, undesirable formulae are generated 36318a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // by uses within other loops that have some non-trivial address mode or 36328a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // use the postinc form of the IV. LSR needs to provide these formulae 36338a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // as the basis of rediscovering the desired formula that uses an AddRec 36348a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // corresponding to the existing phi. Once all formulae have been 36358a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // generated, these initial losers may be pruned. 36368a5d792944582de8e63e96440dbd2cde754351adAndrew Trick DEBUG(dbgs() << " Filtering loser "; F.print(dbgs()); 36378a5d792944582de8e63e96440dbd2cde754351adAndrew Trick dbgs() << "\n"); 3638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 36398a5d792944582de8e63e96440dbd2cde754351adAndrew Trick else { 36408a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallVector<const SCEV *, 2> Key; 36418a5d792944582de8e63e96440dbd2cde754351adAndrew Trick for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(), 36428a5d792944582de8e63e96440dbd2cde754351adAndrew Trick JE = F.BaseRegs.end(); J != JE; ++J) { 36438a5d792944582de8e63e96440dbd2cde754351adAndrew Trick const SCEV *Reg = *J; 36448a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx)) 36458a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Key.push_back(Reg); 36468a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 36478a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (F.ScaledReg && 36488a5d792944582de8e63e96440dbd2cde754351adAndrew Trick RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx)) 36498a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Key.push_back(F.ScaledReg); 36508a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // Unstable sort by host order ok, because this is only used for 36518a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // uniquifying. 36528a5d792944582de8e63e96440dbd2cde754351adAndrew Trick std::sort(Key.begin(), Key.end()); 36538a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 36548a5d792944582de8e63e96440dbd2cde754351adAndrew Trick std::pair<BestFormulaeTy::const_iterator, bool> P = 36558a5d792944582de8e63e96440dbd2cde754351adAndrew Trick BestFormulae.insert(std::make_pair(Key, FIdx)); 36568a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (P.second) 36578a5d792944582de8e63e96440dbd2cde754351adAndrew Trick continue; 36588a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 3659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &Best = LU.Formulae[P.first->second]; 3660fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman 3661fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Cost CostBest; 3662fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Regs.clear(); 36638a5d792944582de8e63e96440dbd2cde754351adAndrew Trick CostBest.RateFormula(Best, Regs, VisitedRegs, L, LU.Offsets, SE, DT); 3664fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman if (CostF < CostBest) 3665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::swap(F, Best); 36666458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs()); 3667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 36686458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman " in favor of formula "; Best.print(dbgs()); 3669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 36708a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 3671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 36728a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ChangedFormulae = true; 3673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 36748a5d792944582de8e63e96440dbd2cde754351adAndrew Trick LU.DeleteFormula(F); 36758a5d792944582de8e63e96440dbd2cde754351adAndrew Trick --FIdx; 36768a5d792944582de8e63e96440dbd2cde754351adAndrew Trick --NumForms; 36778a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Any = true; 367859dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman } 367959dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 368057aaa0b264d9d23d9a14235334b15d4117e32a3bDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 3681b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 3682b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 368359dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 368459dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman // Reset this to prepare for the next use. 3685572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulae.clear(); 3686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3687572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3688c6519f916b5922de81c53547fd21364994195a70Dan Gohman DEBUG(if (ChangedFormulae) { 36899214b82c5446791b280821bbd892dca633130f80Dan Gohman dbgs() << "\n" 36909214b82c5446791b280821bbd892dca633130f80Dan Gohman "After filtering out undesirable candidates:\n"; 3691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs()); 3692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 3693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3695d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman// This is a rough guess that seems to work fairly well. 3696d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmanstatic const size_t ComplexityLimit = UINT16_MAX; 3697d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 3698d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// EstimateSearchSpaceComplexity - Estimate the worst-case number of 3699d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// solutions the solver might have to consider. It almost never considers 3700d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// this many solutions because it prune the search space, but the pruning 3701d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// isn't always sufficient. 3702d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmansize_t LSRInstance::EstimateSearchSpaceComplexity() const { 37030d6715a413a23c021d1042719888966bb8d11872Dan Gohman size_t Power = 1; 3704d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 3705d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman E = Uses.end(); I != E; ++I) { 3706d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t FSize = I->Formulae.size(); 3707d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (FSize >= ComplexityLimit) { 3708d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power = ComplexityLimit; 3709d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 3710d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 3711d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power *= FSize; 3712d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (Power >= ComplexityLimit) 3713d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 3714d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 3715d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman return Power; 3716d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman} 3717d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 37184aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByDetectingSupersets - When one formula uses a superset 37194aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of the registers of another formula, it won't help reduce register 37204aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// pressure (though it may not necessarily hurt register pressure); remove 37214aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// it to simplify the system. 37224aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByDetectingSupersets() { 3723a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3724a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3725a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3726a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by eliminating formulae " 3727a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "which use a superset of registers used by other " 3728a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "formulae.\n"); 3729a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3730a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3731a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 3732a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool Any = false; 3733a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 3734a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula &F = LU.Formulae[i]; 3735f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // Look for a formula with a constant or GV in a register. If the use 3736f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // also has a formula with that same value in an immediate field, 3737f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // delete the one that uses a register. 3738a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 3739a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) { 3740a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) { 3741a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 3742a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseOffs += C->getValue()->getSExtValue(); 3743a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 3744a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 3745a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 3746a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 3747a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 3748a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 3749a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 3750a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 3751a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3752a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3753a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) { 3754a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) 3755a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!F.AM.BaseGV) { 3756a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 3757a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseGV = GV; 3758a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 3759a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 3760a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 3761a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 3762a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 3763a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 3764a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 3765a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 3766a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 3767a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3768a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3769a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3770a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3771a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3772a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3773a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (Any) 3774a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.RecomputeRegs(LUIdx, RegUses); 3775a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3776a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3777a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3778a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 3779a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 37804aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 3781a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 37824aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByCollapsingUnrolledCode - When there are many registers 37834aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// for expressions like A, A+1, A+2, etc., allocate a single register for 37844aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// them. 37854aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() { 3786a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3787a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3788a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3789a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming that uses " 3790a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "separated by a constant offset will use the same " 3791a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "registers.\n"); 3792a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3793f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // This is especially useful for unrolled loops. 3794f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman 3795a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3796a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 3797402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 3798402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 3799402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 3800a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.AM.BaseOffs != 0 && F.AM.Scale == 0) { 3801191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU)) { 3802191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(*LUThatHas, F.AM.BaseOffs, 3803a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman /*HasBaseReg=*/false, 3804a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind, LU.AccessTy)) { 3805a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); 3806a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 3807a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3808a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop; 3809a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3810191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Update the relocs to reference the new use. 3811191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman for (SmallVectorImpl<LSRFixup>::iterator I = Fixups.begin(), 3812191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman E = Fixups.end(); I != E; ++I) { 3813191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRFixup &Fixup = *I; 3814191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == LUIdx) { 3815191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUThatHas - &Uses.front(); 3816191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.Offset += F.AM.BaseOffs; 3817dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman // Add the new offset to LUThatHas' offset list. 3818dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (LUThatHas->Offsets.back() != Fixup.Offset) { 3819dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->Offsets.push_back(Fixup.Offset); 3820dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset > LUThatHas->MaxOffset) 3821dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MaxOffset = Fixup.Offset; 3822dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset < LUThatHas->MinOffset) 3823dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MinOffset = Fixup.Offset; 3824dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman } 3825191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman DEBUG(dbgs() << "New fixup has offset " 3826191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman << Fixup.Offset << '\n'); 3827191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3828191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == NumUses-1) 3829191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUIdx; 3830191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3831191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 3832c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman // Delete formulae from the new use which are no longer legal. 3833c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman bool Any = false; 3834c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) { 3835c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Formula &F = LUThatHas->Formulae[i]; 3836c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (!isLegalUse(F.AM, 3837c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->MinOffset, LUThatHas->MaxOffset, 3838c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->Kind, LUThatHas->AccessTy, TLI)) { 3839c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 3840c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman dbgs() << '\n'); 3841c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->DeleteFormula(F); 3842c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --i; 3843c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --e; 3844c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Any = true; 3845c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3846c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3847c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (Any) 3848c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses); 3849c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman 3850a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Delete the old use. 3851c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman DeleteUse(LU, LUIdx); 3852a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --LUIdx; 3853a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --NumUses; 3854a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3855a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3856a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3857a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3858a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3859a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3860a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3861a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3862a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 3863a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 38644aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 3865a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 38663228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick/// NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters - Call 38674f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// FilterOutUndesirableDedicatedRegisters again, if necessary, now that 38684f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// we've done more filtering, as it may be able to find more formulae to 38694f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// eliminate. 38704f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohmanvoid LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){ 38714f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 38724f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 38734f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 38744f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "Narrowing the search space by re-filtering out " 38754f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman "undesirable dedicated registers.\n"); 38764f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 38774f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman FilterOutUndesirableDedicatedRegisters(); 38784f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 38794f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 38804f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman print_uses(dbgs())); 38814f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman } 38824f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman} 38834f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 38844aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByPickingWinnerRegs - Pick a register which seems likely 38854aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// to be profitable, and then in any use which has any reference to that 38864aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// register, delete all formulae which do not reference that register. 38874aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() { 388876c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // With all other options exhausted, loop until the system is simple 388976c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // enough to handle. 3890572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Taken; 3891d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman while (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3892572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we have too many of formulae on our hands to conveniently handle. 3893572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use a rough heuristic to thin out the list. 38940da751baf72542ab66518e4549e39da5f34216b4Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Pick the register which is used by the most LSRUses, which is likely 3897572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to be a good reuse register candidate. 3898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Best = 0; 3899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BestNum = 0; 3900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 3901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 3902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Taken.count(Reg)) 3904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3905572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Best) 3906572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3907572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 3908572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Count = RegUses.getUsedByIndices(Reg).count(); 3909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Count > BestNum) { 3910572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestNum = Count; 3912572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3916572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best 39173f46a3abeedba8d517b4182de34c821d752db058Dan Gohman << " will yield profitable reuse.\n"); 3918572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Taken.insert(Best); 3919572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // In any use with formulae which references this register, delete formulae 3921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which don't reference it. 3922b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3923b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LSRUse &LU = Uses[LUIdx]; 3924572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!LU.Regs.count(Best)) continue; 3925572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3926b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 3927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 3928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[i]; 3929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.referencesReg(Best)) { 3930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 3931d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman LU.DeleteFormula(F); 3932572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --e; 3933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --i; 3934b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Any = true; 393559dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?"); 3936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3939b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 3940b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 3941b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 3942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 3946572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3947572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3948572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 39494aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceUsingHeuristics - If there are an extraordinary number of 39504aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// formulae to choose from, use some rough heuristics to prune down the number 39514aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of formulae. This keeps the main solver from taking an extraordinary amount 39524aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of time in some worst-case scenarios. 39534aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceUsingHeuristics() { 39544aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByDetectingSupersets(); 39554aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByCollapsingUnrolledCode(); 39564f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 39574aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByPickingWinnerRegs(); 39584aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 39594aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman 3960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// SolveRecurse - This is the recursive solver. 3961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 3962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 3963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 3964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 3965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 3966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const { 3967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Some ideas: 3968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - prune more: 3969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - use more aggressive filtering 3970572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the formula so that the most profitable solutions are found first 3971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the uses too 3972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - search faster: 39733f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // - don't compute a cost, and then compare. compare while computing a cost 3974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // and bail early. 3975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - track register sets with SmallBitVector 3976572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = Uses[Workspace.size()]; 3978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this use references any register that's already a part of the 3980572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // in-progress solution, consider it a requirement that a formula must 3981572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reference that register in order to be considered. This prunes out 3982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // unprofitable searching. 3983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> ReqRegs; 3984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator I = CurRegs.begin(), 3985572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = CurRegs.end(); I != E; ++I) 39869214b82c5446791b280821bbd892dca633130f80Dan Gohman if (LU.Regs.count(*I)) 3987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ReqRegs.insert(*I); 3988572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 39899214b82c5446791b280821bbd892dca633130f80Dan Gohman bool AnySatisfiedReqRegs = false; 3990572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> NewRegs; 3991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost NewCost; 39929214b82c5446791b280821bbd892dca633130f80Dan Gohmanretry: 3993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 3994572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 3995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F = *I; 3996572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3997572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore formulae which do not use any of the required registers. 3998572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(), 3999572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = ReqRegs.end(); J != JE; ++J) { 4000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *J; 4001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((!F.ScaledReg || F.ScaledReg != Reg) && 4002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(F.BaseRegs.begin(), F.BaseRegs.end(), Reg) == 4003572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.end()) 4004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto skip; 4005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 40069214b82c5446791b280821bbd892dca633130f80Dan Gohman AnySatisfiedReqRegs = true; 4007572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4008572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Evaluate the cost of the current formula. If it's already worse than 4009572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the current best, prune the search at that point. 4010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost = CurCost; 4011572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs = CurRegs; 4012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost.RateFormula(F, NewRegs, VisitedRegs, L, LU.Offsets, SE, DT); 4013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NewCost < SolutionCost) { 4014572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.push_back(&F); 4015572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Workspace.size() != Uses.size()) { 4016572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, NewCost, 4017572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs, VisitedRegs); 4018572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.getNumRegs() == 1 && Workspace.size() == 1) 4019572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]); 4020572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4021572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "New best at "; NewCost.print(dbgs()); 40228bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick dbgs() << ".\n Regs:"; 4023572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator 4024572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = NewRegs.begin(), E = NewRegs.end(); I != E; ++I) 4025572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << **I; 4026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 4027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost = NewCost; 4029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution = Workspace; 4030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.pop_back(); 4032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman skip:; 4034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 40359214b82c5446791b280821bbd892dca633130f80Dan Gohman 403680ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (!EnableRetry && !AnySatisfiedReqRegs) 403780ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 403880ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 40399214b82c5446791b280821bbd892dca633130f80Dan Gohman // If none of the formulae had all of the required registers, relax the 40409214b82c5446791b280821bbd892dca633130f80Dan Gohman // constraint so that we don't exclude all formulae. 40419214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!AnySatisfiedReqRegs) { 404259dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman assert(!ReqRegs.empty() && "Solver failed even without required registers"); 40439214b82c5446791b280821bbd892dca633130f80Dan Gohman ReqRegs.clear(); 40449214b82c5446791b280821bbd892dca633130f80Dan Gohman goto retry; 40459214b82c5446791b280821bbd892dca633130f80Dan Gohman } 4046572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4047572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 404876c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solve - Choose one formula from each use. Return the results in the given 404976c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solution vector. 4050572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const { 4051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Workspace; 4052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost SolutionCost; 4053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost.Loose(); 4054572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost CurCost; 4055572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> CurRegs; 4056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> VisitedRegs; 4057572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.reserve(Uses.size()); 4058572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4059f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // SolveRecurse does all the work. 4060572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, CurCost, 4061572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CurRegs, VisitedRegs); 406280ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (Solution.empty()) { 406380ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick DEBUG(dbgs() << "\nNo Satisfactory Solution\n"); 406480ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 406580ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick } 4066572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4067572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we've now made all our decisions. 4068572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\n" 4069572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The chosen solution requires "; SolutionCost.print(dbgs()); 4070572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"; 4071572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Uses.size(); i != e; ++i) { 4072572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 4073572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses[i].print(dbgs()); 4074572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 4075572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman " "; 4076572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution[i]->print(dbgs()); 4077572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'; 4078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 4079a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman 4080a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman assert(Solution.size() == Uses.size() && "Malformed solution!"); 4081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4083e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// HoistInsertPosition - Helper for AdjustInsertPositionForExpand. Climb up 4084e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// the dominator tree far as we can go while still being dominated by the 4085e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// input positions. This helps canonicalize the insert position, which 4086e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// encourages sharing. 4087e5f76877aee6f33964de105893f0ef338661ecadDan GohmanBasicBlock::iterator 4088e5f76877aee6f33964de105893f0ef338661ecadDan GohmanLSRInstance::HoistInsertPosition(BasicBlock::iterator IP, 4089e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) 4090e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const { 4091e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (;;) { 4092e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IPLoop = LI.getLoopFor(IP->getParent()); 4093e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0; 4094e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4095e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock *IDom; 4096d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) { 40970fe46d9b480ab4851e1fc8bc589d1ed9c8b2a70eDan Gohman if (!Rung) return IP; 4098d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman Rung = Rung->getIDom(); 4099d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman if (!Rung) return IP; 4100d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman IDom = Rung->getBlock(); 4101e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4102e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Don't climb into a loop though. 4103e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IDomLoop = LI.getLoopFor(IDom); 4104e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0; 4105e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDomDepth <= IPLoopDepth && 4106e5f76877aee6f33964de105893f0ef338661ecadDan Gohman (IDomDepth != IPLoopDepth || IDomLoop == IPLoop)) 4107e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 4108e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4109e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4110e5f76877aee6f33964de105893f0ef338661ecadDan Gohman bool AllDominate = true; 4111e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *BetterPos = 0; 4112e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Tentative = IDom->getTerminator(); 4113e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(), 4114e5f76877aee6f33964de105893f0ef338661ecadDan Gohman E = Inputs.end(); I != E; ++I) { 4115e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Inst = *I; 4116e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (Inst == Tentative || !DT.dominates(Inst, Tentative)) { 4117e5f76877aee6f33964de105893f0ef338661ecadDan Gohman AllDominate = false; 4118e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 4119e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4120e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Attempt to find an insert position in the middle of the block, 4121e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // instead of at the end, so that it can be used for other expansions. 4122e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDom == Inst->getParent() && 4123e5f76877aee6f33964de105893f0ef338661ecadDan Gohman (!BetterPos || DT.dominates(BetterPos, Inst))) 41247d9663c70b3300070298d716dba6e6f6ce2d1e3eDouglas Gregor BetterPos = llvm::next(BasicBlock::iterator(Inst)); 4125e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4126e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (!AllDominate) 4127e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 4128e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (BetterPos) 4129e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = BetterPos; 4130e5f76877aee6f33964de105893f0ef338661ecadDan Gohman else 4131e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = Tentative; 4132e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4133e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4134e5f76877aee6f33964de105893f0ef338661ecadDan Gohman return IP; 4135e5f76877aee6f33964de105893f0ef338661ecadDan Gohman} 4136e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4137e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// AdjustInsertPositionForExpand - Determine an input position which will be 4138d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman/// dominated by the operands and which will dominate the result. 4139d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanBasicBlock::iterator 4140b5c26ef9da16052597d59a412eaae32098aa1be0Andrew TrickLSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator LowestIP, 4141e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRFixup &LF, 4142b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick const LSRUse &LU, 4143b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick SCEVExpander &Rewriter) const { 4144d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Collect some instructions which must be dominated by the 4145448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // expanding replacement. These must be dominated by any operands that 4146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // will be required in the expansion. 4147572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Instruction *, 4> Inputs; 4148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace)) 4149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 4150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) 4151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = 4152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1))) 4153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 4154448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.PostIncLoops.count(L)) { 4155448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.isUseFullyOutsideLoop(L)) 4156069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(L->getLoopLatch()->getTerminator()); 4157069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman else 4158069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(IVIncInsertPos); 4159069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman } 4160701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // The expansion must also be dominated by the increment positions of any 4161701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // loops it for which it is using post-inc mode. 4162701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (PostIncLoopSet::const_iterator I = LF.PostIncLoops.begin(), 4163701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman E = LF.PostIncLoops.end(); I != E; ++I) { 4164701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman const Loop *PIL = *I; 4165701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (PIL == L) continue; 4166701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman 4167e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Be dominated by the loop exit. 4168701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman SmallVector<BasicBlock *, 4> ExitingBlocks; 4169701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman PIL->getExitingBlocks(ExitingBlocks); 4170701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (!ExitingBlocks.empty()) { 4171701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BasicBlock *BB = ExitingBlocks[0]; 4172701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i) 4173701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]); 4174701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman Inputs.push_back(BB->getTerminator()); 4175701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 4176701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 4177572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4178b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick assert(!isa<PHINode>(LowestIP) && !isa<LandingPadInst>(LowestIP) 4179b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick && !isa<DbgInfoIntrinsic>(LowestIP) && 4180b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick "Insertion point must be a normal instruction"); 4181b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick 4182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Then, climb up the immediate dominator tree as far as we can go while 4183572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // still being dominated by the input positions. 4184b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick BasicBlock::iterator IP = HoistInsertPosition(LowestIP, Inputs); 4185d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4186d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Don't insert instructions before PHI nodes. 4187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (isa<PHINode>(IP)) ++IP; 4188d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4189a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling // Ignore landingpad instructions. 4190a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling while (isa<LandingPadInst>(IP)) ++IP; 4191a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling 4192d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Ignore debug intrinsics. 4193449f31cb9dbf4762935b63946e8120dbe98808ffDan Gohman while (isa<DbgInfoIntrinsic>(IP)) ++IP; 4194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4195b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick // Set IP below instructions recently inserted by SCEVExpander. This keeps the 4196b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick // IP consistent across expansions and allows the previously inserted 4197b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick // instructions to be reused by subsequent expansion. 4198b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick while (Rewriter.isInsertedInstruction(IP) && IP != LowestIP) ++IP; 4199b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick 4200d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman return IP; 4201d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman} 4202d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 420376c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Expand - Emit instructions for the leading candidate expression for this 420476c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// LSRUse (this is called "expanding"). 4205d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanValue *LSRInstance::Expand(const LSRFixup &LF, 4206d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const Formula &F, 4207d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman BasicBlock::iterator IP, 4208d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SCEVExpander &Rewriter, 4209d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const { 4210d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const LSRUse &LU = Uses[LF.LUIdx]; 4211d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4212d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Determine an input position which will be dominated by the operands and 4213d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // which will dominate the result. 4214b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter); 4215d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Inform the Rewriter if we have a post-increment use, so that it can 4217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // perform an advantageous expansion. 4218448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.setPostInc(LF.PostIncLoops); 4219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type that the user actually needs. 4221db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 4222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This will be the type that we'll initially expand to. 4223db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *Ty = F.getType(); 4224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Ty) 4225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // No type known; just expand directly to the ultimate type. 4226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 4227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy)) 4228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand directly to the ultimate type if it's the right size. 4229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 4230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type to do integer arithmetic in. 4231db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = SE.getEffectiveSCEVType(Ty); 4232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Build up a list of operands to add together to form the full base. 4234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Ops; 4235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the BaseRegs portion. 4237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 4238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 4239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 4240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Reg->isZero() && "Zero allocated in a base register!"); 4241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4242448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 4243448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 4244448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Reg = TransformForPostIncUse(Denormalize, Reg, 4245448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 4246448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 4247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, 0, IP))); 4249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4251087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 4252087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman if (!Ops.empty()) { 4253087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 4254087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 4255087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 4256087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman } 4257087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 4258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the ScaledReg portion. 4259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *ICmpScaledV = 0; 4260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale != 0) { 4261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledS = F.ScaledReg; 4262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4263448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 4264448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 4265448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman ScaledS = TransformForPostIncUse(Denormalize, ScaledS, 4266448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 4267448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 4268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 4270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An interesting way of "folding" with an icmp is to use a negated 4271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scale, which we'll implement by inserting it into the other operand 4272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of the icmp. 4273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == -1 && 4274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The only scale supported by ICmpZero uses is -1!"); 4275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Rewriter.expandCodeFor(ScaledS, 0, IP); 4276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise just expand the scaled register and an explicit scale, 4278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which is expected to be matched as part of the address. 4279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, 0, IP)); 4280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getMulExpr(ScaledS, 4281deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(ScaledS->getType(), F.AM.Scale)); 4282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(ScaledS); 4283087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 4284087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 4285087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 4286087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 4287087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 4288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4291087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the GV portion. 4292087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman if (F.AM.BaseGV) { 4293087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(F.AM.BaseGV)); 4294087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 4295087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 4296087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 4297087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 4298087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 4299087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman } 4300087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 4301087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the immediate portion. 4302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)F.AM.BaseOffs + LF.Offset; 4303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) { 4304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 4305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // The other interesting way of "folding" with an ICmpZero is to use a 4306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // negated immediate. 4307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!ICmpScaledV) 4308dae36ba802f12966e4fc44d99097a55ff0b7d87bEli Friedman ICmpScaledV = ConstantInt::get(IntTy, -(uint64_t)Offset); 4309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 4310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(ICmpScaledV)); 4311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = ConstantInt::get(IntTy, Offset); 4312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Just add the immediate values. These again are expected to be matched 4315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // as part of the address. 4316087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset))); 4317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4320cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Expand the unfolded offset portion. 4321cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman int64_t UnfoldedOffset = F.UnfoldedOffset; 4322cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (UnfoldedOffset != 0) { 4323cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Just add the immediate values. 4324cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, 4325cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman UnfoldedOffset))); 4326cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 4327cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 4328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Emit instructions summing all the operands. 4329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *FullS = Ops.empty() ? 4330deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(IntTy, 0) : 4331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getAddExpr(Ops); 4332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *FullV = Rewriter.expandCodeFor(FullS, Ty, IP); 4333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We're done expanding now, so reset the rewriter. 4335448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.clearPostInc(); 4336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An ICmpZero Formula represents an ICmp which we're handling as a 4338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // comparison against zero. Now that we've expanded an expression for that 4339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // form, update the ICmp's other operand. 4340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 4341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *CI = cast<ICmpInst>(LF.UserInst); 4342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(CI->getOperand(1)); 4343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!F.AM.BaseGV && "ICmp does not support folding a global value and " 4344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 4345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale == -1) { 4346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpScaledV->getType() != OpTy) { 4347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 4348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false, 4349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 4350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV, OpTy, "tmp", CI); 4351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Cast; 4352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, ICmpScaledV); 4354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == 0 && 4356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "ICmp does not support folding a global value and " 4357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 4358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy), 4359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman -(uint64_t)Offset); 4360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getType() != OpTy) 4361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 4362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 4363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C, OpTy); 4364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, C); 4366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return FullV; 4370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 43723a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// RewriteForPHI - Helper for Rewrite. PHI nodes are special because the use 43733a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// of their operands effectively happens in their predecessor blocks, so the 43743a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// expression may need to be expanded in multiple places. 43753a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohmanvoid LSRInstance::RewriteForPHI(PHINode *PN, 43763a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const LSRFixup &LF, 43773a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 43783a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 43793a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 43803a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const { 43813a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman DenseMap<BasicBlock *, Value *> Inserted; 43823a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 43833a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (PN->getIncomingValue(i) == LF.OperandValToReplace) { 43843a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman BasicBlock *BB = PN->getIncomingBlock(i); 43853a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 43863a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is a critical edge, split the edge so that we do not insert 43873a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // the code on all predecessor/successor paths. We do this unless this 43883a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // is the canonical backedge for this loop, which complicates post-inc 43893a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // users. 43903a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 && 43913ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman !isa<IndirectBrInst>(BB->getTerminator())) { 439289d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling BasicBlock *Parent = PN->getParent(); 439389d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling Loop *PNLoop = LI.getLoopFor(Parent); 439489d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling if (!PNLoop || Parent != PNLoop->getHeader()) { 43953ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Split the critical edge. 43968b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling BasicBlock *NewBB = 0; 43978b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling if (!Parent->isLandingPad()) { 4398f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick NewBB = SplitCriticalEdge(BB, Parent, P, 4399f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick /*MergeIdenticalEdges=*/true, 4400f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick /*DontDeleteUselessPhis=*/true); 44018b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling } else { 44028b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling SmallVector<BasicBlock*, 2> NewBBs; 44038b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling SplitLandingPadPredecessors(Parent, BB, "", "", P, NewBBs); 44048b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling NewBB = NewBBs[0]; 44058b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling } 44063ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 44073ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // If PN is outside of the loop and BB is in the loop, we want to 44083ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // move the block to be immediately before the PHI block, not 44093ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // immediately after BB. 44103ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman if (L->contains(BB) && !L->contains(PN)) 44113ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman NewBB->moveBefore(PN->getParent()); 44123ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 44133ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Splitting the edge can reduce the number of PHI entries we have. 44143ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman e = PN->getNumIncomingValues(); 44153ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman BB = NewBB; 44163ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman i = PN->getBasicBlockIndex(BB); 44173ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman } 44183a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 44193a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 44203a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair = 44213a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Inserted.insert(std::make_pair(BB, static_cast<Value *>(0))); 44223a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (!Pair.second) 44233a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, Pair.first->second); 44243a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman else { 4425454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, BB->getTerminator(), Rewriter, DeadInsts); 44263a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 44273a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 4428db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 44293a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (FullV->getType() != OpTy) 44303a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV = 44313a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, 44323a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman OpTy, false), 44333a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV, LF.OperandValToReplace->getType(), 44343a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman "tmp", BB->getTerminator()); 44353a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 44363a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, FullV); 44373a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pair.first->second = FullV; 44383a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 44393a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 44403a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman} 44413a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 4442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Rewrite - Emit instructions for the leading candidate expression for this 4443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse (this is called "expanding"), and update the UserInst to reference 4444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the newly expanded value. 4445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Rewrite(const LSRFixup &LF, 4446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 4447572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 4448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 4449572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const { 4450572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // First, find an insertion point that dominates UserInst. For PHI nodes, 4451572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // find the nearest block which dominates all the relevant uses. 4452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) { 4453454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman RewriteForPHI(PN, LF, F, Rewriter, DeadInsts, P); 4454572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4455454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, LF.UserInst, Rewriter, DeadInsts); 4456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4457572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 4458db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 4459572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FullV->getType() != OpTy) { 4460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 4461572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false), 4462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV, OpTy, "tmp", LF.UserInst); 4463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV = Cast; 4464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the user. ICmpZero is handled specially here (for now) because 4467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand may have updated one of the operands of the icmp already, and 4468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // its new value may happen to be equal to LF.OperandValToReplace, in 4469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which case doing replaceUsesOfWith leads to replacing both operands 4470572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with the same value. TODO: Reorganize this. 4471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Uses[LF.LUIdx].Kind == LSRUse::ICmpZero) 4472572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->setOperand(0, FullV); 4473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 4474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV); 4475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4477572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(LF.OperandValToReplace); 4478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 448076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// ImplementSolution - Rewrite all the fixup locations with new values, 448176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// following the chosen solution. 4482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 4483572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 4484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) { 4485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Keep track of instructions we may have made dead, so that 4486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // we can remove them after we are done working. 4487572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<WeakVH, 16> DeadInsts; 4488572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 44895e7645be4c9dd2193add44d30b5fef8036d7a3ceAndrew Trick SCEVExpander Rewriter(SE, "lsr"); 44908bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick#ifndef NDEBUG 44918bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick Rewriter.setDebugType(DEBUG_TYPE); 44928bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick#endif 4493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.disableCanonicalMode(); 4494c5701910604cdf65811fabd31d41e38f1d1d4eb1Andrew Trick Rewriter.enableLSRMode(); 4495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.setIVIncInsertPos(L, IVIncInsertPos); 4496572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 449764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Mark phi nodes that terminate chains so the expander tries to reuse them. 449864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(), 449964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) { 450064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (PHINode *PN = dyn_cast<PHINode>(ChainI->back().UserInst)) 450164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Rewriter.setChainedPhi(PN); 450264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 450364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 4504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the new value definitions and update the users. 4505402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 4506402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Fixups.end(); I != E; ++I) { 4507402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const LSRFixup &Fixup = *I; 4508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4509402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman Rewrite(Fixup, *Solution[Fixup.LUIdx], Rewriter, DeadInsts, P); 4510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 4512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 451422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(), 451522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) { 451622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick GenerateIVChain(*ChainI, Rewriter, DeadInsts); 451722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Changed = true; 451822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 4519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clean up after ourselves. This must be done before deleting any 4520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // instructions. 4521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.clear(); 4522f7912df4cbdb44aeac9ac9907c192dfc1e22646dDan Gohman 4523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= DeleteTriviallyDeadInstructions(DeadInsts); 4524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4525010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 4526572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P) 4527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : IU(P->getAnalysis<IVUsers>()), 4528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE(P->getAnalysis<ScalarEvolution>()), 4529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT(P->getAnalysis<DominatorTree>()), 4530e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LI(P->getAnalysis<LoopInfo>()), 4531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TLI(tli), L(l), Changed(false), IVIncInsertPos(0) { 45325792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 4533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If LoopSimplify form is not available, stay out of trouble. 4534acdb4aaf9b1f2edd96163c27bcc4e0557014f51eAndrew Trick if (!L->isLoopSimplifyForm()) 4535acdb4aaf9b1f2edd96163c27bcc4e0557014f51eAndrew Trick return; 4536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 453775ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // If there's no interesting work to be done, bail early. 453875ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick if (IU.empty()) return; 453975ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick 454075ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick#ifndef NDEBUG 45410f080913d1ff80bb61476724304359e14822b193Andrew Trick // All dominating loops must have preheaders, or SCEVExpander may not be able 45420f080913d1ff80bb61476724304359e14822b193Andrew Trick // to materialize an AddRecExpr whose Start is an outer AddRecExpr. 45430f080913d1ff80bb61476724304359e14822b193Andrew Trick // 454475ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // IVUsers analysis should only create users that are dominated by simple loop 454575ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // headers. Since this loop should dominate all of its users, its user list 454675ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // should be empty if this loop itself is not within a simple loop nest. 45470f080913d1ff80bb61476724304359e14822b193Andrew Trick for (DomTreeNode *Rung = DT.getNode(L->getLoopPreheader()); 45480f080913d1ff80bb61476724304359e14822b193Andrew Trick Rung; Rung = Rung->getIDom()) { 45490f080913d1ff80bb61476724304359e14822b193Andrew Trick BasicBlock *BB = Rung->getBlock(); 45500f080913d1ff80bb61476724304359e14822b193Andrew Trick const Loop *DomLoop = LI.getLoopFor(BB); 45510f080913d1ff80bb61476724304359e14822b193Andrew Trick if (DomLoop && DomLoop->getHeader() == BB) { 455275ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick assert(DomLoop->getLoopPreheader() && "LSR needs a simplified loop nest"); 45530f080913d1ff80bb61476724304359e14822b193Andrew Trick } 4554acdb4aaf9b1f2edd96163c27bcc4e0557014f51eAndrew Trick } 455575ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick#endif // DEBUG 4556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\nLSR on loop "; 4558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(dbgs(), L->getHeader(), /*PrintType=*/false); 4559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"); 4560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4561402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // First, perform some low-level loop optimizations. 4562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OptimizeShadowIV(); 4563c6519f916b5922de81c53547fd21364994195a70Dan Gohman OptimizeLoopTermCond(); 4564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 456537eb38d3f8531115f17f4e829013ccb513952badAndrew Trick // If loop preparation eliminates all interesting IV users, bail. 456637eb38d3f8531115f17f4e829013ccb513952badAndrew Trick if (IU.empty()) return; 456737eb38d3f8531115f17f4e829013ccb513952badAndrew Trick 45685219f86a0bab11dd6895a31653e371e9871a6734Andrew Trick // Skip nested loops until we can model them better with formulae. 45690c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick if (!EnableNested && !L->empty()) { 45700c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n"); 45715219f86a0bab11dd6895a31653e371e9871a6734Andrew Trick return; 45720c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick } 45730c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick 4574402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // Start collecting data and preparing for the solver. 45756c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick CollectChains(); 4576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectInterestingTypesAndFactors(); 4577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectFixupsAndInitialFormulae(); 4578572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectLoopInvariantFixupsAndFormulae(); 4579572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 458022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(!Uses.empty() && "IVUsers reported at least one use"); 4581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n"; 4582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 4583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now use the reuse data to generate a bunch of interesting ways 4585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to formulate the values needed for the uses. 4586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateAllReuseFormulae(); 4587572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FilterOutUndesirableDedicatedRegisters(); 4589572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NarrowSearchSpaceUsingHeuristics(); 4590572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4591572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Solution; 4592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solve(Solution); 4593572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Release memory that is no longer needed. 4595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.clear(); 4596572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 4597572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.clear(); 4598572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 459980ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (Solution.empty()) 460080ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 460180ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 4602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 4603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Formulae should be legal. 4604572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 4605572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 4606572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 4607572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 4608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) 4609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(isLegalUse(J->AM, LU.MinOffset, LU.MaxOffset, 4610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI) && 4611572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Illegal formula generated!"); 4612572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 4613572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 4614010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 4615572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now that we've decided what we want, make it so. 4616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImplementSolution(Solution, P); 4617572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4618169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 4619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_factors_and_types(raw_ostream &OS) const { 4620572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors.empty() && Types.empty()) return; 46211ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 4622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR has identified the following interesting factors and types: "; 4623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 4624eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 4625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 4626572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 4627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 4628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 4629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '*' << *I; 4630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 46318b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 4632db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner for (SmallSetVector<Type *, 4>::const_iterator 4633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 4634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 4635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 4636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '(' << **I << ')'; 4637572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4640c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 4641572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_fixups(raw_ostream &OS) const { 4642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following fixup sites:\n"; 4643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 4644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Fixups.end(); I != E; ++I) { 4645572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 46469f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman I->print(OS); 4647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4650010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman 4651572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_uses(raw_ostream &OS) const { 4652572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following uses:\n"; 4653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 4654572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 4655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 4656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 4657572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.print(OS); 4658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 4660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) { 4661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << " "; 4662572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J->print(OS); 4663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 46656bec5bb344fc0374431aed1cb63418de607a1aecDan Gohman } 4666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print(raw_ostream &OS) const { 4669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_factors_and_types(OS); 4670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_fixups(OS); 4671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(OS); 4672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::dump() const { 4675572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 4676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 4679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LoopStrengthReduce : public LoopPass { 4681572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TLI - Keep a pointer of a TargetLowering to consult for determining 4682572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// transformation profitability. 4683572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 4684572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4685572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 4686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static char ID; // Pass ID, replacement for typeid 4687572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman explicit LoopStrengthReduce(const TargetLowering *tli = 0); 4688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4689572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 4690572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool runOnLoop(Loop *L, LPPassManager &LPM); 4691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void getAnalysisUsage(AnalysisUsage &AU) const; 4692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 4693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4695572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4696572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanchar LoopStrengthReduce::ID = 0; 46972ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce", 4698ce665bd2e2b581ab0858d1afe359192bac96b868Owen Anderson "Loop Strength Reduction", false, false) 46992ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(DominatorTree) 47002ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(ScalarEvolution) 47012ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(IVUsers) 4702205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopInfo) 4703205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopSimplify) 47042ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce", 47052ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson "Loop Strength Reduction", false, false) 47062ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson 4707572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4708572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanPass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) { 4709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return new LoopStrengthReduce(TLI); 4710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4712572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLoopStrengthReduce::LoopStrengthReduce(const TargetLowering *tli) 4713081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson : LoopPass(ID), TLI(tli) { 4714081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry()); 4715081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson } 4716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4717572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const { 4718572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We split critical edges, so we change the CFG. However, we do update 4719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // many analyses if they are around. 47206793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreservedID(LoopSimplifyID); 4721572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 47226793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequired<LoopInfo>(); 47236793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreserved<LoopInfo>(); 47246793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequiredID(LoopSimplifyID); 4725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<DominatorTree>(); 4726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<DominatorTree>(); 4727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<ScalarEvolution>(); 4728572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<ScalarEvolution>(); 47292c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // Requiring LoopSimplify a second time here prevents IVUsers from running 47302c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // twice, since LoopSimplify was invalidated by running ScalarEvolution. 47312c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich AU.addRequiredID(LoopSimplifyID); 4732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<IVUsers>(); 4733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<IVUsers>(); 4734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) { 4737572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 4738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Run the main LSR transformation. 4740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= LSRInstance(TLI, L, this).getChanged(); 4741eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 4742f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick // Remove any extra phis created by processing inner loops. 47439fff2187a21f765ed87a25c48552a6942450f3e2Dan Gohman Changed |= DeleteDeadPHIs(L->getHeader()); 4744f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick if (EnablePhiElim) { 4745f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick SmallVector<WeakVH, 16> DeadInsts; 4746f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick SCEVExpander Rewriter(getAnalysis<ScalarEvolution>(), "lsr"); 4747f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick#ifndef NDEBUG 4748f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick Rewriter.setDebugType(DEBUG_TYPE); 4749f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick#endif 4750f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick unsigned numFolded = Rewriter. 4751f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick replaceCongruentIVs(L, &getAnalysis<DominatorTree>(), DeadInsts, TLI); 4752f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick if (numFolded) { 4753f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick Changed = true; 4754f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick DeleteTriviallyDeadInstructions(DeadInsts); 4755f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick DeleteDeadPHIs(L->getHeader()); 4756f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick } 4757f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick } 47581ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng return Changed; 4759eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 4760