LoopStrengthReduce.cpp revision 3228cc259b5ca00e46af36da369a451f5736cbf4
12d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// The LLVM Compiler Infrastructure 4eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 9eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 10cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// This transformation analyzes and transforms the induction variables (and 11cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// computations derived from them) into forms suitable for efficient execution 12cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// on the target. 13cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// 14eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// This pass performs a strength reduction on array references inside loops that 15cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// have as one or more of their components the loop induction variable, it 16cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// rewrites expressions to take advantage of scaled-index addressing modes 17cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// available on the target, and it performs a variety of other optimizations 18cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// related to loop induction variables. 19eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 20572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// Terminology note: this code has a lot of handling for "post-increment" or 21572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// "post-inc" users. This is not talking about post-increment addressing modes; 22572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it is instead talking about code like this: 23572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 24572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i = phi [ 0, %entry ], [ %i.next, %latch ] 25572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// ... 26572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i.next = add %i, 1 27572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %c = icmp eq %i.next, %n 28572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 29572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however 30572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it's useful to think about these as the same register, with some uses using 31572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the register before the add and some using // it after. In this 32572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// example, the icmp is a post-increment user, since it uses %i.next, which is 33572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the induction variable after the increment. The other common 34572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// case of post-increment users is users outside the loop. 35572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 36572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: More sophistication in the way Formulae are generated and filtered. 37572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 38572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Handle multiple loops at a time. 39572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 40572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Should TargetLowering::AddrMode::BaseGV be changed to a ConstantExpr 41572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// instead of a GlobalValue? 42572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 43572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When truncation is free, truncate ICmp users' operands to make it a 44572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// smaller encoding (on x86 at least). 45572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 46572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When a negated register is used by an add (such as in a list of 47572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// multiple base registers, or as the increment expression in an addrec), 48572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// we may not actually need both reg and (-1 * reg) in registers; the 49572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// negation can be implemented by using a sub instead of an add. The 50572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// lack of support for taking this into consideration when making 51572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// register pressure decisions is partly worked around by the "Special" 52572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// use kind. 53572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 54eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 55eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 56be3e5212e23edc9210f774fac18d801de252e906Chris Lattner#define DEBUG_TYPE "loop-reduce" 57eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Scalar.h" 58eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Constants.h" 59eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Instructions.h" 60e5b01bea7b9b7dce7c24484d2d915b0c118d4d07Dan Gohman#include "llvm/IntrinsicInst.h" 612f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/DerivedTypes.h" 6281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman#include "llvm/Analysis/IVUsers.h" 63572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/Analysis/Dominators.h" 640f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel#include "llvm/Analysis/LoopPass.h" 65169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Analysis/ScalarEvolutionExpander.h" 669fc5cdf77c812aaa80419036de27576d45894d0dChris Lattner#include "llvm/Assembly/Writer.h" 67e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner#include "llvm/Transforms/Utils/BasicBlockUtils.h" 68eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Utils/Local.h" 69572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SmallBitVector.h" 70572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SetVector.h" 71572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/DenseSet.h" 72169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/Debug.h" 73afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman#include "llvm/Support/ValueHandle.h" 74460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar#include "llvm/Support/raw_ostream.h" 75d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng#include "llvm/Target/TargetLowering.h" 76cfb1d4235fe3291028341e6acf4139723b4749e3Jeff Cohen#include <algorithm> 77eaa13851a7fe604363577350c5cf65c257c4d41aNate Begemanusing namespace llvm; 78eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 79572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 80572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 81572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegSortData - This class holds data which is used to order reuse candidates. 82572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegSortData { 83572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 84572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UsedByIndices - This represents the set of LSRUse indices which reference 85572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// a particular register. 86572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallBitVector UsedByIndices; 87572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 88572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData() {} 89572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 90572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 91572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 92572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 93572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 94572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 95572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 96572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::print(raw_ostream &OS) const { 97572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "[NumUses=" << UsedByIndices.count() << ']'; 98572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 99dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::dump() const { 101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 103dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 1047979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmannamespace { 1057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegUseTracker - Map register candidates to information about how they are 107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// used. 108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegUseTracker { 109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, RegSortData> RegUsesTy; 110d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 11190bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy RegUsesMap; 112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 16> RegSequence; 113d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegister(const SCEV *Reg, size_t LUIdx); 116b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void DropRegister(const SCEV *Reg, size_t LUIdx); 117c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void SwapAndDropUse(size_t LUIdx, size_t LastLUIdx); 118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const; 120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &getUsedByIndices(const SCEV *Reg) const; 122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void clear(); 124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::iterator iterator; 126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator; 127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator begin() { return RegSequence.begin(); } 128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator end() { return RegSequence.end(); } 129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator begin() const { return RegSequence.begin(); } 130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator end() const { return RegSequence.end(); } 131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 136572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) { 137572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegUsesTy::iterator, bool> Pair = 13890bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.insert(std::make_pair(Reg, RegSortData())); 139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData &RSD = Pair.first->second; 140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.push_back(Reg); 142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1)); 143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.set(LUIdx); 144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 146b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid 147b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan GohmanRegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) { 148b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUsesTy::iterator It = RegUsesMap.find(Reg); 149b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(It != RegUsesMap.end()); 150b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegSortData &RSD = It->second; 151b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(RSD.UsedByIndices.size() > LUIdx); 152b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RSD.UsedByIndices.reset(LUIdx); 153b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 154b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 155a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanvoid 156c6897706b7c3796ac24535c9ea1449c0411f8c7aDan GohmanRegUseTracker::SwapAndDropUse(size_t LUIdx, size_t LastLUIdx) { 157c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman assert(LUIdx <= LastLUIdx); 158c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 159c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. The data structure is not optimized for this purpose; 160c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // we must iterate through it and update each of the bit vectors. 161a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (RegUsesTy::iterator I = RegUsesMap.begin(), E = RegUsesMap.end(); 162c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman I != E; ++I) { 163c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman SmallBitVector &UsedByIndices = I->second.UsedByIndices; 164c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman if (LUIdx < UsedByIndices.size()) 165c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices[LUIdx] = 166c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : 0; 167c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx)); 168c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman } 169a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 170a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 172572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { 17346fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 17446fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman if (I == RegUsesMap.end()) 17546fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman return false; 17646fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman const SmallBitVector &UsedByIndices = I->second.UsedByIndices; 177572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int i = UsedByIndices.find_first(); 178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (i == -1) return false; 179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((size_t)i != LUIdx) return true; 180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return UsedByIndices.find_next(i) != -1; 181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 183572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanconst SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { 18490bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 18590bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman assert(I != RegUsesMap.end() && "Unknown register!"); 186572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return I->second.UsedByIndices; 187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegUseTracker::clear() { 19090bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.clear(); 191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.clear(); 192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Formula - This class holds information that describes a formula for 197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// computing satisfying a use. It may include broken-out immediates and scaled 198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// registers. 199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct Formula { 200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AM - This is used to represent complex addressing, as well as other kinds 201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// of interesting uses. 202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// BaseRegs - The list of "base" registers for this use. When this is 205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// non-empty, AM.HasBaseReg should be set to true. 206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> BaseRegs; 207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// ScaledReg - The 'scaled' register for this use. This should be non-null 209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// when AM.Scale is not zero. 210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledReg; 211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula() : ScaledReg(0) {} 213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 214dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE); 215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned getNumRegs() const; 217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *getType() const; 218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2195ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman void DeleteBaseReg(const SCEV *&S); 2205ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool referencesReg(const SCEV *S) const; 222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool hasRegsUsedByUsesOtherThan(size_t LUIdx, 223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const; 224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2313f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// DoInitialMatch - Recursion helper for InitialMatch. 232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic void DoInitialMatch(const SCEV *S, Loop *L, 233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Good, 234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Bad, 235dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman ScalarEvolution &SE) { 236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect expressions which properly dominate the loop header. 237dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(S, L->getHeader())) { 238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(S); 239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 241d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at add operands. 243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) 246dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(*I, L, Good, Bad, SE); 247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 249169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at addrec operands. 251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) 252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!AR->getStart()->isZero()) { 253dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(AR->getStart(), L, Good, Bad, SE); 254deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), 255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE), 2563228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags() 2573228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick AR->getLoop(), SCEV::FlagAnyWrap), 258dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman L, Good, Bad, SE); 259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2607979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 2612d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle a multiplication by -1 (negation) if it didn't fold. 263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) 264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mul->getOperand(0)->isAllOnesValue()) { 265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end()); 266572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewMul = SE.getMulExpr(Ops); 267572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyGood; 269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyBad; 270dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(NewMul, L, MyGood, MyBad, SE); 271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( 272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getEffectiveSCEVType(NewMul->getType()))); 273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(), 274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyGood.end(); I != E; ++I) 275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(SE.getMulExpr(NegOne, *I)); 276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(), 277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyBad.end(); I != E; ++I) 278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(SE.getMulExpr(NegOne, *I)); 279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 281eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we can't do anything interesting. Just stuff the whole thing into a 283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // register and hope for the best. 284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(S); 285a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 286844731a7f1909f55935e3514c9e713a62d67662eDan Gohman 287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InitialMatch - Incorporate loop-variant parts of S into this Formula, 288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// attempting to keep all loop-invariant and loop-computable values in a 289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// single base register. 290dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohmanvoid Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { 291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Good; 292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Bad; 293dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(S, L, Good, Bad, SE); 294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Good.empty()) { 295e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Good); 296e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 297e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Bad.empty()) { 301e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Bad); 302e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 303e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 307eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getNumRegs - Return the total number of register operands used by this 309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// formula. This does not include register uses implied by non-constant 310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrec strides. 311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanunsigned Formula::getNumRegs() const { 312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !!ScaledReg + BaseRegs.size(); 313a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 31456a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getType - Return the type of this formula, if it has one, or null 316572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// otherwise. This type is meaningless except for the bit size. 317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanconst Type *Formula::getType() const { 318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !BaseRegs.empty() ? BaseRegs.front()->getType() : 319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledReg ? ScaledReg->getType() : 320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV ? AM.BaseGV->getType() : 321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 0; 322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 32356a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 3245ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteBaseReg - Delete the given base reg from the BaseRegs list. 3255ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohmanvoid Formula::DeleteBaseReg(const SCEV *&S) { 3265ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&S != &BaseRegs.back()) 3275ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(S, BaseRegs.back()); 3285ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman BaseRegs.pop_back(); 3295ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 3305ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// referencesReg - Test if this formula references the given register. 332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::referencesReg(const SCEV *S) const { 333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return S == ScaledReg || 334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(BaseRegs.begin(), BaseRegs.end(), S) != BaseRegs.end(); 335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 336eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// hasRegsUsedByUsesOtherThan - Test whether this formula uses registers 338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// which are used by uses other than the use with the given index. 339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx, 340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const { 341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx)) 343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) 346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(*I, LUIdx)) 347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::print(raw_ostream &OS) const { 352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) { 354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, AM.BaseGV, /*PrintType=*/false); 356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.BaseOffs; 360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) { 363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "reg(" << **I << ')'; 365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 366c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (AM.HasBaseReg && BaseRegs.empty()) { 367c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 368c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: HasBaseReg**"; 369c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } else if (!AM.HasBaseReg && !BaseRegs.empty()) { 370c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 371c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: !HasBaseReg**"; 372c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } 373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0) { 374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.Scale << "*reg("; 376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *ScaledReg; 378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "<unknown>"; 380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ')'; 381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 384572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::dump() const { 385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 388aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddRecSExtable - Return true if the given addrec can be sign-extended 389aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 390aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { 391aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman const Type *WideTy = 392ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1); 393aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy)); 394aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 395aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 396aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddSExtable - Return true if the given add can be sign-extended 397aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 398aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) { 399aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman const Type *WideTy = 400ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1); 401aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy)); 402aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 403aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 404473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman/// isMulSExtable - Return true if the given mul can be sign-extended 405aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 406473e63512acb3751da7dffbb10e0452cb581b265Dan Gohmanstatic bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) { 407aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman const Type *WideTy = 408473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman IntegerType::get(SE.getContext(), 409473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman SE.getTypeSizeInBits(M->getType()) * M->getNumOperands()); 410473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy)); 411aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 412aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 413f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// getExactSDiv - Return an expression for LHS /s RHS, if it can be determined 414f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// and if the remainder is known to be zero, or null otherwise. If 415f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified 416f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// to Y, ignoring that the multiplication may overflow, which is useful when 417f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// the result will be used in a context where the most significant bits are 418f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// ignored. 419f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohmanstatic const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, 420f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman ScalarEvolution &SE, 421f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman bool IgnoreSignificantBits = false) { 422572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle the trivial case, which works for any SCEV type. 423572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LHS == RHS) 424deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman return SE.getConstant(LHS->getType(), 1); 425572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 426d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle a few RHS special cases. 427d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS); 428d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RC) { 429d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 430d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do 431d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // some folding. 432d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA.isAllOnesValue()) 433d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getMulExpr(LHS, RC); 434d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s 1 as x. 435d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA == 1) 436d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return LHS; 437d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman } 438572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 439572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a division of a constant by a constant. 440572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) { 441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!RC) 442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 443d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &LA = C->getValue()->getValue(); 444d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 445d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (LA.srem(RA) != 0) 446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 447d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getConstant(LA.sdiv(RA)); 448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 449572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 450aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over addrec operands, if the addrec doesn't overflow. 451572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) { 452aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddRecSExtable(AR, SE)) { 453f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, 454f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 455aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Step) return 0; 456694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, 457694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman IgnoreSignificantBits); 458694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman if (!Start) return 0; 4593228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FlagNW is independent of the start value, step direction, and is 4603228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // preserved with smaller magnitude steps. 4613228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 4623228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); 463aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 4642ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 467aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over add operands, if the add doesn't overflow. 468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) { 469aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddSExtable(Add, SE)) { 470aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman SmallVector<const SCEV *, 8> Ops; 471aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 472aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman I != E; ++I) { 473f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Op = getExactSDiv(*I, RHS, SE, 474f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 475aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Op) return 0; 476aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman Ops.push_back(Op); 477aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 478aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return SE.getAddExpr(Ops); 479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4802ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a multiply operand that we can pull RHS out of. 4842ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) { 485aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) { 486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 487572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Found = false; 488572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVMulExpr::op_iterator I = Mul->op_begin(), E = Mul->op_end(); 489572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 4904766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman const SCEV *S = *I; 491572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Found) 4924766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman if (const SCEV *Q = getExactSDiv(S, RHS, SE, 493f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits)) { 4944766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman S = Q; 495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Found = true; 496572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4974766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman Ops.push_back(S); 498a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Found ? SE.getMulExpr(Ops) : 0; 500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5012ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 5022ea09e05466613a22e1211f52c30cd01af563983Dan Gohman } 503a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise we don't know. 505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractImmediate - If S involves the addition of a constant integer value, 509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that integer value, and mutate S to point to a new SCEV with that 510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { 512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { 513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getValue()->getValue().getMinSignedBits() <= 64) { 514deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(C->getType(), 0); 515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return C->getValue()->getSExtValue(); 516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 520e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 521e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 526e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 5273228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5283228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5293228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 53121e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman } 532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 534572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 535572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractSymbol - If S involves the addition of a GlobalValue address, 536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that symbol, and mutate S to point to a new SCEV with that 537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 538572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { 539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) { 541deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(GV->getType(), 0); 542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return GV; 543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.back(), SE); 547e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 548e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.front(), SE); 553e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 5543228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5553228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5563228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 560169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 561169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 5627979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// isAddressUse - Returns true if the specified instruction is using the 5637979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// specified value as an address. 5647979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmanstatic bool isAddressUse(Instruction *Inst, Value *OperandVal) { 5657979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman bool isAddress = isa<LoadInst>(Inst); 5667979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 5677979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (SI->getOperand(1) == OperandVal) 5687979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 5697979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 5707979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 5717979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 5727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 5737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 5747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::prefetch: 5757979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_loadu_dq: 5767979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_loadu_pd: 5777979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_loadu_ps: 5787979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 5797979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 5807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 5817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 582ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif if (II->getArgOperand(0) == OperandVal) 5837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 5847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 5857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 5867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 5877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return isAddress; 588169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 589169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 5907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// getAccessType - Return the type of the memory being accessed. 5917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmanstatic const Type *getAccessType(const Instruction *Inst) { 5927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const Type *AccessTy = Inst->getType(); 5937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) 5947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AccessTy = SI->getOperand(0)->getType(); 5957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 5967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 5977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 5987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 5997979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 6007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 6017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 6027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 6037979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 604ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif AccessTy = II->getArgOperand(0)->getType(); 6057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 6067979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6077979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // All pointers have the same requirements, so canonicalize them to an 610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // arbitrary pointer type to minimize variation. 611572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const PointerType *PTy = dyn_cast<PointerType>(AccessTy)) 612572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = PointerType::get(IntegerType::get(PTy->getContext(), 1), 613572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PTy->getAddressSpace()); 614572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 6157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return AccessTy; 616a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 61781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 618572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DeleteTriviallyDeadInstructions - If any of the instructions is the 619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// specified set are trivially dead, delete them and see if this makes any of 620572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// their operands subsequently dead. 621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool 622572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanDeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) { 623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 6242f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!DeadInsts.empty()) { 626f097b59e0e2231d431a6c660c65fa5cae22d9a44Gabor Greif Instruction *I = dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()); 62781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (I == 0 || !isInstructionTriviallyDead(I)) 629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 630221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 631572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) 632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *U = dyn_cast<Instruction>(*OI)) { 633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *OI = 0; 634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (U->use_empty()) 635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(U); 636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 637221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I->eraseFromParent(); 639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 6412f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Changed; 6437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 6442f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 645572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 6462f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Cost - This class is used to measure and compare candidate formulae. 648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass Cost { 649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TODO: Some of these could be merged. Also, a lexical ordering 650572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// isn't always optimal. 651572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumRegs; 652572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned AddRecCost; 653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumIVMuls; 654572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumBaseAdds; 655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned ImmCost; 656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned SetupCost; 657572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost() 660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : NumRegs(0), AddRecCost(0), NumIVMuls(0), NumBaseAdds(0), ImmCost(0), 661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost(0) {} 662572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool operator<(const Cost &Other) const; 664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Loose(); 666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateFormula(const Formula &F, 668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 675572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateRegister(const SCEV *Reg, 679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 681572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 6829214b82c5446791b280821bbd892dca633130f80Dan Gohman void RatePrimaryRegister(const SCEV *Reg, 6839214b82c5446791b280821bbd892dca633130f80Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 6849214b82c5446791b280821bbd892dca633130f80Dan Gohman const Loop *L, 6859214b82c5446791b280821bbd892dca633130f80Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 6870e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 6897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 690572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RateRegister - Tally up interesting quantities from the given register. 691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateRegister(const SCEV *Reg, 692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 6959214b82c5446791b280821bbd892dca633130f80Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) { 6969214b82c5446791b280821bbd892dca633130f80Dan Gohman if (AR->getLoop() == L) 6979214b82c5446791b280821bbd892dca633130f80Dan Gohman AddRecCost += 1; /// TODO: This should be a function of the stride. 6989214b82c5446791b280821bbd892dca633130f80Dan Gohman 6999214b82c5446791b280821bbd892dca633130f80Dan Gohman // If this is an addrec for a loop that's already been visited by LSR, 7009214b82c5446791b280821bbd892dca633130f80Dan Gohman // don't second-guess its addrec phi nodes. LSR isn't currently smart 7019214b82c5446791b280821bbd892dca633130f80Dan Gohman // enough to reason about more than one loop at a time. Consider these 7029214b82c5446791b280821bbd892dca633130f80Dan Gohman // registers free and leave them alone. 7039214b82c5446791b280821bbd892dca633130f80Dan Gohman else if (L->contains(AR->getLoop()) || 7049214b82c5446791b280821bbd892dca633130f80Dan Gohman (!AR->getLoop()->contains(L) && 7059214b82c5446791b280821bbd892dca633130f80Dan Gohman DT.dominates(L->getHeader(), AR->getLoop()->getHeader()))) { 7069214b82c5446791b280821bbd892dca633130f80Dan Gohman for (BasicBlock::iterator I = AR->getLoop()->getHeader()->begin(); 7079214b82c5446791b280821bbd892dca633130f80Dan Gohman PHINode *PN = dyn_cast<PHINode>(I); ++I) 7089214b82c5446791b280821bbd892dca633130f80Dan Gohman if (SE.isSCEVable(PN->getType()) && 7099214b82c5446791b280821bbd892dca633130f80Dan Gohman (SE.getEffectiveSCEVType(PN->getType()) == 7109214b82c5446791b280821bbd892dca633130f80Dan Gohman SE.getEffectiveSCEVType(AR->getType())) && 7119214b82c5446791b280821bbd892dca633130f80Dan Gohman SE.getSCEV(PN) == AR) 7129214b82c5446791b280821bbd892dca633130f80Dan Gohman return; 7139214b82c5446791b280821bbd892dca633130f80Dan Gohman 7149214b82c5446791b280821bbd892dca633130f80Dan Gohman // If this isn't one of the addrecs that the loop already has, it 7159214b82c5446791b280821bbd892dca633130f80Dan Gohman // would require a costly new phi and add. TODO: This isn't 7169214b82c5446791b280821bbd892dca633130f80Dan Gohman // precisely modeled right now. 7179214b82c5446791b280821bbd892dca633130f80Dan Gohman ++NumBaseAdds; 7189214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!Regs.count(AR->getStart())) 719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RateRegister(AR->getStart(), Regs, L, SE, DT); 7209214b82c5446791b280821bbd892dca633130f80Dan Gohman } 7217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 7229214b82c5446791b280821bbd892dca633130f80Dan Gohman // Add the step value register, if it needs one. 7239214b82c5446791b280821bbd892dca633130f80Dan Gohman // TODO: The non-affine case isn't precisely modeled here. 7249214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) 7259214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!Regs.count(AR->getStart())) 726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RateRegister(AR->getOperand(1), Regs, L, SE, DT); 727a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 7289214b82c5446791b280821bbd892dca633130f80Dan Gohman ++NumRegs; 7299214b82c5446791b280821bbd892dca633130f80Dan Gohman 7309214b82c5446791b280821bbd892dca633130f80Dan Gohman // Rough heuristic; favor registers which don't require extra setup 7319214b82c5446791b280821bbd892dca633130f80Dan Gohman // instructions in the preheader. 7329214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!isa<SCEVUnknown>(Reg) && 7339214b82c5446791b280821bbd892dca633130f80Dan Gohman !isa<SCEVConstant>(Reg) && 7349214b82c5446791b280821bbd892dca633130f80Dan Gohman !(isa<SCEVAddRecExpr>(Reg) && 7359214b82c5446791b280821bbd892dca633130f80Dan Gohman (isa<SCEVUnknown>(cast<SCEVAddRecExpr>(Reg)->getStart()) || 7369214b82c5446791b280821bbd892dca633130f80Dan Gohman isa<SCEVConstant>(cast<SCEVAddRecExpr>(Reg)->getStart())))) 7379214b82c5446791b280821bbd892dca633130f80Dan Gohman ++SetupCost; 73823c3fde39aa35334e74c26273a8973a872871e34Dan Gohman 73923c3fde39aa35334e74c26273a8973a872871e34Dan Gohman NumIVMuls += isa<SCEVMulExpr>(Reg) && 74017ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman SE.hasComputableLoopEvolution(Reg, L); 7419214b82c5446791b280821bbd892dca633130f80Dan Gohman} 7429214b82c5446791b280821bbd892dca633130f80Dan Gohman 7439214b82c5446791b280821bbd892dca633130f80Dan Gohman/// RatePrimaryRegister - Record this register in the set. If we haven't seen it 7449214b82c5446791b280821bbd892dca633130f80Dan Gohman/// before, rate it. 7459214b82c5446791b280821bbd892dca633130f80Dan Gohmanvoid Cost::RatePrimaryRegister(const SCEV *Reg, 7467fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 7477fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman const Loop *L, 7487fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 7499214b82c5446791b280821bbd892dca633130f80Dan Gohman if (Regs.insert(Reg)) 7509214b82c5446791b280821bbd892dca633130f80Dan Gohman RateRegister(Reg, Regs, L, SE, DT); 7517979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 7527979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateFormula(const Formula &F, 754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 757572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 758572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the registers. 760572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEV *ScaledReg = F.ScaledReg) { 761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(ScaledReg)) { 762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 763572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 7647979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 7659214b82c5446791b280821bbd892dca633130f80Dan Gohman RatePrimaryRegister(ScaledReg, Regs, L, SE, DT); 7663821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner } 767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(BaseReg)) { 771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 7737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 7749214b82c5446791b280821bbd892dca633130f80Dan Gohman RatePrimaryRegister(BaseReg, Regs, L, SE, DT); 775572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.BaseRegs.size() > 1) 778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumBaseAdds += F.BaseRegs.size() - 1; 779572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the non-zero immediates. 781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 782572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)*I + F.AM.BaseOffs; 784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.BaseGV) 785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += 64; // Handle symbolic values conservatively. 786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This should probably be the pointer size. 787572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Offset != 0) 788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += APInt(64, Offset, true).getMinSignedBits(); 789572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 790169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 791169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 792572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Loose - Set this cost to a loosing value. 793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::Loose() { 794572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumRegs = ~0u; 795572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AddRecCost = ~0u; 796572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumIVMuls = ~0u; 797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumBaseAdds = ~0u; 798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost = ~0u; 799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost = ~0u; 800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 80156a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// operator< - Choose the lower cost. 803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Cost::operator<(const Cost &Other) const { 804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumRegs != Other.NumRegs) 805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumRegs < Other.NumRegs; 806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != Other.AddRecCost) 807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return AddRecCost < Other.AddRecCost; 808572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != Other.NumIVMuls) 809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumIVMuls < Other.NumIVMuls; 810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != Other.NumBaseAdds) 811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumBaseAdds < Other.NumBaseAdds; 812572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != Other.ImmCost) 813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return ImmCost < Other.ImmCost; 814572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != Other.SetupCost) 815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return SetupCost < Other.SetupCost; 816572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 8187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::print(raw_ostream &OS) const { 820572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << NumRegs << " reg" << (NumRegs == 1 ? "" : "s"); 821572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != 0) 822572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", with addrec cost " << AddRecCost; 823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != 0) 824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumIVMuls << " IV mul" << (NumIVMuls == 1 ? "" : "s"); 825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != 0) 826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumBaseAdds << " base add" 827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << (NumBaseAdds == 1 ? "" : "s"); 828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != 0) 829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << ImmCost << " imm cost"; 830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != 0) 831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << SetupCost << " setup cost"; 832572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 83344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::dump() const { 835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 83644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner} 83744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 838572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 83944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 840572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRFixup - An operand value in an instruction which is to be replaced 841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// with some equivalent, possibly strength-reduced, replacement. 842572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct LSRFixup { 843572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UserInst - The instruction which will be updated. 844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *UserInst; 84556a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// OperandValToReplace - The operand of the instruction which will 847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// be replaced. The operand may be used more than once; every instance 848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// will be replaced. 849572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OperandValToReplace; 85056a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 851448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman /// PostIncLoops - If this user is to use the post-incremented value of an 852572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable, this variable is non-null and holds the loop 853572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// associated with the induction variable. 854448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet PostIncLoops; 85526d91f16464db56283087176a73981048331dd2dChris Lattner 856572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LUIdx - The index of the LSRUse describing the expression which 857572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// this fixup needs, minus an offset (below). 858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 85926d91f16464db56283087176a73981048331dd2dChris Lattner 860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Offset - A constant offset to be added to the LSRUse expression. 861572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// This allows multiple fixups to share the same LSRUse with different 862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// offsets, for example in an unrolled loop. 863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset; 864572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 865448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman bool isUseFullyOutsideLoop(const Loop *L) const; 866448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 867572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup(); 868572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 869572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 871572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 872572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 873572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 874572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 875572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRFixup::LSRFixup() 876ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman : UserInst(0), OperandValToReplace(0), LUIdx(~size_t(0)), Offset(0) {} 877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 878448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// isUseFullyOutsideLoop - Test whether this fixup always uses its 879448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// value outside of the given loop. 880448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohmanbool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const { 881448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // PHI nodes use their value in their incoming blocks. 882448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) { 883448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 884448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (PN->getIncomingValue(i) == OperandValToReplace && 885448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman L->contains(PN->getIncomingBlock(i))) 886448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return false; 887448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return true; 888448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 889448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 890448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return !L->contains(UserInst); 891448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman} 892448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 893572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::print(raw_ostream &OS) const { 894572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "UserInst="; 895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Store is common and interesting enough to be worth special-casing. 896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) { 897572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "store "; 898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, Store->getOperand(0), /*PrintType=*/false); 899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (UserInst->getType()->isVoidTy()) 900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << UserInst->getOpcodeName(); 901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, UserInst, /*PrintType=*/false); 903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", OperandValToReplace="; 905572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, OperandValToReplace, /*PrintType=*/false); 906572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 907448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (PostIncLoopSet::const_iterator I = PostIncLoops.begin(), 908448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman E = PostIncLoops.end(); I != E; ++I) { 909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", PostIncLoop="; 910448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman WriteAsOperand(OS, (*I)->getHeader(), /*PrintType=*/false); 911934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 912a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LUIdx != ~size_t(0)) 914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", LUIdx=" << LUIdx; 915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 916572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) 917572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offset=" << Offset; 9187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 9197979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::dump() const { 921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 9227979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 9237979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 924572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 925a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 926572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// UniquifierDenseMapInfo - A DenseMapInfo implementation for holding 927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DenseMaps and DenseSets of sorted SmallVectors of const SCEV*. 928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct UniquifierDenseMapInfo { 929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getEmptyKey() { 930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 931572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-1)); 932572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 934934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 935572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getTombstoneKey() { 936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-2)); 938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static unsigned getHashValue(const SmallVector<const SCEV *, 2> &V) { 942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Result = 0; 943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = V.begin(), 944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = V.end(); I != E; ++I) 945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Result ^= DenseMapInfo<const SCEV *>::getHashValue(*I); 9467979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Result; 947a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 948a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 949572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static bool isEqual(const SmallVector<const SCEV *, 2> &LHS, 950572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVector<const SCEV *, 2> &RHS) { 951572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return LHS == RHS; 952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 955572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse - This class holds the state that LSR keeps for each use in 956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// IVUsers, as well as uses invented by LSR itself. It includes information 957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// about what kinds of things can be folded into the user, information about 958572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the user itself, and information about how the use may be satisfied. 959572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Represent multiple users of the same expression in common? 960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRUse { 961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<SmallVector<const SCEV *, 2>, UniquifierDenseMapInfo> Uniquifier; 962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// KindType - An enum for a kind of use, indicating what types of 965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// scaled and immediate operands it might support. 966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman enum KindType { 967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Basic, ///< A normal use, with no folding. 968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Special, ///< A special case of basic, allowing -1 scales. 969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Address, ///< An address use; folding according to TargetLowering 970572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpZero ///< An equality icmp with both operands folded into one. 971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Add a generic icmp too? 972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 9737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman KindType Kind; 975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *AccessTy; 9767979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<int64_t, 8> Offsets; 978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset; 979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MaxOffset; 980203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 981572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AllFixupsOutsideLoop - This records whether all of the fixups using this 982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LSRUse are outside of the loop, in which case some special-case heuristics 983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// may be used. 984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool AllFixupsOutsideLoop; 9857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 986a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// WidestFixupType - This records the widest use type for any fixup using 987a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// this LSRUse. FindUseWithSimilarFormula can't consider uses with different 988a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// max fixup widths to be equivalent, because the narrower one may be relying 989a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// on the implicit truncation to truncate away bogus bits. 990a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman const Type *WidestFixupType; 991a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Formulae - A list of ways to build a value that can satisfy this user. 993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// After the list is populated, one of these is selected heuristically and 994572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// used to formulate a replacement for OperandValToReplace in UserInst. 995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Formula, 12> Formulae; 996589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 997572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Regs - The set of register candidates used by all formulae in this LSRUse. 998572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Regs; 999934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse(KindType K, const Type *T) : Kind(K), AccessTy(T), 1001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MinOffset(INT64_MAX), 1002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MaxOffset(INT64_MIN), 1003a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman AllFixupsOutsideLoop(true), 1004a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman WidestFixupType(0) {} 100556a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1006a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool HasFormulaWithSameRegs(const Formula &F) const; 1007454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman bool InsertFormula(const Formula &F); 1008d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman void DeleteFormula(Formula &F); 1009b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses); 1010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1011572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1014572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1015b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman} 1016b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 1017a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// HasFormula - Test whether this use as a formula which has the same 1018a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// registers as the given formula. 1019a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanbool LSRUse::HasFormulaWithSameRegs(const Formula &F) const { 1020a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1021a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1022a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1023a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman std::sort(Key.begin(), Key.end()); 1024a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return Uniquifier.count(Key); 1025a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 1026a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 1028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 1029454d26dc43207ec537d843229db6f5e6a302e23dDan Gohmanbool LSRUse::InsertFormula(const Formula &F) { 1030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::sort(Key.begin(), Key.end()); 1034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1035572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Uniquifier.insert(Key).second) 1036572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1037572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1038572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Using a register to hold the value of 0 is not profitable. 1039572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert((!F.ScaledReg || !F.ScaledReg->isZero()) && 1040572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Zero allocated in a scaled register!"); 1041572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 1042572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = 1043572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) 1044572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!(*I)->isZero() && "Zero allocated in a base register!"); 1045572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 1046572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1047572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Add the formula to the list. 1048572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formulae.push_back(F); 104956a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1050572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record registers now being used by this use. 1051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) Regs.insert(F.ScaledReg); 1052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1054572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1055572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1057d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman/// DeleteFormula - Remove the given formula from this use's list. 1058d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohmanvoid LSRUse::DeleteFormula(Formula &F) { 10595ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&F != &Formulae.back()) 10605ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(F, Formulae.back()); 1061d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman Formulae.pop_back(); 1062a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman assert(!Formulae.empty() && "LSRUse has no formulae left!"); 1063d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman} 1064d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman 1065b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman/// RecomputeRegs - Recompute the Regs field, and update RegUses. 1066b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) { 1067b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 1068b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman SmallPtrSet<const SCEV *, 4> OldRegs = Regs; 1069b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.clear(); 1070402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = Formulae.begin(), 1071402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Formulae.end(); I != E; ++I) { 1072402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 1073b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (F.ScaledReg) Regs.insert(F.ScaledReg); 1074b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1075b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman } 1076b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1077b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Update the RegTracker. 1078b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (SmallPtrSet<const SCEV *, 4>::iterator I = OldRegs.begin(), 1079b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman E = OldRegs.end(); I != E; ++I) 1080b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (!Regs.count(*I)) 1081b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUses.DropRegister(*I, LUIdx); 1082b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 1083b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::print(raw_ostream &OS) const { 1085572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR Use: Kind="; 1086572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Basic: OS << "Basic"; break; 1088572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Special: OS << "Special"; break; 1089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case ICmpZero: OS << "ICmpZero"; break; 1090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Address: 1091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "Address of "; 10921df9859c40492511b8aa4321eb76496005d3b75bDuncan Sands if (AccessTy->isPointerTy()) 1093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "pointer"; // the full pointer type could be really verbose 10947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else 1095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *AccessTy; 10967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 10971bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offsets={"; 1099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 1100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 1101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *I; 1102ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes if (llvm::next(I) != E) 1103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ','; 1104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '}'; 1106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AllFixupsOutsideLoop) 1108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", all-fixups-outside-loop"; 1109a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 1110a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (WidestFixupType) 1111a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman OS << ", widest fixup type: " << *WidestFixupType; 11127979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 1113d6b62a572210aff965a55626cf36a68821838844Evan Cheng 1114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::dump() const { 1115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 1116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 111756a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// isLegalUse - Test whether the use described by AM is "legal", meaning it can 1119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// be completely folded into the user instruction at isel time. This includes 1120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// address-mode folding and special icmp tricks. 1121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(const TargetLowering::AddrMode &AM, 1122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, const Type *AccessTy, 1123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Address: 1126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can 1127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // completely fold this address. 1128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) return TLI->isLegalAddressingMode(AM, AccessTy); 1129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise, just guess that reg+reg addressing is legal. 1131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.BaseOffs == 0 && AM.Scale <= 1; 1132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::ICmpZero: 1134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // There's not even a target hook for querying whether it would be legal to 1135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // fold a GV into an ICmp. 1136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) 1137572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1138579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner 1139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only has two operands; don't allow more than two non-trivial parts. 1140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.HasBaseReg && AM.BaseOffs != 0) 1141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 11421bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by 1144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // putting the scaled register in the other operand of the icmp. 1145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.Scale != -1) 11467979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 114781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can fold an 1149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // integer immediate on an icmp. 1150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 1151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) return TLI->isLegalICmpImmediate(-AM.BaseOffs); 1152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 11557979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return true; 115681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1157572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Basic: 1158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Only handle single-register values. 1159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.Scale == 0 && AM.BaseOffs == 0; 116081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Special: 1162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Only handle -1 scales, or no scale. 1163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return AM.Scale == 0 || AM.Scale == -1; 1164572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1166572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1167572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 116881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(TargetLowering::AddrMode AM, 1170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, const Type *AccessTy, 1172572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1173572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1174572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MinOffset) > AM.BaseOffs) != 1175572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MinOffset > 0)) 1176572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1177572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MinOffset; 1178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(AM, Kind, AccessTy, TLI)) { 1179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs - MinOffset; 1180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MaxOffset) > AM.BaseOffs) != 1182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MaxOffset > 0)) 11837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MaxOffset; 1185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 11867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 11885f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 11895f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(int64_t BaseOffs, 1191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV, 1192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, const Type *AccessTy, 1194454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman const TargetLowering *TLI) { 1195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1206a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Canonicalize a scale of 1 to a base register if the formula doesn't 1207a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // already have a base register. 1208a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!AM.HasBaseReg && AM.Scale == 1) { 1209a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.Scale = 0; 1210a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.HasBaseReg = true; 1211a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1212a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 1214a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1215a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(const SCEV *S, 1217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, const Type *AccessTy, 1220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI, 1221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE) { 1222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (S->isZero()) return true; 1224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t BaseOffs = ExtractImmediate(S, SE); 1228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV = ExtractSymbol(S, SE); 1229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If there's anything else involved, it's not foldable. 1231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!S->isZero()) return false; 1232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, MinOffset, MaxOffset, Kind, AccessTy, TLI); 1245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 124656a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1247b6211710acdf558b3b45c2d198e74aa602496893Dan Gohmannamespace { 1248b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 12491e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// UseMapDenseMapInfo - A DenseMapInfo implementation for holding 12501e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// DenseMaps and DenseSets of pairs of const SCEV* and LSRUse::Kind. 12511e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohmanstruct UseMapDenseMapInfo { 12521e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getEmptyKey() { 12531e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-1), LSRUse::Basic); 12541e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 12551e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 12561e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getTombstoneKey() { 12571e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-2), LSRUse::Basic); 12581e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 12591e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 12601e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static unsigned 12611e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman getHashValue(const std::pair<const SCEV *, LSRUse::KindType> &V) { 12621e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman unsigned Result = DenseMapInfo<const SCEV *>::getHashValue(V.first); 12631e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman Result ^= DenseMapInfo<unsigned>::getHashValue(unsigned(V.second)); 12641e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return Result; 12651e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 12661e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 12671e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static bool isEqual(const std::pair<const SCEV *, LSRUse::KindType> &LHS, 12681e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman const std::pair<const SCEV *, LSRUse::KindType> &RHS) { 12691e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return LHS == RHS; 12701e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 12711e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman}; 12721e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 1273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRInstance - This class holds state for the main loop strength reduction 1274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// logic. 1275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRInstance { 1276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers &IU; 1277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE; 1278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DominatorTree &DT; 1279e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LoopInfo &LI; 1280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 1281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loop *const L; 1282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed; 1283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// IVIncInsertPos - This is the insert position that the current loop's 1285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable increment should be placed. In simple loops, this is 1286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// the latch block's terminator. But in more complicated cases, this is a 1287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// position which will dominate all the in-loop post-increment users. 1288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *IVIncInsertPos; 1289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Factors - Interesting factors between use strides. 1291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<int64_t, 8> Factors; 1292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Types - Interesting use types, to facilitate truncation reuse. 1294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const Type *, 4> Types; 1295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Fixups - The list of operands which are to be replaced. 1297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRFixup, 16> Fixups; 1298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Uses - The list of interesting uses. 1300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRUse, 16> Uses; 1301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// RegUses - Track which uses use which register candidates. 1303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUseTracker RegUses; 1304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void OptimizeShadowIV(); 1306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse); 1307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse); 1308c6519f916b5922de81c53547fd21364994195a70Dan Gohman void OptimizeLoopTermCond(); 1309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectInterestingTypesAndFactors(); 1311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectFixupsAndInitialFormulae(); 1312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &getNewFixup() { 1314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Fixups.push_back(LSRFixup()); 1315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Fixups.back(); 1316a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 131732e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng 1318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Support for sharing of LSRUses between LSRFixups. 13191e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman typedef DenseMap<std::pair<const SCEV *, LSRUse::KindType>, 13201e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman size_t, 13211e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMapDenseMapInfo> UseMapTy; 1322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UseMapTy UseMap; 1323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1324191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 1325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, const Type *AccessTy); 1326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> getUse(const SCEV *&Expr, 1328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, 1329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *AccessTy); 1330572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1331c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void DeleteUse(LSRUse &LU, size_t LUIdx); 13325ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 1333191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU); 1334a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 1336454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegisters(const Formula &F, size_t LUIdx); 1339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F); 1340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectLoopInvariantFixupsAndFormulae(); 1342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base, 1344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth = 0); 1345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base); 1346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base); 1351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCrossUseConstantOffsets(); 1352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateAllReuseFormulae(); 1353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void FilterOutUndesirableDedicatedRegisters(); 1355d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 1356d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t EstimateSearchSpaceComplexity() const; 13574aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByDetectingSupersets(); 13584aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByCollapsingUnrolledCode(); 13594f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 13604aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByPickingWinnerRegs(); 1361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void NarrowSearchSpaceUsingHeuristics(); 1362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 1364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 1365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 1366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 1367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 1368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const; 1369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Solve(SmallVectorImpl<const Formula *> &Solution) const; 1370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1371e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock::iterator 1372e5f76877aee6f33964de105893f0ef338661ecadDan Gohman HoistInsertPosition(BasicBlock::iterator IP, 1373e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) const; 1374e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP, 1375e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRFixup &LF, 1376e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRUse &LU) const; 1377d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 1378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *Expand(const LSRFixup &LF, 1379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1380454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman BasicBlock::iterator IP, 1381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1382454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const; 13833a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman void RewriteForPHI(PHINode *PN, const LSRFixup &LF, 13843a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 13853a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 13863a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 13873a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const; 1388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Rewrite(const LSRFixup &LF, 1389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 1392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const; 1393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 1394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P); 1395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRInstance(const TargetLowering *tli, Loop *l, Pass *P); 1397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool getChanged() const { return Changed; } 1399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_factors_and_types(raw_ostream &OS) const; 1401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_fixups(raw_ostream &OS) const; 1402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_uses(raw_ostream &OS) const; 1403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 14067979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1407a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1408c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// OptimizeShadowIV - If IV is used in a int-to-float cast 14103f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// inside the loop then try to eliminate the cast operation. 1411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::OptimizeShadowIV() { 1412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 1413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 1414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 1415c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1416572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); 1417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != E; /* empty */) { 1418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers::const_iterator CandidateUI = UI; 1419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ++UI; 1420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *ShadowUse = CandidateUI->getUser(); 1421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *DestTy = NULL; 1422c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1423572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* If shadow use is a int->float cast then insert a second IV 1424572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman to eliminate this cast. 1425c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1426572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i) 1427572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo((double)i); 1428c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1429572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman is transformed into 1430c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1431572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman double d = 0.0; 1432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i, ++d) 1433572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo(d); 1434572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman */ 1435572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) 1436572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = UCast->getDestTy(); 1437572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) 1438572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = SCast->getDestTy(); 1439572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DestTy) continue; 14405792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) { 1442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If target does not support DestTy natively then do not apply 1443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // this transformation. 1444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman EVT DVT = TLI->getValueType(DestTy); 1445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI->isTypeLegal(DVT)) continue; 14467e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 14477e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); 1449572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!PH) continue; 1450572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getNumIncomingValues() != 2) continue; 1451a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *SrcTy = PH->getType(); 1453572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int Mantissa = DestTy->getFPMantissaWidth(); 1454572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mantissa == -1) continue; 1455572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa) 1456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 14572f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 1458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Entry, Latch; 1459572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { 1460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 0; 1461572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 1; 1462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 1463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 1; 1464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 0; 1465a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1466eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 1467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); 1468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Init) continue; 1469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *NewInit = ConstantFP::get(DestTy, Init->getZExtValue()); 1470a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *Incr = 1472572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); 1473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Incr) continue; 1474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOpcode() != Instruction::Add 1475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman && Incr->getOpcode() != Instruction::Sub) 1476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1477aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 1478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Initialize new IV, double d = 0.0 in above example. */ 1479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *C = NULL; 1480572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOperand(0) == PH) 1481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(1)); 1482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Incr->getOperand(1) == PH) 1483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(0)); 1484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1486bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman 1487572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C) continue; 1488a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1489572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore negative constants, as the code below doesn't handle them 1490572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // correctly. TODO: Remove this restriction. 1491572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C->getValue().isStrictlyPositive()) continue; 1492a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Add new PHINode. */ 1494572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode *NewPH = PHINode::Create(DestTy, "IV.S.", PH); 1495a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1496572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* create new increment. '++d' in above example. */ 1497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue()); 1498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *NewIncr = 1499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ? 1500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction::FAdd : Instruction::FSub, 1501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH, CFP, "IV.S.next.", Incr); 1502a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); 1504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); 15058b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 1506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Remove cast operation */ 1507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->replaceAllUsesWith(NewPH); 1508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->eraseFromParent(); 1509c6519f916b5922de81c53547fd21364994195a70Dan Gohman Changed = true; 1510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 15117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1512a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1513cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 15147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// FindIVUserForCond - If Cond has an operand that is an expression of an IV, 15157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// set the IV user and stride information and return true, otherwise return 15167979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// false. 1517ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanbool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) { 1518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UI->getUser() == Cond) { 1520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // NOTE: we could handle setcc instructions with multiple uses here, but 1521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // InstCombine does it as well for simple uses, it's not clear that it 1522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // occurs enough in real life to handle. 1523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CondUse = UI; 1524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1525a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1527cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 1528cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 15297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// OptimizeMax - Rewrite the loop's terminating condition if it uses 15307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a max computation. 15317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This is a narrow solution to a specific, but acute, problem. For loops 15337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// like this: 15347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 15367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 15377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 15387979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 15397979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15407979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the trip count isn't just 'n', because 'n' might not be positive. And 15417979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// unfortunately this can come up even for loops where the user didn't use 15427979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a C do-while loop. For example, seemingly well-behaved top-test loops 15437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// will commonly be lowered like this: 15447979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman// 15457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// if (n > 0) { 15467979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 15477979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 15487979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 15497979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 15507979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } 15517979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15527979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// and then it's possible for subsequent optimization to obscure the if 15537979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// test in such a way that indvars can't find it. 15547979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15557979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// When indvars can't find the if test in loops like this, it creates a 15567979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max expression, which allows it to give the loop a canonical 15577979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// induction variable: 15587979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15597979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 15607979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max = n < 1 ? 1 : n; 15617979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 15627979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 15637979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i != max); 15647979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15657979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// Canonical induction variables are necessary because the loop passes 15667979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// are designed around them. The most obvious example of this is the 15677979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// LoopInfo analysis, which doesn't remember trip count values. It 15687979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// expects to be able to rediscover the trip count each time it is 1569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needed, and it does this using a simple analysis that only succeeds if 15707979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the loop has a canonical induction variable. 15717979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// However, when it comes time to generate code, the maximum operation 15737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// can be quite costly, especially if it's inside of an outer loop. 15747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15757979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This function solves this problem by detecting this type of loop and 15767979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting 15777979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the instructions for the maximum computation. 15787979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 1579572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { 15807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check that the loop matches the pattern we're looking for. 15817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() != CmpInst::ICMP_EQ && 15827979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->getPredicate() != CmpInst::ICMP_NE) 15837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1584ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 15857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); 15867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!Sel || !Sel->hasOneUse()) return Cond; 1587ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 1588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 15897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 15907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1591deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); 1592ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 15937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Add one to the backedge-taken count to get the trip count. 15944065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); 15951d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (IterationCount != SE.getSCEV(Sel)) return Cond; 15961d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 15971d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Check for a max calculation that matches the pattern. There's no check 15981d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for ICMP_ULE here because the comparison would be with zero, which 15991d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // isn't interesting. 16001d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; 16011d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman const SCEVNAryExpr *Max = 0; 16021d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) { 16031d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLE; 16041d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 16051d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) { 16061d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLT; 16071d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 16081d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) { 16091d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_ULT; 16101d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = U; 16111d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else { 16121d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // No match; bail. 16137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 16141d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } 1615ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16167979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // To handle a max with more than two operands, this optimization would 16177979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // require additional checking and setup. 16187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Max->getNumOperands() != 2) 16197979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 16207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 16217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxLHS = Max->getOperand(0); 16227979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxRHS = Max->getOperand(1); 16231d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 16241d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // ScalarEvolution canonicalizes constants to the left. For < and >, look 16251d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for a comparison with 1. For <= and >=, a comparison with zero. 16261d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!MaxLHS || 16271d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One))) 16281d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 16291d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 16307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the relevant induction variable for conformance to 16317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // the pattern. 1632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); 16337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 16347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!AR || !AR->isAffine() || 16357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AR->getStart() != One || 1636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE) != One) 16377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1638ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16397979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman assert(AR->getLoop() == L && 16407979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman "Loop condition operand is an addrec in a different loop!"); 1641bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman 16427979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the right operand of the select, and remember it, as it will 16437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // be used in the new comparison instruction. 16447979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Value *NewRHS = 0; 16451d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (ICmpInst::isTrueWhenEqual(Pred)) { 16461d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Look for n+1, and grab n. 16471d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1))) 16481d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 16491d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 16501d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 16511d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 16521d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2))) 16531d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 16541d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 16551d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 16561d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 16571d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!NewRHS) 16581d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 16591d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS) 16607979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(1); 1661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS) 16627979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(2); 1663caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS)) 1664caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman NewRHS = SU->getValue(); 16651d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman else 1666caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman // Max doesn't match expected pattern. 1667caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman return Cond; 1668ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16697979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Determine the new comparison opcode. It may be signed or unsigned, 16707979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // and the original comparison may be either equality or inequality. 16717979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() == CmpInst::ICMP_EQ) 16727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Pred = CmpInst::getInversePredicate(Pred); 16732781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman 16747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Ok, everything looks ok to change the condition into an SLT or SGE and 16757979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // delete the max calculation. 16767979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman ICmpInst *NewCond = 16777979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp"); 1678ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16797979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Delete the max calculation instructions. 16807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->replaceAllUsesWith(NewCond); 16817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman CondUse->setUser(NewCond); 16827979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); 16837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->eraseFromParent(); 16847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Sel->eraseFromParent(); 16857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cmp->use_empty()) 16867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cmp->eraseFromParent(); 16877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return NewCond; 1688ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman} 1689ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 169056a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach/// OptimizeLoopTermCond - Change loop terminating condition to use the 16912d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng/// postinc iv when possible. 1692c6519f916b5922de81c53547fd21364994195a70Dan Gohmanvoid 1693572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::OptimizeLoopTermCond() { 1694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<Instruction *, 4> PostIncs; 1695572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 16965792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng BasicBlock *LatchBlock = L->getLoopLatch(); 1697076e085698c484354c9e131f1bd8fd001581397bEvan Cheng SmallVector<BasicBlock*, 8> ExitingBlocks; 1698076e085698c484354c9e131f1bd8fd001581397bEvan Cheng L->getExitingBlocks(ExitingBlocks); 169956a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1700076e085698c484354c9e131f1bd8fd001581397bEvan Cheng for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 1701076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BasicBlock *ExitingBlock = ExitingBlocks[i]; 1702010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 1703572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Get the terminating condition for the loop if possible. If we 1704076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // can, we want to change it to use a post-incremented version of its 1705076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // induction variable, to allow coalescing the live ranges for the IV into 1706076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // one register value. 1707cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1708076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); 1709076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (!TermBr) 1710076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1711076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // FIXME: Overly conservative, termination condition could be an 'or' etc.. 1712076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition())) 1713076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1714076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 1715076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Search IVUsesByStride to find Cond's IVUse if there is one. 1716076e085698c484354c9e131f1bd8fd001581397bEvan Cheng IVStrideUse *CondUse = 0; 1717076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); 1718572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!FindIVUserForCond(Cond, CondUse)) 1719076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1720076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 17217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // If the trip count is computed in terms of a max (due to ScalarEvolution 17227979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // being unable to find a sufficient guard, for example), change the loop 17237979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // comparison to use SLT or ULT instead of NE. 1724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // One consequence of doing this now is that it disrupts the count-down 1725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // optimization. That's not always a bad thing though, because in such 1726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // cases it may still be worthwhile to avoid a max. 1727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cond = OptimizeMax(Cond, CondUse); 1728572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1729572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this exiting block dominates the latch block, it may also use 1730572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the post-inc value if it won't be shared with other uses. 1731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for dominance. 1732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(ExitingBlock, LatchBlock)) 17337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman continue; 17347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively avoid trying to use the post-inc value in non-latch 1736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // exits if there may be pre-inc users in intervening blocks. 1737590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (LatchBlock != ExitingBlock) 1738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Test if the use is reachable from the exiting block. This dominator 1740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // query is a conservative approximation of reachability. 1741572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*UI != CondUse && 1742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) { 1743572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume there may be reuse if the quotient of their 1744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // strides could be a legal scale. 1745c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *A = IU.getStride(*CondUse, L); 1746c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *B = IU.getStride(*UI, L); 1747448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (!A || !B) continue; 1748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) != 1749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) { 1750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) > 1751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) 1752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman B = SE.getSignExtendExpr(B, A->getType()); 1753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman A = SE.getSignExtendExpr(A, B->getType()); 1755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *D = 1757f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) { 17589f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const ConstantInt *C = D->getValue(); 1759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Stride of one or negative one can have reuse with non-addresses. 17609f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->isOne() || C->isAllOnesValue()) 1761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Avoid weird situations. 17639f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->getValue().getMinSignedBits() >= 64 || 17649f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman C->getValue().isMinSignedValue()) 1765572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1766590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // Without TLI, assume that any stride might be valid, and so any 1767590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // use might be shared. 1768590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (!TLI) 1769590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman goto decline_post_inc; 1770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for possible scaled-address reuse. 1771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *AccessTy = getAccessType(UI->getUser()); 1772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 17739f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman AM.Scale = C->getSExtValue(); 17742763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 1775572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = -AM.Scale; 17772763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 1778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1779572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 178263c9463c62fce8cbe02176dfa2d73f375a06f1f2David Greene DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: " 1783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << *Cond << '\n'); 178456a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1785076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // It's possible for the setcc instruction to be anywhere in the loop, and 1786076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // possible for it to have multiple users. If it is not immediately before 1787076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // the exiting block branch, move it. 1788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*++BasicBlock::iterator(Cond) != TermBr) { 1789572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Cond->hasOneUse()) { 1790076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->moveBefore(TermBr); 1791076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } else { 1792572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clone the terminating condition and insert into the loopend. 1793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OldCond = Cond; 1794076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond = cast<ICmpInst>(Cond->clone()); 1795076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->setName(L->getHeader()->getName() + ".termcond"); 1796076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ExitingBlock->getInstList().insert(TermBr, Cond); 1797076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 1798076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Clone the IVUse, as the old use still exists! 1799c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace()); 1800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TermBr->replaceUsesOfWith(OldCond, Cond); 1801076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } 1802010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 1803010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 1804076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // If we get to here, we know that we can transform the setcc instruction to 1805076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // use the post-incremented version of the IV, allowing us to coalesce the 1806076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // live ranges for the IV correctly. 1807448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman CondUse->transformToPostInc(L); 1808076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Changed = true; 18095792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PostIncs.insert(Cond); 1811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman decline_post_inc:; 1812c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen } 1813c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 1814572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine an insertion point for the loop induction variable increment. It 1815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // must dominate all the post-inc comparisons we just set up, and it must 1816572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // dominate the loop latch edge. 1817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = L->getLoopLatch()->getTerminator(); 1818572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<Instruction *, 4>::const_iterator I = PostIncs.begin(), 1819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = PostIncs.end(); I != E; ++I) { 1820572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BasicBlock *BB = 1821572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT.findNearestCommonDominator(IVIncInsertPos->getParent(), 1822572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (*I)->getParent()); 1823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BB == (*I)->getParent()) 1824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = *I; 1825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (BB != IVIncInsertPos->getParent()) 1826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = BB->getTerminator(); 1827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1829a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 183076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// reconcileNewOffset - Determine if the given use can accomodate a fixup 183176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// at the given offset and other details. If so, update the use and 183276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// return true. 1833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 1834191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan GohmanLSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 1835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, const Type *AccessTy) { 1836191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMinOffset = LU.MinOffset; 1837191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMaxOffset = LU.MaxOffset; 1838191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman const Type *NewAccessTy = AccessTy; 1839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1840572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched kind. It's tempting to collapse mismatched kinds to 1841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // something conservative, however this can pessimize in the case that one of 1842572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the uses will have all its uses outside the loop, for example. 1843572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != Kind) 1844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1845572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume HasBaseReg is true for now. 1846191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset < LU.MinOffset) { 1847191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(LU.MaxOffset - NewOffset, 0, HasBaseReg, 1848454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 18497979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1850191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMinOffset = NewOffset; 1851191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } else if (NewOffset > LU.MaxOffset) { 1852191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(NewOffset - LU.MinOffset, 0, HasBaseReg, 1853454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 18547979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1855191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMaxOffset = NewOffset; 1856a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1857572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched access type, and fall back conservatively as needed. 185874e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // TODO: Be less conservative when the type is similar and can use the same 185974e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // addressing modes. 1860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Kind == LSRUse::Address && AccessTy != LU.AccessTy) 1861191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewAccessTy = Type::getVoidTy(AccessTy->getContext()); 1862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the use. 1864191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MinOffset = NewMinOffset; 1865191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MaxOffset = NewMaxOffset; 1866191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.AccessTy = NewAccessTy; 1867191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset != LU.Offsets.back()) 1868191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(NewOffset); 1869572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1871a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1872572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getUse - Return an LSRUse index and an offset value for a fixup which 1873572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needs the given expression, with the given kind and optional access type. 18743f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// Either reuse an existing use or create a new one, as needed. 1875572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstd::pair<size_t, int64_t> 1876572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::getUse(const SCEV *&Expr, 1877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, const Type *AccessTy) { 1878572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Copy = Expr; 1879572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = ExtractImmediate(Expr, SE); 1880572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Basic uses can't accept any offset, for example. 1882454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!isAlwaysFoldable(Offset, 0, /*HasBaseReg=*/true, Kind, AccessTy, TLI)) { 1883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Expr = Copy; 1884572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = 0; 1885a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1886a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1887572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<UseMapTy::iterator, bool> P = 18881e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMap.insert(std::make_pair(std::make_pair(Expr, Kind), 0)); 1889572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!P.second) { 1890572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // A use already existed with this base. 1891572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = P.first->second; 1892572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 1893191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy)) 1894572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Reuse this use. 1895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 1896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1897a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Create a new use. 1899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = Uses.size(); 1900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman P.first->second = LUIdx; 1901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses.push_back(LSRUse(Kind, AccessTy)); 1902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 1903a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1904191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // We don't need to track redundant offsets, but we don't need to go out 1905191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // of our way here to avoid them. 1906191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LU.Offsets.empty() || Offset != LU.Offsets.back()) 1907191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(Offset); 1908191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 1909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MinOffset = Offset; 1910572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MaxOffset = Offset; 1911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 1912a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1913a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 19145ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteUse - Delete the given use from the Uses list. 1915c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohmanvoid LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) { 1916191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (&LU != &Uses.back()) 19175ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(LU, Uses.back()); 19185ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman Uses.pop_back(); 1919c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 1920c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. 1921c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman RegUses.SwapAndDropUse(LUIdx, Uses.size()); 19225ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 19235ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 1924a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// FindUseWithFormula - Look for a use distinct from OrigLU which is has 1925a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// a formula that has the same registers as the given formula. 1926a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRUse * 1927a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRInstance::FindUseWithSimilarFormula(const Formula &OrigF, 1928191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman const LSRUse &OrigLU) { 1929191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Search all uses for the formula. This could be more clever. 1930a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 1931a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 19326a832715325f740a0e78acd194abad9104389075Dan Gohman // Check whether this use is close enough to OrigLU, to see whether it's 19336a832715325f740a0e78acd194abad9104389075Dan Gohman // worthwhile looking through its formulae. 19346a832715325f740a0e78acd194abad9104389075Dan Gohman // Ignore ICmpZero uses because they may contain formulae generated by 19356a832715325f740a0e78acd194abad9104389075Dan Gohman // GenerateICmpZeroScales, in which case adding fixup offsets may 19366a832715325f740a0e78acd194abad9104389075Dan Gohman // be invalid. 1937a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (&LU != &OrigLU && 1938a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind != LSRUse::ICmpZero && 1939a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy && 1940a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType == OrigLU.WidestFixupType && 1941a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.HasFormulaWithSameRegs(OrigF)) { 19426a832715325f740a0e78acd194abad9104389075Dan Gohman // Scan through this use's formulae. 1943402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 1944402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 1945402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 19466a832715325f740a0e78acd194abad9104389075Dan Gohman // Check to see if this formula has the same registers and symbols 19476a832715325f740a0e78acd194abad9104389075Dan Gohman // as OrigF. 1948a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.BaseRegs == OrigF.BaseRegs && 1949a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.ScaledReg == OrigF.ScaledReg && 1950a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.AM.BaseGV == OrigF.AM.BaseGV && 1951e39a47c5b3bdab84f98046fc133a89170a9f0ee9Dan Gohman F.AM.Scale == OrigF.AM.Scale) { 1952191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (F.AM.BaseOffs == 0) 1953a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return &LU; 19546a832715325f740a0e78acd194abad9104389075Dan Gohman // This is the formula where all the registers and symbols matched; 19556a832715325f740a0e78acd194abad9104389075Dan Gohman // there aren't going to be any others. Since we declined it, we 19566a832715325f740a0e78acd194abad9104389075Dan Gohman // can skip the rest of the formulae and procede to the next LSRUse. 1957a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 1958a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1959a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1960a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1961a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1962a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 19636a832715325f740a0e78acd194abad9104389075Dan Gohman // Nothing looked good. 1964a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return 0; 1965a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 1966a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectInterestingTypesAndFactors() { 1968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> Strides; 1969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 19701b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Collect interesting types and strides. 1971448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman SmallVector<const SCEV *, 4> Worklist; 1972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 1973c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *Expr = IU.getExpr(*UI); 1974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect interesting types. 1976448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Types.insert(SE.getEffectiveSCEVType(Expr->getType())); 1977448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 1978448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // Add strides for mentioned loops. 1979448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(Expr); 1980448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman do { 1981448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman const SCEV *S = Worklist.pop_back_val(); 1982448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 1983448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Strides.insert(AR->getStepRecurrence(SE)); 1984448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(AR->getStart()); 1985448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 1986403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(Add->op_begin(), Add->op_end()); 1987448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 1988448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } while (!Worklist.empty()); 19891b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman } 19901b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman 19911b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Compute interesting factors from the set of interesting strides. 19921b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator 19931b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman I = Strides.begin(), E = Strides.end(); I != E; ++I) 1994572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = 1995ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes llvm::next(I); NewStrideIter != E; ++NewStrideIter) { 19961b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman const SCEV *OldStride = *I; 1997572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewStride = *NewStrideIter; 1998a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1999572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) != 2000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) { 2001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) > 2002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) 2003572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType()); 2004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType()); 2006572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2007572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *Factor = 2008f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride, 2009f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2011572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVConstant *Factor = 2013454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride, 2014454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman NewStride, 2015f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2016572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2017572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2018a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2019a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2020586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2021572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If all uses use the same type, don't bother looking for truncation-based 2022572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reuse. 2023572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Types.size() == 1) 2024572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 2025572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_factors_and_types(dbgs())); 2027c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen} 2028c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectFixupsAndInitialFormulae() { 2030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 2031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record the uses. 2032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 2033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst = UI->getUser(); 2034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI->getOperandValToReplace(); 2035448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.PostIncLoops = UI->getPostIncLoops(); 20360f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel 2037572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind = LSRUse::Basic; 2038572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *AccessTy = 0; 2039572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAddressUse(LF.UserInst, LF.OperandValToReplace)) { 2040572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::Address; 2041572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = getAccessType(LF.UserInst); 2042572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2043572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2044c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *S = IU.getExpr(*UI); 2045572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2046572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Equality (== and !=) ICmps are special. We can rewrite (i == N) as 2047572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // (N - i == 0), and this allows (N - i) to be the expression that we work 2048572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with rather than just N or i, so we can consider the register 2049572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // requirements for both N and i at the same time. Limiting this code to 2050572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps is not a problem because all interesting loops use 2051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps, thanks to IndVarSimplify. 2052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpInst *CI = dyn_cast<ICmpInst>(LF.UserInst)) 2053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (CI->isEquality()) { 2054572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Swap the operands if needed to put the OperandValToReplace on the 2055572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // left, for consistency. 2056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *NV = CI->getOperand(1); 2057572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NV == LF.OperandValToReplace) { 2058572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, CI->getOperand(0)); 2059572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(0, NV); 2060f182b23f8fed014c26da48d94890080c7d916a0cDan Gohman NV = CI->getOperand(1); 20619da1bf4845e670096b9bf9e62c40960af1697ea0Dan Gohman Changed = true; 2062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2064572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // x == y --> x - y == 0 2065572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *N = SE.getSCEV(NV); 206617ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (SE.isLoopInvariant(N, L)) { 2067572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::ICmpZero; 2068572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman S = SE.getMinusSCEV(N, S); 2069572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2070572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2071572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // -1 and the negations of all interesting strides (except the negation 2072572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of -1) are now also interesting. 2073572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Factors.size(); i != e; ++i) 2074572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors[i] != -1) 2075572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-(uint64_t)Factors[i]); 2076572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-1); 2077572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2079572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Set up the initial formula for this use. 2080572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy); 2081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 2082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 2083572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 2084448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 2085a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 2086a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 2087a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 2088a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 2089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is the first use of this LSRUse, give it a formula. 2091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Formulae.empty()) { 2092454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman InsertInitialFormula(S, LU, LF.LUIdx); 2093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), LF.LUIdx); 2094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2097572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_fixups(dbgs())); 2098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 210076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertInitialFormula - Insert a formula for the given expression into 210176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given use, separating out loop-variant portions from loop-invariant 210276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// and loop-computable portions. 2103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2104454d26dc43207ec537d843229db6f5e6a302e23dDan GohmanLSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { 2105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2106dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman F.InitialMatch(S, L, SE); 2107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Initial formula already exists!"); (void)Inserted; 2109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 211176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertSupplementalFormula - Insert a simple single-register formula for 211276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given expression into the given use. 2113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2114572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::InsertSupplementalFormula(const SCEV *S, 2115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU, size_t LUIdx) { 2116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(S); 2118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.HasBaseReg = true; 2119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Supplemental formula already exists!"); (void)Inserted; 2121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CountRegisters - Note which registers are used by the given formula, 2124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// updating RegUses. 2125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) { 2126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) 2127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(F.ScaledReg, LUIdx); 2128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 2129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) 2130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(*I, LUIdx); 2131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 2134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 2135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) { 2136454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!LU.InsertFormula(F)) 21377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 213880b0f8c0628d848d36f04440687b61e3b2d3dff1Dan Gohman 2139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(F, LUIdx); 2140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 2141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectLoopInvariantFixupsAndFormulae - Check for other uses of 2144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-invariant values which we're tracking. These other uses will pin these 2145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// values in registers, making them less profitable for elimination. 2146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: This currently misses non-constant addrec step registers. 2147572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Should this give more weight to users inside the loop? 2148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2149572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::CollectLoopInvariantFixupsAndFormulae() { 2150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); 2151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 8> Inserted; 2152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!Worklist.empty()) { 2154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *S = Worklist.pop_back_val(); 2155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S)) 2157403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(N->op_begin(), N->op_end()); 2158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S)) 2159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(C->getOperand()); 2160572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) { 2161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getLHS()); 2162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getRHS()); 2163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 2164572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Inserted.insert(U)) continue; 2165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Value *V = U->getValue(); 2166a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 2167a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Look for instructions defined outside the loop. 2168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (L->contains(Inst)) continue; 2169a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman } else if (isa<UndefValue>(V)) 2170a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Undef doesn't have a live range, so it doesn't matter. 2171a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman continue; 217260ad781c61815ca5b8dc2a45a102e1c8af65992fGabor Greif for (Value::const_use_iterator UI = V->use_begin(), UE = V->use_end(); 2173572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != UE; ++UI) { 2174572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Instruction *UserInst = dyn_cast<Instruction>(*UI); 2175572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore non-instructions. 2176572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!UserInst) 2177572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions in other functions (as can happen with 2179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Constants). 2180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UserInst->getParent()->getParent() != L->getHeader()->getParent()) 2181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions not dominated by the loop. 2183572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const BasicBlock *UseBB = !isa<PHINode>(UserInst) ? 2184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UserInst->getParent() : 2185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman cast<PHINode>(UserInst)->getIncomingBlock( 2186572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode::getIncomingValueNumForOperand(UI.getOperandNo())); 2187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(L->getHeader(), UseBB)) 2188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore uses which are part of other SCEV expressions, to avoid 2190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // analyzing them multiple times. 21914a2a68336638e99a475b09a9278399db6749618fDan Gohman if (SE.isSCEVable(UserInst->getType())) { 21924a2a68336638e99a475b09a9278399db6749618fDan Gohman const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); 21934a2a68336638e99a475b09a9278399db6749618fDan Gohman // If the user is a no-op, look through to its uses. 21944a2a68336638e99a475b09a9278399db6749618fDan Gohman if (!isa<SCEVUnknown>(UserS)) 21954a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 21964a2a68336638e99a475b09a9278399db6749618fDan Gohman if (UserS == U) { 21974a2a68336638e99a475b09a9278399db6749618fDan Gohman Worklist.push_back( 21984a2a68336638e99a475b09a9278399db6749618fDan Gohman SE.getUnknown(const_cast<Instruction *>(UserInst))); 21994a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 22004a2a68336638e99a475b09a9278399db6749618fDan Gohman } 22014a2a68336638e99a475b09a9278399db6749618fDan Gohman } 2202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore icmp instructions which are already being analyzed. 2203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) { 2204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned OtherIdx = !UI.getOperandNo(); 2205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx)); 220617ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L)) 2207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 2211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst = const_cast<Instruction *>(UserInst); 2212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI.getUse(); 2213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, LSRUse::Basic, 0); 2214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 2215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 2216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 2217448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 2218a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 2219a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 2220a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 2221a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 2222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman InsertSupplementalFormula(U, LU, LF.LUIdx); 2223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), Uses.size() - 1); 2224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 2225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectSubexprs - Split S into subexpressions which can be pulled out into 2231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// separate registers. If C is non-null, multiply each subexpression by C. 2232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic void CollectSubexprs(const SCEV *S, const SCEVConstant *C, 2233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Ops, 22343e3f15bb09eaca4d259e531c9a1ecafb5710b57bDan Gohman const Loop *L, 2235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE) { 2236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 2237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break out add operands. 2238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 2239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) 22403e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(*I, C, Ops, L, SE); 2241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 2243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Split a non-zero base out of an addrec. 2244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!AR->getStart()->isZero()) { 2245deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman CollectSubexprs(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), 2246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE), 22473228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick AR->getLoop(), 22483228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick //FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 22493228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap), 22503e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman C, Ops, L, SE); 22513e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(AR->getStart(), C, Ops, L, SE); 2252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 2255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break (C * (a + b + c)) into C*a + C*b + C*c. 2256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mul->getNumOperands() == 2) 2257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *Op0 = 2258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<SCEVConstant>(Mul->getOperand(0))) { 2259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectSubexprs(Mul->getOperand(1), 2260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0, 22613e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman Ops, L, SE); 2262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 22663e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // Otherwise use the value itself, optionally with a scale applied. 22673e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman Ops.push_back(C ? SE.getMulExpr(C, S) : S); 2268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateReassociations - Split out subexpressions from adds and the bases of 2271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrecs. 2272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx, 2273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base, 2274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth) { 2275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Arbitrarily cap recursion to protect compile time. 2276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Depth >= 3) return; 2277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 2279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = Base.BaseRegs[i]; 2280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 22813e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman SmallVector<const SCEV *, 8> AddOps; 22823e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(BaseReg, 0, AddOps, L, SE); 22833e3f15bb09eaca4d259e531c9a1ecafb5710b57bDan Gohman 2284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddOps.size() == 1) continue; 2285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), 2287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = AddOps.end(); J != JE; ++J) { 22883e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 22893e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // Loop-variant "unknown" values are uninteresting; we won't be able to 22903e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // do anything meaningful with them. 229117ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L)) 22923e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman continue; 22933e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 2294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't pull a constant into a register if the constant could be folded 2295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // into an immediate field. 2296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAlwaysFoldable(*J, LU.MinOffset, LU.MaxOffset, 2297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 2298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 2299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect all operands except *J. 2302403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman SmallVector<const SCEV *, 8> InnerAddOps 23034eaee28e347f33b3e23d2fb7275fc5222d5482fdDan Gohman (((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); 2304403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman InnerAddOps.append 2305ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes (llvm::next(J), ((const SmallVector<const SCEV *, 8> &)AddOps).end()); 2306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't leave just a constant behind in a register if the constant could 2308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // be folded into an immediate field. 2309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InnerAddOps.size() == 1 && 2310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isAlwaysFoldable(InnerAddOps[0], LU.MinOffset, LU.MaxOffset, 2311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 2312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 2313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2315fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); 2316fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman if (InnerSum->isZero()) 2317fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman continue; 2318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2319fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman F.BaseRegs[i] = InnerSum; 2320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(*J); 2321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InsertFormula(LU, LUIdx, F)) 2322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If that formula hadn't been seen before, recurse to find more like 2323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // it. 2324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth+1); 2325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCombinations - Generate a formula consisting of all of the 2330572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-dominating registers added into a single register. 2331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx, 2332441a38993e89104c424e00c95cce63c7351f4fc3Dan Gohman Formula Base) { 23333f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // This method is only interesting on a plurality of registers. 2334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.BaseRegs.size() <= 1) return; 2335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.clear(); 2338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 2339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 2340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Base.BaseRegs.begin(), E = Base.BaseRegs.end(); I != E; ++I) { 2341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 2342dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(BaseReg, L->getHeader()) && 234317ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman !SE.hasComputableLoopEvolution(BaseReg, L)) 2344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(BaseReg); 2345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(BaseReg); 2347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Ops.size() > 1) { 2349ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman const SCEV *Sum = SE.getAddExpr(Ops); 2350ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // TODO: If Sum is zero, it probably means ScalarEvolution missed an 2351ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // opportunity to fold something. For now, just ignore such cases 23523f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // rather than proceed with zero in a register. 2353ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman if (!Sum->isZero()) { 2354ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman F.BaseRegs.push_back(Sum); 2355ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2356ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman } 2357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateSymbolicOffsets - Generate reuse formulae using symbolic offsets. 2361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, 2362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 2363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We can't add a symbolic offset if the address already contains one. 2364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 2365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 2367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 2368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *GV = ExtractSymbol(G, SE); 2369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || !GV) 2370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseGV = GV; 2373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 2374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 2375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 2377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets. 2382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, 2383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 2384572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: For now, just add the min and max offset, because it usually isn't 2385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // worthwhile looking at everything inbetween. 2386c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman SmallVector<int64_t, 2> Worklist; 2387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MinOffset); 2388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MaxOffset != LU.MinOffset) 2389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MaxOffset); 2390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 2392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 2393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Worklist.begin(), 2395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Worklist.end(); I != E; ++I) { 2396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)Base.AM.BaseOffs - *I; 2398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(F.AM, LU.MinOffset - *I, LU.MaxOffset - *I, 2399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 2400c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // Add the offset to the base register. 24014065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G); 2402c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // If it cancelled out, drop the base register, otherwise update it. 2403c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman if (NewG->isZero()) { 2404c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman std::swap(F.BaseRegs[i], F.BaseRegs.back()); 2405c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs.pop_back(); 2406c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman } else 2407c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs[i] = NewG; 2408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(G, SE); 2414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || Imm == 0) 2415572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2416572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Imm; 2418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 2419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 2420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 2422572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2423572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2424572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2425572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2426572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateICmpZeroScales - For ICmpZero, check to see if we can scale up 2427572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the comparison. For example, x == y -> x*c == y*c. 2428572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, 2429572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 2430572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != LSRUse::ICmpZero) return; 2431572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 2433572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *IntTy = Base.getType(); 2434572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 2435572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(IntTy) > 64) return; 2436572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2437572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't do this if there is more than one offset. 2438572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MinOffset != LU.MaxOffset) return; 2439572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2440572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Base.AM.BaseGV && "ICmpZero use is not legal!"); 2441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 2443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 2444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 2445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 2446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2447572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that the multiplication doesn't overflow. 24482ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (Base.AM.BaseOffs == INT64_MIN && Factor == -1) 2449968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 24502ea09e05466613a22e1211f52c30cd01af563983Dan Gohman int64_t NewBaseOffs = (uint64_t)Base.AM.BaseOffs * Factor; 24512ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (NewBaseOffs / Factor != Base.AM.BaseOffs) 2452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2453572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2454572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the use offset doesn't overflow. 2455572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = LU.MinOffset; 2456968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman if (Offset == INT64_MIN && Factor == -1) 2457968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 2458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = (uint64_t)Offset * Factor; 2459378c0b35a7b9884f1de2a9762825424fbf1813acDan Gohman if (Offset / Factor != LU.MinOffset) 2460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2461572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 24622ea09e05466613a22e1211f52c30cd01af563983Dan Gohman Formula F = Base; 24632ea09e05466613a22e1211f52c30cd01af563983Dan Gohman F.AM.BaseOffs = NewBaseOffs; 24642ea09e05466613a22e1211f52c30cd01af563983Dan Gohman 2465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that this scale is legal. 2466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, Offset, Offset, LU.Kind, LU.AccessTy, TLI)) 2467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compensate for the use having MinOffset built into it. 2470572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Offset - LU.MinOffset; 2471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2472deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 2473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with each base register doesn't overflow. 2475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) { 2476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS); 2477f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i]) 2478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto next; 2479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2480572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the scaled register doesn't overflow. 2482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) { 2483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS); 2484f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg) 2485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2487572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2488572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we make it here and it's legal, add it. 2489572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2490572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman next:; 2491572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2492572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2494572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateScales - Generate stride factor reuse formulae by making use of 2495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// scaled-offset address modes, for example. 2496ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) { 2497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 2498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *IntTy = Base.getType(); 2499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 2500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this Formula already has a scaled register, we can't add another one. 2502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.Scale != 0) return; 2503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 2505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 2506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 2507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 2508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.Scale = Factor; 2510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.HasBaseReg = Base.BaseRegs.size() > 1; 2511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check whether this scale is going to be legal. 2512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 2513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 2514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // As a special-case, handle special out-of-loop Basic users specially. 2515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Reconsider this special case. 2516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::Basic && 2517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 2518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::Special, LU.AccessTy, TLI) && 2519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.AllFixupsOutsideLoop) 2520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind = LSRUse::Special; 2521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For an ICmpZero, negating a solitary base register won't lead to 2525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // new solutions. 2526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero && 2527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !Base.AM.HasBaseReg && Base.AM.BaseOffs == 0 && !Base.AM.BaseGV) 2528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For each addrec base reg, apply the scale, if possible. 2530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) 2531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = 2532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i])) { 2533deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 2534572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FactorS->isZero()) 2535572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Divide out the factor, ignoring high bits, since we'll be 2537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scaling the value back up in the end. 2538f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) { 2539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This could be optimized to avoid all the copying. 2540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = Quotient; 25425ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman F.DeleteBaseReg(F.BaseRegs[i]); 2543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2547572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateTruncates - Generate reuse formulae from different IV types. 2550ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) { 2551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This requires TargetLowering to tell us which truncates are free. 2552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI) return; 2553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't bother truncating symbolic values. 2555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 2556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 2558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *DstTy = Base.getType(); 2559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DstTy) return; 2560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DstTy = SE.getEffectiveSCEVType(DstTy); 2561572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const Type *, 4>::const_iterator 2563572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 2564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *SrcTy = *I; 2565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SrcTy != DstTy && TLI->isTruncateFree(SrcTy, DstTy)) { 2566572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, *I); 2569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(), 2570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = F.BaseRegs.end(); J != JE; ++J) 2571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *J = SE.getAnyExtendExpr(*J, SrcTy); 2572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2573572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This assumes we've done basic processing on all uses and 2574572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // have an idea what the register usage is. 2575572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses)) 2576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2578572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2579572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2580572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 2584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 25856020d85c41987b0b7890d91bf66187aac6e8a3a1Dan Gohman/// WorkItem - Helper class for GenerateCrossUseConstantOffsets. It's used to 2586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// defer modifications so that the search phase doesn't have to worry about 2587572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the data structures moving underneath it. 2588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct WorkItem { 2589572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 2590572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm; 2591572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg; 2592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2593572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WorkItem(size_t LI, int64_t I, const SCEV *R) 2594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : LUIdx(LI), Imm(I), OrigReg(R) {} 2595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2596572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 2597572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 2598572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 2599572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2600572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2601572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::print(raw_ostream &OS) const { 2603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx 2604572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << " , add offset " << Imm; 2605572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2606572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2607572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::dump() const { 2608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 2609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2611572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCrossUseConstantOffsets - Look for registers which are a constant 2612572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// distance apart and try to form reuse opportunities between them. 2613572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCrossUseConstantOffsets() { 2614572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Group the registers by their value without any added constant offset. 2615572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef std::map<int64_t, const SCEV *> ImmMapTy; 2616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy; 2617572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegMapTy Map; 2618572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; 2619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Sequence; 2620572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 2621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 2622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 2623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(Reg, SE); 2624572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegMapTy::iterator, bool> Pair = 2625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.insert(std::make_pair(Reg, ImmMapTy())); 2626572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 2627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.push_back(Reg); 2628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pair.first->second.insert(std::make_pair(Imm, *I)); 2629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(*I); 2630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2631572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now examine each set of registers with the same base value. Build up 2633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // a list of work to do and do the work in a separate step so that we're 2634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // not adding formulae and register counts while we're searching. 2635191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallVector<WorkItem, 32> WorkItems; 2636191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems; 2637572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(), 2638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Sequence.end(); I != E; ++I) { 2639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 2640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const ImmMapTy &Imms = Map.find(Reg)->second; 2641572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2642cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman // It's not worthwhile looking for reuse if there's only one offset. 2643cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (Imms.size() == 1) 2644cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman continue; 2645cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman 2646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':'; 2647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 2648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 2649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << J->first; 2650572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 2651572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2652572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Examine each offset. 2653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 2654572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) { 2655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = J->second; 2656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2657572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t JImm = J->first; 2658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg); 2659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isa<SCEVConstant>(OrigReg) && 2661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg].count() == 1) { 2662572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n'); 2663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively examine offsets between this orig reg a few selected 2667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // other orig regs. 2668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator OtherImms[] = { 2669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imms.begin(), prior(Imms.end()), 2670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imms.upper_bound((Imms.begin()->first + prior(Imms.end())->first) / 2) 2671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 2672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = array_lengthof(OtherImms); i != e; ++i) { 2673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator M = OtherImms[i]; 2674cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (M == J || M == JE) continue; 2675572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compute the difference between the two. 2677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = (uint64_t)JImm - M->first; 2678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (int LUIdx = UsedByIndices.find_first(); LUIdx != -1; 2679191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LUIdx = UsedByIndices.find_next(LUIdx)) 2680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Make a memo of this use, offset, and register tuple. 2681191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (UniqueItems.insert(std::make_pair(LUIdx, Imm))) 2682191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg)); 2683572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2684572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2685572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2687572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.clear(); 2688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.clear(); 2689572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap.clear(); 2690191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman UniqueItems.clear(); 2691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now iterate through the worklist and add new formulae. 2693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<WorkItem>::const_iterator I = WorkItems.begin(), 2694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = WorkItems.end(); I != E; ++I) { 2695572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const WorkItem &WI = *I; 2696572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = WI.LUIdx; 2697572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2698572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = WI.Imm; 2699572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = WI.OrigReg; 2700572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2701572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType()); 2702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); 2703572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BitWidth = SE.getTypeSizeInBits(IntTy); 2704572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 27053f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // TODO: Use a more targeted data structure. 2706572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) { 27079f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const Formula &F = LU.Formulae[L]; 2708572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in the scaled register. 2709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg == OrigReg) { 2710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offs = (uint64_t)F.AM.BaseOffs + 2711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imm * (uint64_t)F.AM.Scale; 2712572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't create 50 + reg(-50). 2713572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.referencesReg(SE.getSCEV( 2714572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt::get(IntTy, -(uint64_t)Offs)))) 2715572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 2717572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = Offs; 2718572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 2719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 2720572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2721572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg); 2722572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2723572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new scale is a constant in a register, and adding the constant 2724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // value to the immediate would produce a value closer to zero than the 2725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // immediate itself, then the formula isn't worthwhile. 2726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg)) 2727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getValue()->getValue().isNegative() != 2728572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (NewF.AM.BaseOffs < 0) && 2729572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (C->getValue()->getValue().abs() * APInt(BitWidth, F.AM.Scale)) 2730e05678132345eb8a632362dbd320ee7d36226e67Dan Gohman .ule(abs64(NewF.AM.BaseOffs))) 2731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // OK, looks good. 2734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 2735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 2736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in a base register. 2737572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) { 2738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = F.BaseRegs[N]; 2739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseReg != OrigReg) 2740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2741572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 2742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = (uint64_t)NewF.AM.BaseOffs + Imm; 2743572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 2744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 2745572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2746572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg); 2747572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new formula has a constant in a register, and adding the 2749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // constant value to the immediate would produce a value closer to 2750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // zero than the immediate itself, then the formula isn't worthwhile. 2751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 2752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J = NewF.BaseRegs.begin(), JE = NewF.BaseRegs.end(); 2753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 2754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*J)) 2755360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman if ((C->getValue()->getValue() + NewF.AM.BaseOffs).abs().slt( 2756360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman abs64(NewF.AM.BaseOffs)) && 2757360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman (C->getValue()->getValue() + 2758360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman NewF.AM.BaseOffs).countTrailingZeros() >= 2759360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman CountTrailingZeros_64(NewF.AM.BaseOffs)) 2760572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto skip_formula; 2761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, looks good. 2763572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 2764572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 2765572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman skip_formula:; 2766572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateAllReuseFormulae - Generate formulae for each use. 2773572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2774572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::GenerateAllReuseFormulae() { 2775c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman // This is split into multiple loops so that hasRegsUsedByUsesOtherThan 2776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // queries are more precise. 2777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2779572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae[i]); 2781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2782572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCombinations(LU, LUIdx, LU.Formulae[i]); 2783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2787572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]); 2788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2789572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]); 2790572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2791572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]); 2792572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateScales(LU, LUIdx, LU.Formulae[i]); 2794c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman } 2795c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2796c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman LSRUse &LU = Uses[LUIdx]; 2797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateTruncates(LU, LUIdx, LU.Formulae[i]); 2799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCrossUseConstantOffsets(); 28023902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman 28033902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman DEBUG(dbgs() << "\n" 28043902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman "After generating reuse formulae:\n"; 28053902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman print_uses(dbgs())); 2806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2808f63d70f218807d7522e8bbe2d9e887ff3ea87b24Dan Gohman/// If there are multiple formulae with the same set of registers used 2809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// by other uses, pick the best one and delete the others. 2810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::FilterOutUndesirableDedicatedRegisters() { 2811fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman DenseSet<const SCEV *> VisitedRegs; 2812fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman SmallPtrSet<const SCEV *, 16> Regs; 2813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 2814c6519f916b5922de81c53547fd21364994195a70Dan Gohman bool ChangedFormulae = false; 2815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 2816572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect the best formula for each unique set of shared registers. This 2818572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // is reset for each use. 2819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<SmallVector<const SCEV *, 2>, size_t, UniquifierDenseMapInfo> 2820572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy; 2821572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy BestFormulae; 2822572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2825ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n'); 2826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2827b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 2828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t FIdx = 0, NumForms = LU.Formulae.size(); 2829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FIdx != NumForms; ++FIdx) { 2830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[FIdx]; 2831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2832572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> Key; 2833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(), 2834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = F.BaseRegs.end(); J != JE; ++J) { 2835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *J; 2836572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx)) 2837572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Key.push_back(Reg); 2838572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg && 2840572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx)) 2841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Key.push_back(F.ScaledReg); 2842572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Unstable sort by host order ok, because this is only used for 2843572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // uniquifying. 2844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::sort(Key.begin(), Key.end()); 2845572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<BestFormulaeTy::const_iterator, bool> P = 2847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulae.insert(std::make_pair(Key, FIdx)); 2848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!P.second) { 2849572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &Best = LU.Formulae[P.first->second]; 2850fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman 2851fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Cost CostF; 2852fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman CostF.RateFormula(F, Regs, VisitedRegs, L, LU.Offsets, SE, DT); 2853fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Regs.clear(); 2854fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Cost CostBest; 2855fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman CostBest.RateFormula(Best, Regs, VisitedRegs, L, LU.Offsets, SE, DT); 2856fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Regs.clear(); 2857fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman if (CostF < CostBest) 2858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::swap(F, Best); 28596458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs()); 2860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 28616458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman " in favor of formula "; Best.print(dbgs()); 2862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 2863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 2864c6519f916b5922de81c53547fd21364994195a70Dan Gohman ChangedFormulae = true; 2865572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 2866d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman LU.DeleteFormula(F); 2867572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --FIdx; 2868572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --NumForms; 2869b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Any = true; 2870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2871572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 287259dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman } 287359dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 287457aaa0b264d9d23d9a14235334b15d4117e32a3bDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 2875b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 2876b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 287759dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 287859dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman // Reset this to prepare for the next use. 2879572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulae.clear(); 2880572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2882c6519f916b5922de81c53547fd21364994195a70Dan Gohman DEBUG(if (ChangedFormulae) { 28839214b82c5446791b280821bbd892dca633130f80Dan Gohman dbgs() << "\n" 28849214b82c5446791b280821bbd892dca633130f80Dan Gohman "After filtering out undesirable candidates:\n"; 2885572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs()); 2886572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 2887572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2888572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2889d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman// This is a rough guess that seems to work fairly well. 2890d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmanstatic const size_t ComplexityLimit = UINT16_MAX; 2891d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 2892d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// EstimateSearchSpaceComplexity - Estimate the worst-case number of 2893d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// solutions the solver might have to consider. It almost never considers 2894d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// this many solutions because it prune the search space, but the pruning 2895d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// isn't always sufficient. 2896d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmansize_t LSRInstance::EstimateSearchSpaceComplexity() const { 28970d6715a413a23c021d1042719888966bb8d11872Dan Gohman size_t Power = 1; 2898d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 2899d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman E = Uses.end(); I != E; ++I) { 2900d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t FSize = I->Formulae.size(); 2901d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (FSize >= ComplexityLimit) { 2902d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power = ComplexityLimit; 2903d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 2904d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 2905d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power *= FSize; 2906d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (Power >= ComplexityLimit) 2907d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 2908d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 2909d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman return Power; 2910d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman} 2911d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 29124aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByDetectingSupersets - When one formula uses a superset 29134aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of the registers of another formula, it won't help reduce register 29144aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// pressure (though it may not necessarily hurt register pressure); remove 29154aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// it to simplify the system. 29164aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByDetectingSupersets() { 2917a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 2918a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 2919a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2920a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by eliminating formulae " 2921a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "which use a superset of registers used by other " 2922a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "formulae.\n"); 2923a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2924a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2925a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 2926a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool Any = false; 2927a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 2928a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula &F = LU.Formulae[i]; 2929f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // Look for a formula with a constant or GV in a register. If the use 2930f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // also has a formula with that same value in an immediate field, 2931f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // delete the one that uses a register. 2932a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 2933a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) { 2934a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) { 2935a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 2936a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseOffs += C->getValue()->getSExtValue(); 2937a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 2938a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 2939a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 2940a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 2941a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 2942a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 2943a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 2944a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 2945a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 2946a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2947a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) { 2948a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) 2949a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!F.AM.BaseGV) { 2950a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 2951a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseGV = GV; 2952a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 2953a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 2954a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 2955a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 2956a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 2957a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 2958a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 2959a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 2960a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 2961a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 2962a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2963a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2964a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2965a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2966a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2967a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (Any) 2968a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.RecomputeRegs(LUIdx, RegUses); 2969a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2970a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2971a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 2972a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 2973a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 29744aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 2975a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 29764aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByCollapsingUnrolledCode - When there are many registers 29774aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// for expressions like A, A+1, A+2, etc., allocate a single register for 29784aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// them. 29794aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() { 2980a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 2981a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 2982a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2983a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming that uses " 2984a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "separated by a constant offset will use the same " 2985a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "registers.\n"); 2986a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2987f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // This is especially useful for unrolled loops. 2988f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman 2989a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2990a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 2991402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 2992402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 2993402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 2994a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.AM.BaseOffs != 0 && F.AM.Scale == 0) { 2995191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU)) { 2996191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(*LUThatHas, F.AM.BaseOffs, 2997a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman /*HasBaseReg=*/false, 2998a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind, LU.AccessTy)) { 2999a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); 3000a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 3001a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3002a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop; 3003a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3004191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Update the relocs to reference the new use. 3005191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman for (SmallVectorImpl<LSRFixup>::iterator I = Fixups.begin(), 3006191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman E = Fixups.end(); I != E; ++I) { 3007191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRFixup &Fixup = *I; 3008191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == LUIdx) { 3009191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUThatHas - &Uses.front(); 3010191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.Offset += F.AM.BaseOffs; 3011dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman // Add the new offset to LUThatHas' offset list. 3012dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (LUThatHas->Offsets.back() != Fixup.Offset) { 3013dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->Offsets.push_back(Fixup.Offset); 3014dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset > LUThatHas->MaxOffset) 3015dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MaxOffset = Fixup.Offset; 3016dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset < LUThatHas->MinOffset) 3017dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MinOffset = Fixup.Offset; 3018dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman } 3019191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman DEBUG(dbgs() << "New fixup has offset " 3020191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman << Fixup.Offset << '\n'); 3021191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3022191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == NumUses-1) 3023191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUIdx; 3024191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3025191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 3026c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman // Delete formulae from the new use which are no longer legal. 3027c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman bool Any = false; 3028c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) { 3029c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Formula &F = LUThatHas->Formulae[i]; 3030c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (!isLegalUse(F.AM, 3031c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->MinOffset, LUThatHas->MaxOffset, 3032c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->Kind, LUThatHas->AccessTy, TLI)) { 3033c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 3034c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman dbgs() << '\n'); 3035c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->DeleteFormula(F); 3036c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --i; 3037c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --e; 3038c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Any = true; 3039c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3040c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3041c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (Any) 3042c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses); 3043c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman 3044a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Delete the old use. 3045c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman DeleteUse(LU, LUIdx); 3046a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --LUIdx; 3047a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --NumUses; 3048a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3049a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3050a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3051a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3052a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3053a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3054a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3055a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3056a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 3057a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 30584aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 3059a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 30603228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick/// NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters - Call 30614f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// FilterOutUndesirableDedicatedRegisters again, if necessary, now that 30624f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// we've done more filtering, as it may be able to find more formulae to 30634f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// eliminate. 30644f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohmanvoid LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){ 30654f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 30664f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 30674f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 30684f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "Narrowing the search space by re-filtering out " 30694f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman "undesirable dedicated registers.\n"); 30704f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 30714f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman FilterOutUndesirableDedicatedRegisters(); 30724f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 30734f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 30744f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman print_uses(dbgs())); 30754f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman } 30764f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman} 30774f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 30784aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByPickingWinnerRegs - Pick a register which seems likely 30794aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// to be profitable, and then in any use which has any reference to that 30804aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// register, delete all formulae which do not reference that register. 30814aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() { 308276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // With all other options exhausted, loop until the system is simple 308376c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // enough to handle. 3084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Taken; 3085d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman while (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3086572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we have too many of formulae on our hands to conveniently handle. 3087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use a rough heuristic to thin out the list. 30880da751baf72542ab66518e4549e39da5f34216b4Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Pick the register which is used by the most LSRUses, which is likely 3091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to be a good reuse register candidate. 3092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Best = 0; 3093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BestNum = 0; 3094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 3095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 3096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3097572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Taken.count(Reg)) 3098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Best) 3100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 3102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Count = RegUses.getUsedByIndices(Reg).count(); 3103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Count > BestNum) { 3104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestNum = Count; 3106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best 31113f46a3abeedba8d517b4182de34c821d752db058Dan Gohman << " will yield profitable reuse.\n"); 3112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Taken.insert(Best); 3113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // In any use with formulae which references this register, delete formulae 3115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which don't reference it. 3116b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3117b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LSRUse &LU = Uses[LUIdx]; 3118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!LU.Regs.count(Best)) continue; 3119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3120b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 3121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 3122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[i]; 3123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.referencesReg(Best)) { 3124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 3125d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman LU.DeleteFormula(F); 3126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --e; 3127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --i; 3128b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Any = true; 312959dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?"); 3130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3133b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 3134b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 3135b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 3136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3137572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 3140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 31434aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceUsingHeuristics - If there are an extraordinary number of 31444aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// formulae to choose from, use some rough heuristics to prune down the number 31454aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of formulae. This keeps the main solver from taking an extraordinary amount 31464aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of time in some worst-case scenarios. 31474aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceUsingHeuristics() { 31484aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByDetectingSupersets(); 31494aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByCollapsingUnrolledCode(); 31504f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 31514aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByPickingWinnerRegs(); 31524aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 31534aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman 3154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// SolveRecurse - This is the recursive solver. 3155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 3156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 3157572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 3158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 3159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 3160572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const { 3161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Some ideas: 3162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - prune more: 3163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - use more aggressive filtering 3164572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the formula so that the most profitable solutions are found first 3165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the uses too 3166572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - search faster: 31673f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // - don't compute a cost, and then compare. compare while computing a cost 3168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // and bail early. 3169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - track register sets with SmallBitVector 3170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = Uses[Workspace.size()]; 3172572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3173572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this use references any register that's already a part of the 3174572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // in-progress solution, consider it a requirement that a formula must 3175572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reference that register in order to be considered. This prunes out 3176572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // unprofitable searching. 3177572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> ReqRegs; 3178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator I = CurRegs.begin(), 3179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = CurRegs.end(); I != E; ++I) 31809214b82c5446791b280821bbd892dca633130f80Dan Gohman if (LU.Regs.count(*I)) 3181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ReqRegs.insert(*I); 3182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 31839214b82c5446791b280821bbd892dca633130f80Dan Gohman bool AnySatisfiedReqRegs = false; 3184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> NewRegs; 3185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost NewCost; 31869214b82c5446791b280821bbd892dca633130f80Dan Gohmanretry: 3187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 3188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 3189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F = *I; 3190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore formulae which do not use any of the required registers. 3192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(), 3193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = ReqRegs.end(); J != JE; ++J) { 3194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *J; 3195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((!F.ScaledReg || F.ScaledReg != Reg) && 3196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(F.BaseRegs.begin(), F.BaseRegs.end(), Reg) == 3197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.end()) 3198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto skip; 3199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 32009214b82c5446791b280821bbd892dca633130f80Dan Gohman AnySatisfiedReqRegs = true; 3201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Evaluate the cost of the current formula. If it's already worse than 3203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the current best, prune the search at that point. 3204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost = CurCost; 3205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs = CurRegs; 3206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost.RateFormula(F, NewRegs, VisitedRegs, L, LU.Offsets, SE, DT); 3207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NewCost < SolutionCost) { 3208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.push_back(&F); 3209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Workspace.size() != Uses.size()) { 3210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, NewCost, 3211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs, VisitedRegs); 3212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.getNumRegs() == 1 && Workspace.size() == 1) 3213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]); 3214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "New best at "; NewCost.print(dbgs()); 3216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ". Regs:"; 3217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator 3218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = NewRegs.begin(), E = NewRegs.end(); I != E; ++I) 3219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << **I; 3220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 3221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost = NewCost; 3223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution = Workspace; 3224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.pop_back(); 3226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman skip:; 3228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 32299214b82c5446791b280821bbd892dca633130f80Dan Gohman 32309214b82c5446791b280821bbd892dca633130f80Dan Gohman // If none of the formulae had all of the required registers, relax the 32319214b82c5446791b280821bbd892dca633130f80Dan Gohman // constraint so that we don't exclude all formulae. 32329214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!AnySatisfiedReqRegs) { 323359dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman assert(!ReqRegs.empty() && "Solver failed even without required registers"); 32349214b82c5446791b280821bbd892dca633130f80Dan Gohman ReqRegs.clear(); 32359214b82c5446791b280821bbd892dca633130f80Dan Gohman goto retry; 32369214b82c5446791b280821bbd892dca633130f80Dan Gohman } 3237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 323976c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solve - Choose one formula from each use. Return the results in the given 324076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solution vector. 3241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const { 3242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Workspace; 3243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost SolutionCost; 3244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost.Loose(); 3245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost CurCost; 3246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> CurRegs; 3247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> VisitedRegs; 3248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.reserve(Uses.size()); 3249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3250f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // SolveRecurse does all the work. 3251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, CurCost, 3252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CurRegs, VisitedRegs); 3253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we've now made all our decisions. 3255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\n" 3256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The chosen solution requires "; SolutionCost.print(dbgs()); 3257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"; 3258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Uses.size(); i != e; ++i) { 3259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 3260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses[i].print(dbgs()); 3261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 3262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman " "; 3263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution[i]->print(dbgs()); 3264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'; 3265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 3266a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman 3267a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman assert(Solution.size() == Uses.size() && "Malformed solution!"); 3268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3270e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// HoistInsertPosition - Helper for AdjustInsertPositionForExpand. Climb up 3271e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// the dominator tree far as we can go while still being dominated by the 3272e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// input positions. This helps canonicalize the insert position, which 3273e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// encourages sharing. 3274e5f76877aee6f33964de105893f0ef338661ecadDan GohmanBasicBlock::iterator 3275e5f76877aee6f33964de105893f0ef338661ecadDan GohmanLSRInstance::HoistInsertPosition(BasicBlock::iterator IP, 3276e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) 3277e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const { 3278e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (;;) { 3279e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IPLoop = LI.getLoopFor(IP->getParent()); 3280e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0; 3281e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3282e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock *IDom; 3283d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) { 32840fe46d9b480ab4851e1fc8bc589d1ed9c8b2a70eDan Gohman if (!Rung) return IP; 3285d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman Rung = Rung->getIDom(); 3286d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman if (!Rung) return IP; 3287d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman IDom = Rung->getBlock(); 3288e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3289e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Don't climb into a loop though. 3290e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IDomLoop = LI.getLoopFor(IDom); 3291e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0; 3292e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDomDepth <= IPLoopDepth && 3293e5f76877aee6f33964de105893f0ef338661ecadDan Gohman (IDomDepth != IPLoopDepth || IDomLoop == IPLoop)) 3294e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 3295e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3296e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3297e5f76877aee6f33964de105893f0ef338661ecadDan Gohman bool AllDominate = true; 3298e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *BetterPos = 0; 3299e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Tentative = IDom->getTerminator(); 3300e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(), 3301e5f76877aee6f33964de105893f0ef338661ecadDan Gohman E = Inputs.end(); I != E; ++I) { 3302e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Inst = *I; 3303e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (Inst == Tentative || !DT.dominates(Inst, Tentative)) { 3304e5f76877aee6f33964de105893f0ef338661ecadDan Gohman AllDominate = false; 3305e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 3306e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3307e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Attempt to find an insert position in the middle of the block, 3308e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // instead of at the end, so that it can be used for other expansions. 3309e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDom == Inst->getParent() && 3310e5f76877aee6f33964de105893f0ef338661ecadDan Gohman (!BetterPos || DT.dominates(BetterPos, Inst))) 33117d9663c70b3300070298d716dba6e6f6ce2d1e3eDouglas Gregor BetterPos = llvm::next(BasicBlock::iterator(Inst)); 3312e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3313e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (!AllDominate) 3314e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 3315e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (BetterPos) 3316e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = BetterPos; 3317e5f76877aee6f33964de105893f0ef338661ecadDan Gohman else 3318e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = Tentative; 3319e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3320e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3321e5f76877aee6f33964de105893f0ef338661ecadDan Gohman return IP; 3322e5f76877aee6f33964de105893f0ef338661ecadDan Gohman} 3323e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3324e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// AdjustInsertPositionForExpand - Determine an input position which will be 3325d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman/// dominated by the operands and which will dominate the result. 3326d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanBasicBlock::iterator 3327e5f76877aee6f33964de105893f0ef338661ecadDan GohmanLSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator IP, 3328e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRFixup &LF, 3329e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRUse &LU) const { 3330d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Collect some instructions which must be dominated by the 3331448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // expanding replacement. These must be dominated by any operands that 3332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // will be required in the expansion. 3333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Instruction *, 4> Inputs; 3334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace)) 3335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 3336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) 3337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = 3338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1))) 3339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 3340448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.PostIncLoops.count(L)) { 3341448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.isUseFullyOutsideLoop(L)) 3342069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(L->getLoopLatch()->getTerminator()); 3343069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman else 3344069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(IVIncInsertPos); 3345069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman } 3346701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // The expansion must also be dominated by the increment positions of any 3347701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // loops it for which it is using post-inc mode. 3348701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (PostIncLoopSet::const_iterator I = LF.PostIncLoops.begin(), 3349701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman E = LF.PostIncLoops.end(); I != E; ++I) { 3350701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman const Loop *PIL = *I; 3351701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (PIL == L) continue; 3352701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman 3353e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Be dominated by the loop exit. 3354701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman SmallVector<BasicBlock *, 4> ExitingBlocks; 3355701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman PIL->getExitingBlocks(ExitingBlocks); 3356701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (!ExitingBlocks.empty()) { 3357701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BasicBlock *BB = ExitingBlocks[0]; 3358701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i) 3359701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]); 3360701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman Inputs.push_back(BB->getTerminator()); 3361701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 3362701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 3363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Then, climb up the immediate dominator tree as far as we can go while 3365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // still being dominated by the input positions. 3366e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = HoistInsertPosition(IP, Inputs); 3367d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3368d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Don't insert instructions before PHI nodes. 3369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (isa<PHINode>(IP)) ++IP; 3370d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3371d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Ignore debug intrinsics. 3372449f31cb9dbf4762935b63946e8120dbe98808ffDan Gohman while (isa<DbgInfoIntrinsic>(IP)) ++IP; 3373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3374d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman return IP; 3375d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman} 3376d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 337776c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Expand - Emit instructions for the leading candidate expression for this 337876c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// LSRUse (this is called "expanding"). 3379d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanValue *LSRInstance::Expand(const LSRFixup &LF, 3380d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const Formula &F, 3381d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman BasicBlock::iterator IP, 3382d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SCEVExpander &Rewriter, 3383d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const { 3384d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const LSRUse &LU = Uses[LF.LUIdx]; 3385d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3386d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Determine an input position which will be dominated by the operands and 3387d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // which will dominate the result. 3388e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = AdjustInsertPositionForExpand(IP, LF, LU); 3389d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Inform the Rewriter if we have a post-increment use, so that it can 3391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // perform an advantageous expansion. 3392448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.setPostInc(LF.PostIncLoops); 3393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type that the user actually needs. 3395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *OpTy = LF.OperandValToReplace->getType(); 3396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This will be the type that we'll initially expand to. 3397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *Ty = F.getType(); 3398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Ty) 3399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // No type known; just expand directly to the ultimate type. 3400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 3401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy)) 3402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand directly to the ultimate type if it's the right size. 3403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 3404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type to do integer arithmetic in. 3405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Type *IntTy = SE.getEffectiveSCEVType(Ty); 3406572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3407572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Build up a list of operands to add together to form the full base. 3408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Ops; 3409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the BaseRegs portion. 3411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 3412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 3413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Reg->isZero() && "Zero allocated in a base register!"); 3415572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3416448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 3417448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 3418448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Reg = TransformForPostIncUse(Denormalize, Reg, 3419448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 3420448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 3421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3422572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, 0, IP))); 3423572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3424572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3425087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 3426087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman if (!Ops.empty()) { 3427087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 3428087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 3429087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 3430087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman } 3431087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the ScaledReg portion. 3433572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *ICmpScaledV = 0; 3434572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale != 0) { 3435572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledS = F.ScaledReg; 3436572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3437448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 3438448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 3439448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman ScaledS = TransformForPostIncUse(Denormalize, ScaledS, 3440448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 3441448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 3442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 3444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An interesting way of "folding" with an icmp is to use a negated 3445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scale, which we'll implement by inserting it into the other operand 3446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of the icmp. 3447572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == -1 && 3448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The only scale supported by ICmpZero uses is -1!"); 3449572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Rewriter.expandCodeFor(ScaledS, 0, IP); 3450572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3451572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise just expand the scaled register and an explicit scale, 3452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which is expected to be matched as part of the address. 3453572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, 0, IP)); 3454572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getMulExpr(ScaledS, 3455deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(ScaledS->getType(), F.AM.Scale)); 3456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(ScaledS); 3457087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3458087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 3459087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 3460087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 3461087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 3462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3465087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the GV portion. 3466087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman if (F.AM.BaseGV) { 3467087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(F.AM.BaseGV)); 3468087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3469087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 3470087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 3471087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 3472087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 3473087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman } 3474087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3475087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the immediate portion. 3476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)F.AM.BaseOffs + LF.Offset; 3477572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) { 3478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 3479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // The other interesting way of "folding" with an ICmpZero is to use a 3480572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // negated immediate. 3481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!ICmpScaledV) 3482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = ConstantInt::get(IntTy, -Offset); 3483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 3484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(ICmpScaledV)); 3485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = ConstantInt::get(IntTy, Offset); 3486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3487572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3488572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Just add the immediate values. These again are expected to be matched 3489572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // as part of the address. 3490087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset))); 3491572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3492572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3494572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Emit instructions summing all the operands. 3495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *FullS = Ops.empty() ? 3496deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(IntTy, 0) : 3497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getAddExpr(Ops); 3498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *FullV = Rewriter.expandCodeFor(FullS, Ty, IP); 3499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We're done expanding now, so reset the rewriter. 3501448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.clearPostInc(); 3502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An ICmpZero Formula represents an ICmp which we're handling as a 3504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // comparison against zero. Now that we've expanded an expression for that 3505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // form, update the ICmp's other operand. 3506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 3507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *CI = cast<ICmpInst>(LF.UserInst); 3508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(CI->getOperand(1)); 3509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!F.AM.BaseGV && "ICmp does not support folding a global value and " 3510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 3511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale == -1) { 3512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpScaledV->getType() != OpTy) { 3513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 3514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false, 3515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 3516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV, OpTy, "tmp", CI); 3517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Cast; 3518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, ICmpScaledV); 3520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == 0 && 3522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "ICmp does not support folding a global value and " 3523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 3524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy), 3525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman -(uint64_t)Offset); 3526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getType() != OpTy) 3527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 3528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 3529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C, OpTy); 3530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, C); 3532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3534572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3535572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return FullV; 3536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 35383a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// RewriteForPHI - Helper for Rewrite. PHI nodes are special because the use 35393a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// of their operands effectively happens in their predecessor blocks, so the 35403a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// expression may need to be expanded in multiple places. 35413a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohmanvoid LSRInstance::RewriteForPHI(PHINode *PN, 35423a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const LSRFixup &LF, 35433a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 35443a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 35453a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 35463a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const { 35473a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman DenseMap<BasicBlock *, Value *> Inserted; 35483a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 35493a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (PN->getIncomingValue(i) == LF.OperandValToReplace) { 35503a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman BasicBlock *BB = PN->getIncomingBlock(i); 35513a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 35523a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is a critical edge, split the edge so that we do not insert 35533a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // the code on all predecessor/successor paths. We do this unless this 35543a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // is the canonical backedge for this loop, which complicates post-inc 35553a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // users. 35563a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 && 35573ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman !isa<IndirectBrInst>(BB->getTerminator())) { 35583ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman Loop *PNLoop = LI.getLoopFor(PN->getParent()); 35593ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman if (!PNLoop || PN->getParent() != PNLoop->getHeader()) { 35603ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Split the critical edge. 35613ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman BasicBlock *NewBB = SplitCriticalEdge(BB, PN->getParent(), P); 35623ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 35633ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // If PN is outside of the loop and BB is in the loop, we want to 35643ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // move the block to be immediately before the PHI block, not 35653ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // immediately after BB. 35663ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman if (L->contains(BB) && !L->contains(PN)) 35673ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman NewBB->moveBefore(PN->getParent()); 35683ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 35693ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Splitting the edge can reduce the number of PHI entries we have. 35703ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman e = PN->getNumIncomingValues(); 35713ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman BB = NewBB; 35723ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman i = PN->getBasicBlockIndex(BB); 35733ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman } 35743a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 35753a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 35763a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair = 35773a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Inserted.insert(std::make_pair(BB, static_cast<Value *>(0))); 35783a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (!Pair.second) 35793a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, Pair.first->second); 35803a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman else { 3581454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, BB->getTerminator(), Rewriter, DeadInsts); 35823a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 35833a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 35843a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Type *OpTy = LF.OperandValToReplace->getType(); 35853a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (FullV->getType() != OpTy) 35863a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV = 35873a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, 35883a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman OpTy, false), 35893a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV, LF.OperandValToReplace->getType(), 35903a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman "tmp", BB->getTerminator()); 35913a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 35923a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, FullV); 35933a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pair.first->second = FullV; 35943a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 35953a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 35963a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman} 35973a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 3598572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Rewrite - Emit instructions for the leading candidate expression for this 3599572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse (this is called "expanding"), and update the UserInst to reference 3600572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the newly expanded value. 3601572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Rewrite(const LSRFixup &LF, 3602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 3603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 3604572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 3605572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const { 3606572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // First, find an insertion point that dominates UserInst. For PHI nodes, 3607572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // find the nearest block which dominates all the relevant uses. 3608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) { 3609454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman RewriteForPHI(PN, LF, F, Rewriter, DeadInsts, P); 3610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3611454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, LF.UserInst, Rewriter, DeadInsts); 3612572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3613572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 36143a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Type *OpTy = LF.OperandValToReplace->getType(); 3615572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FullV->getType() != OpTy) { 3616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 3617572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false), 3618572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV, OpTy, "tmp", LF.UserInst); 3619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV = Cast; 3620572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the user. ICmpZero is handled specially here (for now) because 3623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand may have updated one of the operands of the icmp already, and 3624572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // its new value may happen to be equal to LF.OperandValToReplace, in 3625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which case doing replaceUsesOfWith leads to replacing both operands 3626572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with the same value. TODO: Reorganize this. 3627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Uses[LF.LUIdx].Kind == LSRUse::ICmpZero) 3628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->setOperand(0, FullV); 3629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 3630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV); 3631572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(LF.OperandValToReplace); 3634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 363676c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// ImplementSolution - Rewrite all the fixup locations with new values, 363776c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// following the chosen solution. 3638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 3639572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 3640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) { 3641572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Keep track of instructions we may have made dead, so that 3642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // we can remove them after we are done working. 3643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<WeakVH, 16> DeadInsts; 3644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3645572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander Rewriter(SE); 3646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.disableCanonicalMode(); 3647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.setIVIncInsertPos(L, IVIncInsertPos); 3648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the new value definitions and update the users. 3650402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 3651402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Fixups.end(); I != E; ++I) { 3652402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const LSRFixup &Fixup = *I; 3653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3654402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman Rewrite(Fixup, *Solution[Fixup.LUIdx], Rewriter, DeadInsts, P); 3655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 3657572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clean up after ourselves. This must be done before deleting any 3660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // instructions. 3661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.clear(); 3662f7912df4cbdb44aeac9ac9907c192dfc1e22646dDan Gohman 3663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= DeleteTriviallyDeadInstructions(DeadInsts); 3664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3665010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 3666572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P) 3667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : IU(P->getAnalysis<IVUsers>()), 3668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE(P->getAnalysis<ScalarEvolution>()), 3669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT(P->getAnalysis<DominatorTree>()), 3670e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LI(P->getAnalysis<LoopInfo>()), 3671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TLI(tli), L(l), Changed(false), IVIncInsertPos(0) { 36725792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 3673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If LoopSimplify form is not available, stay out of trouble. 3674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!L->isLoopSimplifyForm()) return; 3675572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If there's no interesting work to be done, bail early. 3677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (IU.empty()) return; 3678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\nLSR on loop "; 3680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(dbgs(), L->getHeader(), /*PrintType=*/false); 3681572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"); 3682572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3683402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // First, perform some low-level loop optimizations. 3684572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OptimizeShadowIV(); 3685c6519f916b5922de81c53547fd21364994195a70Dan Gohman OptimizeLoopTermCond(); 3686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3687402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // Start collecting data and preparing for the solver. 3688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectInterestingTypesAndFactors(); 3689572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectFixupsAndInitialFormulae(); 3690572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectLoopInvariantFixupsAndFormulae(); 3691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n"; 3693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 3694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3695572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now use the reuse data to generate a bunch of interesting ways 3696572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to formulate the values needed for the uses. 3697572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateAllReuseFormulae(); 3698572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3699572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FilterOutUndesirableDedicatedRegisters(); 3700572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NarrowSearchSpaceUsingHeuristics(); 3701572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Solution; 3703572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solve(Solution); 3704572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3705572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Release memory that is no longer needed. 3706572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.clear(); 3707572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 3708572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.clear(); 3709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 3711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Formulae should be legal. 3712572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 3713572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 3714572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 3715572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 3716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) 3717572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(isLegalUse(J->AM, LU.MinOffset, LU.MaxOffset, 3718572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI) && 3719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Illegal formula generated!"); 3720572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 3721572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 3722010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 3723572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now that we've decided what we want, make it so. 3724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImplementSolution(Solution, P); 3725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3726169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 3727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_factors_and_types(raw_ostream &OS) const { 3728572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors.empty() && Types.empty()) return; 37291ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 3730572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR has identified the following interesting factors and types: "; 3731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 3732eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 3733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 3734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 3735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 3736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 3737572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '*' << *I; 3738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 37398b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 3740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const Type *, 4>::const_iterator 3741572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 3742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 3743572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 3744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '(' << **I << ')'; 3745572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3746572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3747572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3748c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 3749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_fixups(raw_ostream &OS) const { 3750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following fixup sites:\n"; 3751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 3752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Fixups.end(); I != E; ++I) { 3753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 37549f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman I->print(OS); 3755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3757572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3758010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman 3759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_uses(raw_ostream &OS) const { 3760572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following uses:\n"; 3761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 3762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 3763572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 3764572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 3765572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.print(OS); 3766572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 3768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) { 3769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << " "; 3770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J->print(OS); 3771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 37736bec5bb344fc0374431aed1cb63418de607a1aecDan Gohman } 3774572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3775572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print(raw_ostream &OS) const { 3777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_factors_and_types(OS); 3778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_fixups(OS); 3779572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(OS); 3780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3782572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::dump() const { 3783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 3784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 3787572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LoopStrengthReduce : public LoopPass { 3789572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TLI - Keep a pointer of a TargetLowering to consult for determining 3790572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// transformation profitability. 3791572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 3792572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 3794572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static char ID; // Pass ID, replacement for typeid 3795572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman explicit LoopStrengthReduce(const TargetLowering *tli = 0); 3796572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 3798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool runOnLoop(Loop *L, LPPassManager &LPM); 3799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void getAnalysisUsage(AnalysisUsage &AU) const; 3800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 3801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanchar LoopStrengthReduce::ID = 0; 38052ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce", 3806ce665bd2e2b581ab0858d1afe359192bac96b868Owen Anderson "Loop Strength Reduction", false, false) 38072ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(DominatorTree) 38082ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(ScalarEvolution) 38092ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(IVUsers) 3810205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopInfo) 3811205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopSimplify) 38122ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce", 38132ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson "Loop Strength Reduction", false, false) 38142ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson 3815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3816572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanPass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) { 3817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return new LoopStrengthReduce(TLI); 3818572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3820572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLoopStrengthReduce::LoopStrengthReduce(const TargetLowering *tli) 3821081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson : LoopPass(ID), TLI(tli) { 3822081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry()); 3823081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson } 3824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const { 3826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We split critical edges, so we change the CFG. However, we do update 3827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // many analyses if they are around. 38286793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreservedID(LoopSimplifyID); 3829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 38306793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequired<LoopInfo>(); 38316793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreserved<LoopInfo>(); 38326793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequiredID(LoopSimplifyID); 3833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<DominatorTree>(); 3834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<DominatorTree>(); 3835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<ScalarEvolution>(); 3836572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<ScalarEvolution>(); 38372c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // Requiring LoopSimplify a second time here prevents IVUsers from running 38382c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // twice, since LoopSimplify was invalidated by running ScalarEvolution. 38392c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich AU.addRequiredID(LoopSimplifyID); 3840572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<IVUsers>(); 3841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<IVUsers>(); 3842572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3843572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) { 3845572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 3846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Run the main LSR transformation. 3848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= LSRInstance(TLI, L, this).getChanged(); 3849eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 3850afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman // At this point, it is worth checking to see if any recurrence PHIs are also 385135738ac150afafe2359268d4b2169498c6c98c5fDan Gohman // dead, so that we can remove them as well. 38529fff2187a21f765ed87a25c48552a6942450f3e2Dan Gohman Changed |= DeleteDeadPHIs(L->getHeader()); 3853afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman 38541ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng return Changed; 3855eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 3856