LoopStrengthReduce.cpp revision c5701910604cdf65811fabd31d41e38f1d1d4eb1
12d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// The LLVM Compiler Infrastructure 4eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 9eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 10cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// This transformation analyzes and transforms the induction variables (and 11cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// computations derived from them) into forms suitable for efficient execution 12cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// on the target. 13cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// 14eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// This pass performs a strength reduction on array references inside loops that 15cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// have as one or more of their components the loop induction variable, it 16cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// rewrites expressions to take advantage of scaled-index addressing modes 17cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// available on the target, and it performs a variety of other optimizations 18cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// related to loop induction variables. 19eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 20572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// Terminology note: this code has a lot of handling for "post-increment" or 21572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// "post-inc" users. This is not talking about post-increment addressing modes; 22572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it is instead talking about code like this: 23572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 24572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i = phi [ 0, %entry ], [ %i.next, %latch ] 25572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// ... 26572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i.next = add %i, 1 27572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %c = icmp eq %i.next, %n 28572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 29572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however 30572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it's useful to think about these as the same register, with some uses using 31572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the register before the add and some using // it after. In this 32572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// example, the icmp is a post-increment user, since it uses %i.next, which is 33572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the induction variable after the increment. The other common 34572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// case of post-increment users is users outside the loop. 35572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 36572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: More sophistication in the way Formulae are generated and filtered. 37572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 38572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Handle multiple loops at a time. 39572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 40572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Should TargetLowering::AddrMode::BaseGV be changed to a ConstantExpr 41572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// instead of a GlobalValue? 42572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 43572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When truncation is free, truncate ICmp users' operands to make it a 44572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// smaller encoding (on x86 at least). 45572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 46572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When a negated register is used by an add (such as in a list of 47572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// multiple base registers, or as the increment expression in an addrec), 48572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// we may not actually need both reg and (-1 * reg) in registers; the 49572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// negation can be implemented by using a sub instead of an add. The 50572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// lack of support for taking this into consideration when making 51572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// register pressure decisions is partly worked around by the "Special" 52572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// use kind. 53572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 54eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 55eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 56be3e5212e23edc9210f774fac18d801de252e906Chris Lattner#define DEBUG_TYPE "loop-reduce" 57eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Scalar.h" 58eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Constants.h" 59eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Instructions.h" 60e5b01bea7b9b7dce7c24484d2d915b0c118d4d07Dan Gohman#include "llvm/IntrinsicInst.h" 612f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/DerivedTypes.h" 6281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman#include "llvm/Analysis/IVUsers.h" 63572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/Analysis/Dominators.h" 640f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel#include "llvm/Analysis/LoopPass.h" 65169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Analysis/ScalarEvolutionExpander.h" 669fc5cdf77c812aaa80419036de27576d45894d0dChris Lattner#include "llvm/Assembly/Writer.h" 67e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner#include "llvm/Transforms/Utils/BasicBlockUtils.h" 68eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Utils/Local.h" 69572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SmallBitVector.h" 70572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SetVector.h" 71572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/DenseSet.h" 72169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/Debug.h" 7380ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick#include "llvm/Support/CommandLine.h" 74afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman#include "llvm/Support/ValueHandle.h" 75460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar#include "llvm/Support/raw_ostream.h" 76d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng#include "llvm/Target/TargetLowering.h" 77cfb1d4235fe3291028341e6acf4139723b4749e3Jeff Cohen#include <algorithm> 78eaa13851a7fe604363577350c5cf65c257c4d41aNate Begemanusing namespace llvm; 79eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 8080ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Tricknamespace llvm { 810c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trickcl::opt<bool> EnableNested( 820c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick "enable-lsr-nested", cl::Hidden, cl::desc("Enable LSR on nested loops")); 830c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick 8480ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trickcl::opt<bool> EnableRetry( 8580ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick "enable-lsr-retry", cl::Hidden, cl::desc("Enable LSR retry")); 8680ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick} 8780ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 88572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 89572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 90572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegSortData - This class holds data which is used to order reuse candidates. 91572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegSortData { 92572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 93572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UsedByIndices - This represents the set of LSRUse indices which reference 94572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// a particular register. 95572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallBitVector UsedByIndices; 96572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 97572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData() {} 98572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 99572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::print(raw_ostream &OS) const { 106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "[NumUses=" << UsedByIndices.count() << ']'; 107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 108dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::dump() const { 110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 112dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 1137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmannamespace { 1147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegUseTracker - Map register candidates to information about how they are 116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// used. 117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegUseTracker { 118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, RegSortData> RegUsesTy; 119d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 12090bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy RegUsesMap; 121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 16> RegSequence; 122d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegister(const SCEV *Reg, size_t LUIdx); 125b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void DropRegister(const SCEV *Reg, size_t LUIdx); 126c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void SwapAndDropUse(size_t LUIdx, size_t LastLUIdx); 127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const; 129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &getUsedByIndices(const SCEV *Reg) const; 131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void clear(); 133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::iterator iterator; 135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator; 136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator begin() { return RegSequence.begin(); } 137572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator end() { return RegSequence.end(); } 138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator begin() const { return RegSequence.begin(); } 139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator end() const { return RegSequence.end(); } 140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 145572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) { 146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegUsesTy::iterator, bool> Pair = 14790bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.insert(std::make_pair(Reg, RegSortData())); 148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData &RSD = Pair.first->second; 149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.push_back(Reg); 151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1)); 152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.set(LUIdx); 153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 155b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid 156b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan GohmanRegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) { 157b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUsesTy::iterator It = RegUsesMap.find(Reg); 158b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(It != RegUsesMap.end()); 159b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegSortData &RSD = It->second; 160b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(RSD.UsedByIndices.size() > LUIdx); 161b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RSD.UsedByIndices.reset(LUIdx); 162b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 163b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 164a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanvoid 165c6897706b7c3796ac24535c9ea1449c0411f8c7aDan GohmanRegUseTracker::SwapAndDropUse(size_t LUIdx, size_t LastLUIdx) { 166c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman assert(LUIdx <= LastLUIdx); 167c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 168c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. The data structure is not optimized for this purpose; 169c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // we must iterate through it and update each of the bit vectors. 170a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (RegUsesTy::iterator I = RegUsesMap.begin(), E = RegUsesMap.end(); 171c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman I != E; ++I) { 172c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman SmallBitVector &UsedByIndices = I->second.UsedByIndices; 173c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman if (LUIdx < UsedByIndices.size()) 174c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices[LUIdx] = 175c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : 0; 176c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx)); 177c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman } 178a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 179a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 181572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { 18246fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 18346fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman if (I == RegUsesMap.end()) 18446fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman return false; 18546fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman const SmallBitVector &UsedByIndices = I->second.UsedByIndices; 186572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int i = UsedByIndices.find_first(); 187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (i == -1) return false; 188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((size_t)i != LUIdx) return true; 189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return UsedByIndices.find_next(i) != -1; 190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanconst SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { 19390bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 19490bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman assert(I != RegUsesMap.end() && "Unknown register!"); 195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return I->second.UsedByIndices; 196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegUseTracker::clear() { 19990bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.clear(); 200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.clear(); 201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Formula - This class holds information that describes a formula for 206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// computing satisfying a use. It may include broken-out immediates and scaled 207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// registers. 208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct Formula { 209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AM - This is used to represent complex addressing, as well as other kinds 210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// of interesting uses. 211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// BaseRegs - The list of "base" registers for this use. When this is 214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// non-empty, AM.HasBaseReg should be set to true. 215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> BaseRegs; 216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// ScaledReg - The 'scaled' register for this use. This should be non-null 218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// when AM.Scale is not zero. 219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledReg; 220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 221cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// UnfoldedOffset - An additional constant offset which added near the 222cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// use. This requires a temporary register, but the offset itself can 223cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// live in an add immediate field rather than a register. 224cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman int64_t UnfoldedOffset; 225cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 226cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Formula() : ScaledReg(0), UnfoldedOffset(0) {} 227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 228dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE); 229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned getNumRegs() const; 231db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *getType() const; 232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2335ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman void DeleteBaseReg(const SCEV *&S); 2345ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool referencesReg(const SCEV *S) const; 236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool hasRegsUsedByUsesOtherThan(size_t LUIdx, 237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const; 238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2453f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// DoInitialMatch - Recursion helper for InitialMatch. 246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic void DoInitialMatch(const SCEV *S, Loop *L, 247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Good, 248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Bad, 249dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman ScalarEvolution &SE) { 250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect expressions which properly dominate the loop header. 251dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(S, L->getHeader())) { 252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(S); 253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 255d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at add operands. 257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) 260dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(*I, L, Good, Bad, SE); 261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 263169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at addrec operands. 265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) 266572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!AR->getStart()->isZero()) { 267dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(AR->getStart(), L, Good, Bad, SE); 268deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), 269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE), 2703228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags() 2713228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick AR->getLoop(), SCEV::FlagAnyWrap), 272dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman L, Good, Bad, SE); 273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 2752d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle a multiplication by -1 (negation) if it didn't fold. 277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) 278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mul->getOperand(0)->isAllOnesValue()) { 279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end()); 280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewMul = SE.getMulExpr(Ops); 281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyGood; 283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyBad; 284dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(NewMul, L, MyGood, MyBad, SE); 285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( 286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getEffectiveSCEVType(NewMul->getType()))); 287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(), 288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyGood.end(); I != E; ++I) 289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(SE.getMulExpr(NegOne, *I)); 290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(), 291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyBad.end(); I != E; ++I) 292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(SE.getMulExpr(NegOne, *I)); 293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 295eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we can't do anything interesting. Just stuff the whole thing into a 297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // register and hope for the best. 298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(S); 299a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 300844731a7f1909f55935e3514c9e713a62d67662eDan Gohman 301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InitialMatch - Incorporate loop-variant parts of S into this Formula, 302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// attempting to keep all loop-invariant and loop-computable values in a 303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// single base register. 304dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohmanvoid Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { 305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Good; 306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Bad; 307dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(S, L, Good, Bad, SE); 308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Good.empty()) { 309e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Good); 310e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 311e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Bad.empty()) { 315e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Bad); 316e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 317e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 321eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getNumRegs - Return the total number of register operands used by this 323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// formula. This does not include register uses implied by non-constant 324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrec strides. 325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanunsigned Formula::getNumRegs() const { 326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !!ScaledReg + BaseRegs.size(); 327a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 32856a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getType - Return the type of this formula, if it has one, or null 330572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// otherwise. This type is meaningless except for the bit size. 331db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris LattnerType *Formula::getType() const { 332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !BaseRegs.empty() ? BaseRegs.front()->getType() : 333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledReg ? ScaledReg->getType() : 334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV ? AM.BaseGV->getType() : 335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 0; 336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 33756a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 3385ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteBaseReg - Delete the given base reg from the BaseRegs list. 3395ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohmanvoid Formula::DeleteBaseReg(const SCEV *&S) { 3405ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&S != &BaseRegs.back()) 3415ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(S, BaseRegs.back()); 3425ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman BaseRegs.pop_back(); 3435ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 3445ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// referencesReg - Test if this formula references the given register. 346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::referencesReg(const SCEV *S) const { 347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return S == ScaledReg || 348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(BaseRegs.begin(), BaseRegs.end(), S) != BaseRegs.end(); 349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 350eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// hasRegsUsedByUsesOtherThan - Test whether this formula uses registers 352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// which are used by uses other than the use with the given index. 353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx, 354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const { 355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx)) 357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) 360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(*I, LUIdx)) 361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::print(raw_ostream &OS) const { 366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) { 368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, AM.BaseGV, /*PrintType=*/false); 370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.BaseOffs; 374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) { 377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "reg(" << **I << ')'; 379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 380c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (AM.HasBaseReg && BaseRegs.empty()) { 381c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 382c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: HasBaseReg**"; 383c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } else if (!AM.HasBaseReg && !BaseRegs.empty()) { 384c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 385c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: !HasBaseReg**"; 386c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } 387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0) { 388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.Scale << "*reg("; 390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *ScaledReg; 392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "<unknown>"; 394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ')'; 395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 396cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (UnfoldedOffset != 0) { 397cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (!First) OS << " + "; else First = false; 398cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman OS << "imm(" << UnfoldedOffset << ')'; 399cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::dump() const { 403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 406aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddRecSExtable - Return true if the given addrec can be sign-extended 407aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 408aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { 409db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 410ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1); 411aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy)); 412aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 413aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 414aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddSExtable - Return true if the given add can be sign-extended 415aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 416aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) { 417db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 418ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1); 419aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy)); 420aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 421aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 422473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman/// isMulSExtable - Return true if the given mul can be sign-extended 423aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 424473e63512acb3751da7dffbb10e0452cb581b265Dan Gohmanstatic bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) { 425db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 426473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman IntegerType::get(SE.getContext(), 427473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman SE.getTypeSizeInBits(M->getType()) * M->getNumOperands()); 428473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy)); 429aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 430aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 431f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// getExactSDiv - Return an expression for LHS /s RHS, if it can be determined 432f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// and if the remainder is known to be zero, or null otherwise. If 433f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified 434f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// to Y, ignoring that the multiplication may overflow, which is useful when 435f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// the result will be used in a context where the most significant bits are 436f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// ignored. 437f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohmanstatic const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, 438f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman ScalarEvolution &SE, 439f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman bool IgnoreSignificantBits = false) { 440572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle the trivial case, which works for any SCEV type. 441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LHS == RHS) 442deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman return SE.getConstant(LHS->getType(), 1); 443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 444d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle a few RHS special cases. 445d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS); 446d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RC) { 447d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 448d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do 449d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // some folding. 450d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA.isAllOnesValue()) 451d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getMulExpr(LHS, RC); 452d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s 1 as x. 453d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA == 1) 454d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return LHS; 455d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman } 456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 457572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a division of a constant by a constant. 458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) { 459572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!RC) 460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 461d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &LA = C->getValue()->getValue(); 462d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 463d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (LA.srem(RA) != 0) 464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 465d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getConstant(LA.sdiv(RA)); 466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 468aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over addrec operands, if the addrec doesn't overflow. 469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) { 470aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddRecSExtable(AR, SE)) { 471f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, 472f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 473aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Step) return 0; 474694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, 475694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman IgnoreSignificantBits); 476694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman if (!Start) return 0; 4773228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FlagNW is independent of the start value, step direction, and is 4783228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // preserved with smaller magnitude steps. 4793228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 4803228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); 481aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 4822ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 485aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over add operands, if the add doesn't overflow. 486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) { 487aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddSExtable(Add, SE)) { 488aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman SmallVector<const SCEV *, 8> Ops; 489aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 490aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman I != E; ++I) { 491f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Op = getExactSDiv(*I, RHS, SE, 492f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 493aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Op) return 0; 494aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman Ops.push_back(Op); 495aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 496aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return SE.getAddExpr(Ops); 497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4982ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a multiply operand that we can pull RHS out of. 5022ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) { 503aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) { 504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Found = false; 506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVMulExpr::op_iterator I = Mul->op_begin(), E = Mul->op_end(); 507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 5084766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman const SCEV *S = *I; 509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Found) 5104766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman if (const SCEV *Q = getExactSDiv(S, RHS, SE, 511f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits)) { 5124766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman S = Q; 513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Found = true; 514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5154766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman Ops.push_back(S); 516a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Found ? SE.getMulExpr(Ops) : 0; 518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5192ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 5202ea09e05466613a22e1211f52c30cd01af563983Dan Gohman } 521a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise we don't know. 523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractImmediate - If S involves the addition of a constant integer value, 527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that integer value, and mutate S to point to a new SCEV with that 528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { 530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { 531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getValue()->getValue().getMinSignedBits() <= 64) { 532deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(C->getType(), 0); 533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return C->getValue()->getSExtValue(); 534572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 535572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 538e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 539e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 544e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 5453228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5463228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5473228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 54921e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman } 550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractSymbol - If S involves the addition of a GlobalValue address, 554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that symbol, and mutate S to point to a new SCEV with that 555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { 557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) { 559deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(GV->getType(), 0); 560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return GV; 561572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 563572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.back(), SE); 565e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 566e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.front(), SE); 571e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 5723228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5733228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5743228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 575572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 578169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 579169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 5807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// isAddressUse - Returns true if the specified instruction is using the 5817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// specified value as an address. 5827979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmanstatic bool isAddressUse(Instruction *Inst, Value *OperandVal) { 5837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman bool isAddress = isa<LoadInst>(Inst); 5847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 5857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (SI->getOperand(1) == OperandVal) 5867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 5877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 5887979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 5897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 5907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 5917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 5927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::prefetch: 5937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 5947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 5957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 5967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 597ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif if (II->getArgOperand(0) == OperandVal) 5987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 5997979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 6007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return isAddress; 603169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 604169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 6057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// getAccessType - Return the type of the memory being accessed. 606db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattnerstatic Type *getAccessType(const Instruction *Inst) { 607db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = Inst->getType(); 6087979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) 6097979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AccessTy = SI->getOperand(0)->getType(); 6107979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 6117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 6127979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 6137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 6147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 6157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 6167979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 6177979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 6187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 619ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif AccessTy = II->getArgOperand(0)->getType(); 6207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 6217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6227979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 624572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // All pointers have the same requirements, so canonicalize them to an 625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // arbitrary pointer type to minimize variation. 626db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner if (PointerType *PTy = dyn_cast<PointerType>(AccessTy)) 627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = PointerType::get(IntegerType::get(PTy->getContext(), 1), 628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PTy->getAddressSpace()); 629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 6307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return AccessTy; 631a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 63281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DeleteTriviallyDeadInstructions - If any of the instructions is the 634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// specified set are trivially dead, delete them and see if this makes any of 635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// their operands subsequently dead. 636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool 637572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanDeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) { 638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 6392f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!DeadInsts.empty()) { 641f097b59e0e2231d431a6c660c65fa5cae22d9a44Gabor Greif Instruction *I = dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()); 64281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (I == 0 || !isInstructionTriviallyDead(I)) 644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 645221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) 647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *U = dyn_cast<Instruction>(*OI)) { 648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *OI = 0; 649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (U->use_empty()) 650572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(U); 651572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 652221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I->eraseFromParent(); 654572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 6562f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 657572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Changed; 6587979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 6592f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 6612f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 662572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Cost - This class is used to measure and compare candidate formulae. 663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass Cost { 664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TODO: Some of these could be merged. Also, a lexical ordering 665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// isn't always optimal. 666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumRegs; 667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned AddRecCost; 668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumIVMuls; 669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumBaseAdds; 670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned ImmCost; 671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned SetupCost; 672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost() 675572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : NumRegs(0), AddRecCost(0), NumIVMuls(0), NumBaseAdds(0), ImmCost(0), 676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost(0) {} 677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool operator<(const Cost &Other) const; 679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Loose(); 681572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 6827d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick#ifndef NDEBUG 6837d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick // Once any of the metrics loses, they must all remain losers. 6847d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick bool isValid() { 6857d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return ((NumRegs | AddRecCost | NumIVMuls | NumBaseAdds 6867d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick | ImmCost | SetupCost) != ~0u) 6877d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick || ((NumRegs & AddRecCost & NumIVMuls & NumBaseAdds 6887d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick & ImmCost & SetupCost) == ~0u); 6897d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 6907d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick#endif 6917d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick 6927d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick bool isLoser() { 6937d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick assert(isValid() && "invalid cost"); 6947d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return NumRegs == ~0u; 6957d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 6967d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick 697572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateFormula(const Formula &F, 698572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 699572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 700572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 701572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 703572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 704572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 705572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 706572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 707572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 708572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateRegister(const SCEV *Reg, 709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 7129214b82c5446791b280821bbd892dca633130f80Dan Gohman void RatePrimaryRegister(const SCEV *Reg, 7139214b82c5446791b280821bbd892dca633130f80Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 7149214b82c5446791b280821bbd892dca633130f80Dan Gohman const Loop *L, 7159214b82c5446791b280821bbd892dca633130f80Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 7170e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 718572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 7197979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 720572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RateRegister - Tally up interesting quantities from the given register. 721572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateRegister(const SCEV *Reg, 722572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 723572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 7259214b82c5446791b280821bbd892dca633130f80Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) { 7269214b82c5446791b280821bbd892dca633130f80Dan Gohman if (AR->getLoop() == L) 7279214b82c5446791b280821bbd892dca633130f80Dan Gohman AddRecCost += 1; /// TODO: This should be a function of the stride. 7289214b82c5446791b280821bbd892dca633130f80Dan Gohman 7290c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // If this is an addrec for another loop, don't second-guess its addrec phi 7300c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // nodes. LSR isn't currently smart enough to reason about more than one 7310c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // loop at a time. LSR has either already run on inner loops, will not run 7320c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // on other loops, and cannot be expected to change sibling loops. If the 7330c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // AddRec exists, consider it's register free and leave it alone. Otherwise, 7340c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // do not consider this formula at all. 7350c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // FIXME: why do we need to generate such fomulae? 7360c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick else if (!EnableNested || L->contains(AR->getLoop()) || 7379214b82c5446791b280821bbd892dca633130f80Dan Gohman (!AR->getLoop()->contains(L) && 7389214b82c5446791b280821bbd892dca633130f80Dan Gohman DT.dominates(L->getHeader(), AR->getLoop()->getHeader()))) { 7399214b82c5446791b280821bbd892dca633130f80Dan Gohman for (BasicBlock::iterator I = AR->getLoop()->getHeader()->begin(); 7407d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick PHINode *PN = dyn_cast<PHINode>(I); ++I) { 7419214b82c5446791b280821bbd892dca633130f80Dan Gohman if (SE.isSCEVable(PN->getType()) && 7429214b82c5446791b280821bbd892dca633130f80Dan Gohman (SE.getEffectiveSCEVType(PN->getType()) == 7439214b82c5446791b280821bbd892dca633130f80Dan Gohman SE.getEffectiveSCEVType(AR->getType())) && 7449214b82c5446791b280821bbd892dca633130f80Dan Gohman SE.getSCEV(PN) == AR) 7459214b82c5446791b280821bbd892dca633130f80Dan Gohman return; 7467d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 7470c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick if (!EnableNested) { 7480c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick Loose(); 7490c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick return; 7500c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick } 7519214b82c5446791b280821bbd892dca633130f80Dan Gohman // If this isn't one of the addrecs that the loop already has, it 7529214b82c5446791b280821bbd892dca633130f80Dan Gohman // would require a costly new phi and add. TODO: This isn't 7539214b82c5446791b280821bbd892dca633130f80Dan Gohman // precisely modeled right now. 7549214b82c5446791b280821bbd892dca633130f80Dan Gohman ++NumBaseAdds; 7557d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (!Regs.count(AR->getStart())) { 756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RateRegister(AR->getStart(), Regs, L, SE, DT); 7577d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 7587d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 7597d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 7609214b82c5446791b280821bbd892dca633130f80Dan Gohman } 7617979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 7629214b82c5446791b280821bbd892dca633130f80Dan Gohman // Add the step value register, if it needs one. 7639214b82c5446791b280821bbd892dca633130f80Dan Gohman // TODO: The non-affine case isn't precisely modeled here. 76425b689e067697d3b49ae123120703fada030350fAndrew Trick if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) { 76525b689e067697d3b49ae123120703fada030350fAndrew Trick if (!Regs.count(AR->getOperand(1))) { 766572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RateRegister(AR->getOperand(1), Regs, L, SE, DT); 76725b689e067697d3b49ae123120703fada030350fAndrew Trick if (isLoser()) 76825b689e067697d3b49ae123120703fada030350fAndrew Trick return; 76925b689e067697d3b49ae123120703fada030350fAndrew Trick } 77025b689e067697d3b49ae123120703fada030350fAndrew Trick } 771a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 7729214b82c5446791b280821bbd892dca633130f80Dan Gohman ++NumRegs; 7739214b82c5446791b280821bbd892dca633130f80Dan Gohman 7749214b82c5446791b280821bbd892dca633130f80Dan Gohman // Rough heuristic; favor registers which don't require extra setup 7759214b82c5446791b280821bbd892dca633130f80Dan Gohman // instructions in the preheader. 7769214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!isa<SCEVUnknown>(Reg) && 7779214b82c5446791b280821bbd892dca633130f80Dan Gohman !isa<SCEVConstant>(Reg) && 7789214b82c5446791b280821bbd892dca633130f80Dan Gohman !(isa<SCEVAddRecExpr>(Reg) && 7799214b82c5446791b280821bbd892dca633130f80Dan Gohman (isa<SCEVUnknown>(cast<SCEVAddRecExpr>(Reg)->getStart()) || 7809214b82c5446791b280821bbd892dca633130f80Dan Gohman isa<SCEVConstant>(cast<SCEVAddRecExpr>(Reg)->getStart())))) 7819214b82c5446791b280821bbd892dca633130f80Dan Gohman ++SetupCost; 78223c3fde39aa35334e74c26273a8973a872871e34Dan Gohman 78323c3fde39aa35334e74c26273a8973a872871e34Dan Gohman NumIVMuls += isa<SCEVMulExpr>(Reg) && 78417ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman SE.hasComputableLoopEvolution(Reg, L); 7859214b82c5446791b280821bbd892dca633130f80Dan Gohman} 7869214b82c5446791b280821bbd892dca633130f80Dan Gohman 7879214b82c5446791b280821bbd892dca633130f80Dan Gohman/// RatePrimaryRegister - Record this register in the set. If we haven't seen it 7889214b82c5446791b280821bbd892dca633130f80Dan Gohman/// before, rate it. 7899214b82c5446791b280821bbd892dca633130f80Dan Gohmanvoid Cost::RatePrimaryRegister(const SCEV *Reg, 7907fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 7917fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman const Loop *L, 7927fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 7939214b82c5446791b280821bbd892dca633130f80Dan Gohman if (Regs.insert(Reg)) 7949214b82c5446791b280821bbd892dca633130f80Dan Gohman RateRegister(Reg, Regs, L, SE, DT); 7957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 7967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateFormula(const Formula &F, 798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the registers. 804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEV *ScaledReg = F.ScaledReg) { 805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(ScaledReg)) { 806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 8087979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 8099214b82c5446791b280821bbd892dca633130f80Dan Gohman RatePrimaryRegister(ScaledReg, Regs, L, SE, DT); 8107d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 8117d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 8123821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner } 813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 814572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 816572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(BaseReg)) { 817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 818572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 8197979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 8209214b82c5446791b280821bbd892dca633130f80Dan Gohman RatePrimaryRegister(BaseReg, Regs, L, SE, DT); 8217d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 8227d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 825cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Determine how many (unfolded) adds we'll need inside the loop. 826cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman size_t NumBaseParts = F.BaseRegs.size() + (F.UnfoldedOffset != 0); 827cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (NumBaseParts > 1) 828cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NumBaseAdds += NumBaseParts - 1; 829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the non-zero immediates. 831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 832572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)*I + F.AM.BaseOffs; 834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.BaseGV) 835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += 64; // Handle symbolic values conservatively. 836572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This should probably be the pointer size. 837572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Offset != 0) 838572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += APInt(64, Offset, true).getMinSignedBits(); 839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 8407d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick assert(isValid() && "invalid cost"); 841169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 842169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 8437a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner/// Loose - Set this cost to a losing value. 844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::Loose() { 845572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumRegs = ~0u; 846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AddRecCost = ~0u; 847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumIVMuls = ~0u; 848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumBaseAdds = ~0u; 849572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost = ~0u; 850572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost = ~0u; 851572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 85256a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 853572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// operator< - Choose the lower cost. 854572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Cost::operator<(const Cost &Other) const { 855572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumRegs != Other.NumRegs) 856572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumRegs < Other.NumRegs; 857572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != Other.AddRecCost) 858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return AddRecCost < Other.AddRecCost; 859572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != Other.NumIVMuls) 860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumIVMuls < Other.NumIVMuls; 861572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != Other.NumBaseAdds) 862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumBaseAdds < Other.NumBaseAdds; 863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != Other.ImmCost) 864572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return ImmCost < Other.ImmCost; 865572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != Other.SetupCost) 866572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return SetupCost < Other.SetupCost; 867572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 868572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 8697979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::print(raw_ostream &OS) const { 871572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << NumRegs << " reg" << (NumRegs == 1 ? "" : "s"); 872572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != 0) 873572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", with addrec cost " << AddRecCost; 874572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != 0) 875572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumIVMuls << " IV mul" << (NumIVMuls == 1 ? "" : "s"); 876572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != 0) 877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumBaseAdds << " base add" 878572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << (NumBaseAdds == 1 ? "" : "s"); 879572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != 0) 880572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << ImmCost << " imm cost"; 881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != 0) 882572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << SetupCost << " setup cost"; 883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 88444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 885572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::dump() const { 886572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 88744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner} 88844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 889572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 89044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 891572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRFixup - An operand value in an instruction which is to be replaced 892572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// with some equivalent, possibly strength-reduced, replacement. 893572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct LSRFixup { 894572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UserInst - The instruction which will be updated. 895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *UserInst; 89656a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 897572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// OperandValToReplace - The operand of the instruction which will 898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// be replaced. The operand may be used more than once; every instance 899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// will be replaced. 900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OperandValToReplace; 90156a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 902448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman /// PostIncLoops - If this user is to use the post-incremented value of an 903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable, this variable is non-null and holds the loop 904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// associated with the induction variable. 905448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet PostIncLoops; 90626d91f16464db56283087176a73981048331dd2dChris Lattner 907572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LUIdx - The index of the LSRUse describing the expression which 908572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// this fixup needs, minus an offset (below). 909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 91026d91f16464db56283087176a73981048331dd2dChris Lattner 911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Offset - A constant offset to be added to the LSRUse expression. 912572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// This allows multiple fixups to share the same LSRUse with different 913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// offsets, for example in an unrolled loop. 914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset; 915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 916448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman bool isUseFullyOutsideLoop(const Loop *L) const; 917448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 918572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup(); 919572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 922572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 923572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 924572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 925572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 926572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRFixup::LSRFixup() 927ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman : UserInst(0), OperandValToReplace(0), LUIdx(~size_t(0)), Offset(0) {} 928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 929448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// isUseFullyOutsideLoop - Test whether this fixup always uses its 930448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// value outside of the given loop. 931448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohmanbool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const { 932448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // PHI nodes use their value in their incoming blocks. 933448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) { 934448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 935448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (PN->getIncomingValue(i) == OperandValToReplace && 936448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman L->contains(PN->getIncomingBlock(i))) 937448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return false; 938448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return true; 939448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 940448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 941448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return !L->contains(UserInst); 942448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman} 943448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::print(raw_ostream &OS) const { 945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "UserInst="; 946572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Store is common and interesting enough to be worth special-casing. 947572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) { 948572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "store "; 949572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, Store->getOperand(0), /*PrintType=*/false); 950572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (UserInst->getType()->isVoidTy()) 951572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << UserInst->getOpcodeName(); 952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, UserInst, /*PrintType=*/false); 954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 955572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", OperandValToReplace="; 956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, OperandValToReplace, /*PrintType=*/false); 957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 958448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (PostIncLoopSet::const_iterator I = PostIncLoops.begin(), 959448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman E = PostIncLoops.end(); I != E; ++I) { 960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", PostIncLoop="; 961448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman WriteAsOperand(OS, (*I)->getHeader(), /*PrintType=*/false); 962934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 963a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LUIdx != ~size_t(0)) 965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", LUIdx=" << LUIdx; 966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) 968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offset=" << Offset; 9697979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 9707979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::dump() const { 972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 9737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 9747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 976a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// UniquifierDenseMapInfo - A DenseMapInfo implementation for holding 978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DenseMaps and DenseSets of sorted SmallVectors of const SCEV*. 979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct UniquifierDenseMapInfo { 980572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getEmptyKey() { 981572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-1)); 983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 985934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 986572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getTombstoneKey() { 987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 988572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-2)); 989572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 990572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static unsigned getHashValue(const SmallVector<const SCEV *, 2> &V) { 993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Result = 0; 994572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = V.begin(), 995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = V.end(); I != E; ++I) 996572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Result ^= DenseMapInfo<const SCEV *>::getHashValue(*I); 9977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Result; 998a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 999a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static bool isEqual(const SmallVector<const SCEV *, 2> &LHS, 1001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVector<const SCEV *, 2> &RHS) { 1002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return LHS == RHS; 1003572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1006572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse - This class holds the state that LSR keeps for each use in 1007572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// IVUsers, as well as uses invented by LSR itself. It includes information 1008572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// about what kinds of things can be folded into the user, information about 1009572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the user itself, and information about how the use may be satisfied. 1010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Represent multiple users of the same expression in common? 1011572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRUse { 1012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<SmallVector<const SCEV *, 2>, UniquifierDenseMapInfo> Uniquifier; 1013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1014572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 1015572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// KindType - An enum for a kind of use, indicating what types of 1016572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// scaled and immediate operands it might support. 1017572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman enum KindType { 1018572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Basic, ///< A normal use, with no folding. 1019572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Special, ///< A special case of basic, allowing -1 scales. 1020572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Address, ///< An address use; folding according to TargetLowering 1021572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpZero ///< An equality icmp with both operands folded into one. 1022572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Add a generic icmp too? 1023572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 10247979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1025572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman KindType Kind; 1026db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy; 10277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<int64_t, 8> Offsets; 1029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset; 1030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MaxOffset; 1031203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 1032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AllFixupsOutsideLoop - This records whether all of the fixups using this 1033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LSRUse are outside of the loop, in which case some special-case heuristics 1034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// may be used. 1035572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool AllFixupsOutsideLoop; 10367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1037a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// WidestFixupType - This records the widest use type for any fixup using 1038a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// this LSRUse. FindUseWithSimilarFormula can't consider uses with different 1039a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// max fixup widths to be equivalent, because the narrower one may be relying 1040a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// on the implicit truncation to truncate away bogus bits. 1041db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WidestFixupType; 1042a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 1043572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Formulae - A list of ways to build a value that can satisfy this user. 1044572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// After the list is populated, one of these is selected heuristically and 1045572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// used to formulate a replacement for OperandValToReplace in UserInst. 1046572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Formula, 12> Formulae; 1047589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 1048572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Regs - The set of register candidates used by all formulae in this LSRUse. 1049572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Regs; 1050934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1051db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse(KindType K, Type *T) : Kind(K), AccessTy(T), 1052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MinOffset(INT64_MAX), 1053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MaxOffset(INT64_MIN), 1054a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman AllFixupsOutsideLoop(true), 1055a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman WidestFixupType(0) {} 105656a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1057a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool HasFormulaWithSameRegs(const Formula &F) const; 1058454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman bool InsertFormula(const Formula &F); 1059d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman void DeleteFormula(Formula &F); 1060b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses); 1061572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1064572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1065572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1066b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman} 1067b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 1068a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// HasFormula - Test whether this use as a formula which has the same 1069a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// registers as the given formula. 1070a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanbool LSRUse::HasFormulaWithSameRegs(const Formula &F) const { 1071a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1072a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1073a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1074a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman std::sort(Key.begin(), Key.end()); 1075a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return Uniquifier.count(Key); 1076a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 1077a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 1079572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 1080454d26dc43207ec537d843229db6f5e6a302e23dDan Gohmanbool LSRUse::InsertFormula(const Formula &F) { 1081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1083572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::sort(Key.begin(), Key.end()); 1085572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1086572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Uniquifier.insert(Key).second) 1087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1088572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Using a register to hold the value of 0 is not profitable. 1090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert((!F.ScaledReg || !F.ScaledReg->isZero()) && 1091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Zero allocated in a scaled register!"); 1092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 1093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = 1094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) 1095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!(*I)->isZero() && "Zero allocated in a base register!"); 1096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 1097572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Add the formula to the list. 1099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formulae.push_back(F); 110056a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record registers now being used by this use. 1102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) Regs.insert(F.ScaledReg); 1103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1108d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman/// DeleteFormula - Remove the given formula from this use's list. 1109d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohmanvoid LSRUse::DeleteFormula(Formula &F) { 11105ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&F != &Formulae.back()) 11115ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(F, Formulae.back()); 1112d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman Formulae.pop_back(); 1113a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman assert(!Formulae.empty() && "LSRUse has no formulae left!"); 1114d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman} 1115d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman 1116b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman/// RecomputeRegs - Recompute the Regs field, and update RegUses. 1117b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) { 1118b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 1119b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman SmallPtrSet<const SCEV *, 4> OldRegs = Regs; 1120b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.clear(); 1121402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = Formulae.begin(), 1122402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Formulae.end(); I != E; ++I) { 1123402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 1124b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (F.ScaledReg) Regs.insert(F.ScaledReg); 1125b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1126b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman } 1127b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1128b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Update the RegTracker. 1129b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (SmallPtrSet<const SCEV *, 4>::iterator I = OldRegs.begin(), 1130b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman E = OldRegs.end(); I != E; ++I) 1131b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (!Regs.count(*I)) 1132b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUses.DropRegister(*I, LUIdx); 1133b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 1134b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::print(raw_ostream &OS) const { 1136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR Use: Kind="; 1137572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Basic: OS << "Basic"; break; 1139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Special: OS << "Special"; break; 1140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case ICmpZero: OS << "ICmpZero"; break; 1141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Address: 1142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "Address of "; 11431df9859c40492511b8aa4321eb76496005d3b75bDuncan Sands if (AccessTy->isPointerTy()) 1144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "pointer"; // the full pointer type could be really verbose 11457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else 1146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *AccessTy; 11477979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 11481bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offsets={"; 1150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 1151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 1152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *I; 1153ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes if (llvm::next(I) != E) 1154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ','; 1155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '}'; 1157572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AllFixupsOutsideLoop) 1159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", all-fixups-outside-loop"; 1160a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 1161a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (WidestFixupType) 1162a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman OS << ", widest fixup type: " << *WidestFixupType; 11637979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 1164d6b62a572210aff965a55626cf36a68821838844Evan Cheng 1165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::dump() const { 1166572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 1167572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 116856a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// isLegalUse - Test whether the use described by AM is "legal", meaning it can 1170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// be completely folded into the user instruction at isel time. This includes 1171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// address-mode folding and special icmp tricks. 1172572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(const TargetLowering::AddrMode &AM, 1173db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1174572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1175572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1176572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Address: 1177572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can 1178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // completely fold this address. 1179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) return TLI->isLegalAddressingMode(AM, AccessTy); 1180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise, just guess that reg+reg addressing is legal. 1182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.BaseOffs == 0 && AM.Scale <= 1; 1183572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::ICmpZero: 1185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // There's not even a target hook for querying whether it would be legal to 1186572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // fold a GV into an ICmp. 1187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) 1188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1189579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner 1190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only has two operands; don't allow more than two non-trivial parts. 1191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.HasBaseReg && AM.BaseOffs != 0) 1192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 11931bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by 1195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // putting the scaled register in the other operand of the icmp. 1196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.Scale != -1) 11977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 119881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can fold an 1200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // integer immediate on an icmp. 1201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 1202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) return TLI->isLegalICmpImmediate(-AM.BaseOffs); 1203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 12067979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return true; 120781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Basic: 1209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Only handle single-register values. 1210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.Scale == 0 && AM.BaseOffs == 0; 121181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Special: 1213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Only handle -1 scales, or no scale. 1214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return AM.Scale == 0 || AM.Scale == -1; 1215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 121981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(TargetLowering::AddrMode AM, 1221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1222db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MinOffset) > AM.BaseOffs) != 1226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MinOffset > 0)) 1227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MinOffset; 1229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(AM, Kind, AccessTy, TLI)) { 1230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs - MinOffset; 1231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MaxOffset) > AM.BaseOffs) != 1233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MaxOffset > 0)) 12347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MaxOffset; 1236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 12377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 12395f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 12405f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(int64_t BaseOffs, 1242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV, 1243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1244db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1245454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman const TargetLowering *TLI) { 1246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1257a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Canonicalize a scale of 1 to a base register if the formula doesn't 1258a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // already have a base register. 1259a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!AM.HasBaseReg && AM.Scale == 1) { 1260a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.Scale = 0; 1261a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.HasBaseReg = true; 1262a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1263a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 1265a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1266a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1267572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(const SCEV *S, 1268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1270db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI, 1272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE) { 1273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (S->isZero()) return true; 1275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t BaseOffs = ExtractImmediate(S, SE); 1279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV = ExtractSymbol(S, SE); 1280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If there's anything else involved, it's not foldable. 1282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!S->isZero()) return false; 1283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, MinOffset, MaxOffset, Kind, AccessTy, TLI); 1296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 129756a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1298b6211710acdf558b3b45c2d198e74aa602496893Dan Gohmannamespace { 1299b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 13001e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// UseMapDenseMapInfo - A DenseMapInfo implementation for holding 13011e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// DenseMaps and DenseSets of pairs of const SCEV* and LSRUse::Kind. 13021e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohmanstruct UseMapDenseMapInfo { 13031e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getEmptyKey() { 13041e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-1), LSRUse::Basic); 13051e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 13061e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 13071e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getTombstoneKey() { 13081e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-2), LSRUse::Basic); 13091e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 13101e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 13111e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static unsigned 13121e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman getHashValue(const std::pair<const SCEV *, LSRUse::KindType> &V) { 13131e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman unsigned Result = DenseMapInfo<const SCEV *>::getHashValue(V.first); 13141e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman Result ^= DenseMapInfo<unsigned>::getHashValue(unsigned(V.second)); 13151e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return Result; 13161e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 13171e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 13181e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static bool isEqual(const std::pair<const SCEV *, LSRUse::KindType> &LHS, 13191e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman const std::pair<const SCEV *, LSRUse::KindType> &RHS) { 13201e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return LHS == RHS; 13211e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 13221e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman}; 13231e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 1324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRInstance - This class holds state for the main loop strength reduction 1325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// logic. 1326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRInstance { 1327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers &IU; 1328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE; 1329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DominatorTree &DT; 1330e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LoopInfo &LI; 1331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 1332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loop *const L; 1333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed; 1334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// IVIncInsertPos - This is the insert position that the current loop's 1336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable increment should be placed. In simple loops, this is 1337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// the latch block's terminator. But in more complicated cases, this is a 1338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// position which will dominate all the in-loop post-increment users. 1339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *IVIncInsertPos; 1340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Factors - Interesting factors between use strides. 1342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<int64_t, 8> Factors; 1343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Types - Interesting use types, to facilitate truncation reuse. 1345db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner SmallSetVector<Type *, 4> Types; 1346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Fixups - The list of operands which are to be replaced. 1348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRFixup, 16> Fixups; 1349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Uses - The list of interesting uses. 1351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRUse, 16> Uses; 1352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// RegUses - Track which uses use which register candidates. 1354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUseTracker RegUses; 1355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void OptimizeShadowIV(); 1357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse); 1358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse); 1359c6519f916b5922de81c53547fd21364994195a70Dan Gohman void OptimizeLoopTermCond(); 1360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectInterestingTypesAndFactors(); 1362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectFixupsAndInitialFormulae(); 1363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &getNewFixup() { 1365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Fixups.push_back(LSRFixup()); 1366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Fixups.back(); 1367a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 136832e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng 1369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Support for sharing of LSRUses between LSRFixups. 13701e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman typedef DenseMap<std::pair<const SCEV *, LSRUse::KindType>, 13711e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman size_t, 13721e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMapDenseMapInfo> UseMapTy; 1373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UseMapTy UseMap; 1374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1375191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 1376db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy); 1377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> getUse(const SCEV *&Expr, 1379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, 1380db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy); 1381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1382c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void DeleteUse(LSRUse &LU, size_t LUIdx); 13835ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 1384191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU); 1385a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 1387454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegisters(const Formula &F, size_t LUIdx); 1390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F); 1391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectLoopInvariantFixupsAndFormulae(); 1393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base, 1395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth = 0); 1396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base); 1397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base); 1402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCrossUseConstantOffsets(); 1403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateAllReuseFormulae(); 1404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void FilterOutUndesirableDedicatedRegisters(); 1406d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 1407d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t EstimateSearchSpaceComplexity() const; 14084aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByDetectingSupersets(); 14094aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByCollapsingUnrolledCode(); 14104f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 14114aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByPickingWinnerRegs(); 1412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void NarrowSearchSpaceUsingHeuristics(); 1413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 1415572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 1416572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 1417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 1418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 1419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const; 1420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Solve(SmallVectorImpl<const Formula *> &Solution) const; 1421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1422e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock::iterator 1423e5f76877aee6f33964de105893f0ef338661ecadDan Gohman HoistInsertPosition(BasicBlock::iterator IP, 1424e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) const; 1425e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP, 1426e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRFixup &LF, 1427e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRUse &LU) const; 1428d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 1429572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *Expand(const LSRFixup &LF, 1430572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1431454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman BasicBlock::iterator IP, 1432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1433454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const; 14343a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman void RewriteForPHI(PHINode *PN, const LSRFixup &LF, 14353a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 14363a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 14373a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 14383a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const; 1439572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Rewrite(const LSRFixup &LF, 1440572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 1443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const; 1444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 1445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P); 1446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1447572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRInstance(const TargetLowering *tli, Loop *l, Pass *P); 1448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1449572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool getChanged() const { return Changed; } 1450572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1451572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_factors_and_types(raw_ostream &OS) const; 1452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_fixups(raw_ostream &OS) const; 1453572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_uses(raw_ostream &OS) const; 1454572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1455572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 14577979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1458a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1459c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// OptimizeShadowIV - If IV is used in a int-to-float cast 14613f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// inside the loop then try to eliminate the cast operation. 1462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::OptimizeShadowIV() { 1463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 1464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 1465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 1466c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); 1468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != E; /* empty */) { 1469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers::const_iterator CandidateUI = UI; 1470572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ++UI; 1471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *ShadowUse = CandidateUI->getUser(); 1472db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *DestTy = NULL; 1473c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick bool IsSigned = false; 1474c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* If shadow use is a int->float cast then insert a second IV 1476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman to eliminate this cast. 1477c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i) 1479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo((double)i); 1480c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman is transformed into 1482c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman double d = 0.0; 1484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i, ++d) 1485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo(d); 1486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman */ 1487c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) { 1488c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick IsSigned = false; 1489572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = UCast->getDestTy(); 1490c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick } 1491c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) { 1492c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick IsSigned = true; 1493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = SCast->getDestTy(); 1494c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick } 1495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DestTy) continue; 14965792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) { 1498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If target does not support DestTy natively then do not apply 1499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // this transformation. 1500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman EVT DVT = TLI->getValueType(DestTy); 1501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI->isTypeLegal(DVT)) continue; 15027e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 15037e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); 1505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!PH) continue; 1506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getNumIncomingValues() != 2) continue; 1507a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1508db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *SrcTy = PH->getType(); 1509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int Mantissa = DestTy->getFPMantissaWidth(); 1510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mantissa == -1) continue; 1511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa) 1512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 15132f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 1514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Entry, Latch; 1515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { 1516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 0; 1517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 1; 1518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 1519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 1; 1520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 0; 1521a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1522eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 1523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); 1524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Init) continue; 1525c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick Constant *NewInit = ConstantFP::get(DestTy, IsSigned ? 1526c205a094bd5019773c98adcfbdc21c07c9da1888Andrew Trick (double)Init->getSExtValue() : 1527c205a094bd5019773c98adcfbdc21c07c9da1888Andrew Trick (double)Init->getZExtValue()); 1528a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *Incr = 1530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); 1531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Incr) continue; 1532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOpcode() != Instruction::Add 1533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman && Incr->getOpcode() != Instruction::Sub) 1534572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1535aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 1536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Initialize new IV, double d = 0.0 in above example. */ 1537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *C = NULL; 1538572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOperand(0) == PH) 1539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(1)); 1540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Incr->getOperand(1) == PH) 1541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(0)); 1542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1544bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman 1545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C) continue; 1546a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1547572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore negative constants, as the code below doesn't handle them 1548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // correctly. TODO: Remove this restriction. 1549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C->getValue().isStrictlyPositive()) continue; 1550a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Add new PHINode. */ 15523ecfc861b4365f341c5c969b40e1afccde676e6fJay Foad PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH); 1553a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* create new increment. '++d' in above example. */ 1555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue()); 1556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *NewIncr = 1557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ? 1558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction::FAdd : Instruction::FSub, 1559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH, CFP, "IV.S.next.", Incr); 1560a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1561572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); 1562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); 15638b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 1564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Remove cast operation */ 1565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->replaceAllUsesWith(NewPH); 1566572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->eraseFromParent(); 1567c6519f916b5922de81c53547fd21364994195a70Dan Gohman Changed = true; 1568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 15697979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1570a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1571cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 15727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// FindIVUserForCond - If Cond has an operand that is an expression of an IV, 15737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// set the IV user and stride information and return true, otherwise return 15747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// false. 1575ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanbool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) { 1576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UI->getUser() == Cond) { 1578572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // NOTE: we could handle setcc instructions with multiple uses here, but 1579572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // InstCombine does it as well for simple uses, it's not clear that it 1580572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // occurs enough in real life to handle. 1581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CondUse = UI; 1582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1583a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1585cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 1586cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 15877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// OptimizeMax - Rewrite the loop's terminating condition if it uses 15887979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a max computation. 15897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This is a narrow solution to a specific, but acute, problem. For loops 15917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// like this: 15927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 15947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 15957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 15967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 15977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 15987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the trip count isn't just 'n', because 'n' might not be positive. And 15997979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// unfortunately this can come up even for loops where the user didn't use 16007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a C do-while loop. For example, seemingly well-behaved top-test loops 16017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// will commonly be lowered like this: 16027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman// 16037979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// if (n > 0) { 16047979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 16057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 16067979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 16077979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 16087979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } 16097979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 16107979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// and then it's possible for subsequent optimization to obscure the if 16117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// test in such a way that indvars can't find it. 16127979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 16137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// When indvars can't find the if test in loops like this, it creates a 16147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max expression, which allows it to give the loop a canonical 16157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// induction variable: 16167979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 16177979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 16187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max = n < 1 ? 1 : n; 16197979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 16207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 16217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i != max); 16227979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 16237979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// Canonical induction variables are necessary because the loop passes 16247979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// are designed around them. The most obvious example of this is the 16257979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// LoopInfo analysis, which doesn't remember trip count values. It 16267979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// expects to be able to rediscover the trip count each time it is 1627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needed, and it does this using a simple analysis that only succeeds if 16287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the loop has a canonical induction variable. 16297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 16307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// However, when it comes time to generate code, the maximum operation 16317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// can be quite costly, especially if it's inside of an outer loop. 16327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 16337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This function solves this problem by detecting this type of loop and 16347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting 16357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the instructions for the maximum computation. 16367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 1637572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { 16387979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check that the loop matches the pattern we're looking for. 16397979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() != CmpInst::ICMP_EQ && 16407979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->getPredicate() != CmpInst::ICMP_NE) 16417979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1642ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); 16447979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!Sel || !Sel->hasOneUse()) return Cond; 1645ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 1646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 16477979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 16487979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1649deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); 1650ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16517979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Add one to the backedge-taken count to get the trip count. 16524065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); 16531d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (IterationCount != SE.getSCEV(Sel)) return Cond; 16541d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 16551d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Check for a max calculation that matches the pattern. There's no check 16561d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for ICMP_ULE here because the comparison would be with zero, which 16571d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // isn't interesting. 16581d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; 16591d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman const SCEVNAryExpr *Max = 0; 16601d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) { 16611d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLE; 16621d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 16631d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) { 16641d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLT; 16651d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 16661d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) { 16671d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_ULT; 16681d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = U; 16691d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else { 16701d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // No match; bail. 16717979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 16721d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } 1673ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // To handle a max with more than two operands, this optimization would 16757979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // require additional checking and setup. 16767979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Max->getNumOperands() != 2) 16777979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 16787979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 16797979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxLHS = Max->getOperand(0); 16807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxRHS = Max->getOperand(1); 16811d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 16821d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // ScalarEvolution canonicalizes constants to the left. For < and >, look 16831d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for a comparison with 1. For <= and >=, a comparison with zero. 16841d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!MaxLHS || 16851d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One))) 16861d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 16871d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 16887979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the relevant induction variable for conformance to 16897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // the pattern. 1690572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); 16917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 16927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!AR || !AR->isAffine() || 16937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AR->getStart() != One || 1694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE) != One) 16957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1696ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 16977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman assert(AR->getLoop() == L && 16987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman "Loop condition operand is an addrec in a different loop!"); 1699bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman 17007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the right operand of the select, and remember it, as it will 17017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // be used in the new comparison instruction. 17027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Value *NewRHS = 0; 17031d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (ICmpInst::isTrueWhenEqual(Pred)) { 17041d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Look for n+1, and grab n. 17051d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1))) 17061d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 17071d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 17081d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 17091d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 17101d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2))) 17111d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 17121d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 17131d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 17141d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 17151d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!NewRHS) 17161d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 17171d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS) 17187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(1); 1719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS) 17207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(2); 1721caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS)) 1722caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman NewRHS = SU->getValue(); 17231d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman else 1724caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman // Max doesn't match expected pattern. 1725caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman return Cond; 1726ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 17277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Determine the new comparison opcode. It may be signed or unsigned, 17287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // and the original comparison may be either equality or inequality. 17297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() == CmpInst::ICMP_EQ) 17307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Pred = CmpInst::getInversePredicate(Pred); 17312781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman 17327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Ok, everything looks ok to change the condition into an SLT or SGE and 17337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // delete the max calculation. 17347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman ICmpInst *NewCond = 17357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp"); 1736ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 17377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Delete the max calculation instructions. 17387979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->replaceAllUsesWith(NewCond); 17397979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman CondUse->setUser(NewCond); 17407979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); 17417979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->eraseFromParent(); 17427979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Sel->eraseFromParent(); 17437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cmp->use_empty()) 17447979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cmp->eraseFromParent(); 17457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return NewCond; 1746ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman} 1747ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 174856a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach/// OptimizeLoopTermCond - Change loop terminating condition to use the 17492d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng/// postinc iv when possible. 1750c6519f916b5922de81c53547fd21364994195a70Dan Gohmanvoid 1751572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::OptimizeLoopTermCond() { 1752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<Instruction *, 4> PostIncs; 1753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 17545792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng BasicBlock *LatchBlock = L->getLoopLatch(); 1755076e085698c484354c9e131f1bd8fd001581397bEvan Cheng SmallVector<BasicBlock*, 8> ExitingBlocks; 1756076e085698c484354c9e131f1bd8fd001581397bEvan Cheng L->getExitingBlocks(ExitingBlocks); 175756a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1758076e085698c484354c9e131f1bd8fd001581397bEvan Cheng for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 1759076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BasicBlock *ExitingBlock = ExitingBlocks[i]; 1760010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 1761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Get the terminating condition for the loop if possible. If we 1762076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // can, we want to change it to use a post-incremented version of its 1763076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // induction variable, to allow coalescing the live ranges for the IV into 1764076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // one register value. 1765cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1766076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); 1767076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (!TermBr) 1768076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1769076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // FIXME: Overly conservative, termination condition could be an 'or' etc.. 1770076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition())) 1771076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1772076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 1773076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Search IVUsesByStride to find Cond's IVUse if there is one. 1774076e085698c484354c9e131f1bd8fd001581397bEvan Cheng IVStrideUse *CondUse = 0; 1775076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); 1776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!FindIVUserForCond(Cond, CondUse)) 1777076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1778076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 17797979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // If the trip count is computed in terms of a max (due to ScalarEvolution 17807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // being unable to find a sufficient guard, for example), change the loop 17817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // comparison to use SLT or ULT instead of NE. 1782572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // One consequence of doing this now is that it disrupts the count-down 1783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // optimization. That's not always a bad thing though, because in such 1784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // cases it may still be worthwhile to avoid a max. 1785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cond = OptimizeMax(Cond, CondUse); 1786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1787572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this exiting block dominates the latch block, it may also use 1788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the post-inc value if it won't be shared with other uses. 1789572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for dominance. 1790572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(ExitingBlock, LatchBlock)) 17917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman continue; 17927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively avoid trying to use the post-inc value in non-latch 1794572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // exits if there may be pre-inc users in intervening blocks. 1795590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (LatchBlock != ExitingBlock) 1796572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Test if the use is reachable from the exiting block. This dominator 1798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // query is a conservative approximation of reachability. 1799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*UI != CondUse && 1800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) { 1801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume there may be reuse if the quotient of their 1802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // strides could be a legal scale. 1803c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *A = IU.getStride(*CondUse, L); 1804c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *B = IU.getStride(*UI, L); 1805448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (!A || !B) continue; 1806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) != 1807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) { 1808572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) > 1809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) 1810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman B = SE.getSignExtendExpr(B, A->getType()); 1811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1812572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman A = SE.getSignExtendExpr(A, B->getType()); 1813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1814572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *D = 1815f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) { 18169f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const ConstantInt *C = D->getValue(); 1817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Stride of one or negative one can have reuse with non-addresses. 18189f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->isOne() || C->isAllOnesValue()) 1819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1820572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Avoid weird situations. 18219f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->getValue().getMinSignedBits() >= 64 || 18229f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman C->getValue().isMinSignedValue()) 1823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1824590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // Without TLI, assume that any stride might be valid, and so any 1825590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // use might be shared. 1826590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (!TLI) 1827590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman goto decline_post_inc; 1828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for possible scaled-address reuse. 1829db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = getAccessType(UI->getUser()); 1830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 18319f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman AM.Scale = C->getSExtValue(); 18322763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 1833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = -AM.Scale; 18352763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 1836572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 1837572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1838572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 184063c9463c62fce8cbe02176dfa2d73f375a06f1f2David Greene DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: " 1841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << *Cond << '\n'); 184256a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1843076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // It's possible for the setcc instruction to be anywhere in the loop, and 1844076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // possible for it to have multiple users. If it is not immediately before 1845076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // the exiting block branch, move it. 1846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*++BasicBlock::iterator(Cond) != TermBr) { 1847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Cond->hasOneUse()) { 1848076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->moveBefore(TermBr); 1849076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } else { 1850572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clone the terminating condition and insert into the loopend. 1851572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OldCond = Cond; 1852076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond = cast<ICmpInst>(Cond->clone()); 1853076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->setName(L->getHeader()->getName() + ".termcond"); 1854076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ExitingBlock->getInstList().insert(TermBr, Cond); 1855076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 1856076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Clone the IVUse, as the old use still exists! 18574417e537b65c14b378aeca75b2773582dd102f63Andrew Trick CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace()); 1858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TermBr->replaceUsesOfWith(OldCond, Cond); 1859076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } 1860010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 1861010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 1862076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // If we get to here, we know that we can transform the setcc instruction to 1863076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // use the post-incremented version of the IV, allowing us to coalesce the 1864076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // live ranges for the IV correctly. 1865448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman CondUse->transformToPostInc(L); 1866076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Changed = true; 18675792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1868572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PostIncs.insert(Cond); 1869572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman decline_post_inc:; 1870c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen } 1871c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 1872572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine an insertion point for the loop induction variable increment. It 1873572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // must dominate all the post-inc comparisons we just set up, and it must 1874572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // dominate the loop latch edge. 1875572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = L->getLoopLatch()->getTerminator(); 1876572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<Instruction *, 4>::const_iterator I = PostIncs.begin(), 1877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = PostIncs.end(); I != E; ++I) { 1878572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BasicBlock *BB = 1879572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT.findNearestCommonDominator(IVIncInsertPos->getParent(), 1880572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (*I)->getParent()); 1881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BB == (*I)->getParent()) 1882572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = *I; 1883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (BB != IVIncInsertPos->getParent()) 1884572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = BB->getTerminator(); 1885572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1886572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1887a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 18887a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner/// reconcileNewOffset - Determine if the given use can accommodate a fixup 188976c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// at the given offset and other details. If so, update the use and 189076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// return true. 1891572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 1892191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan GohmanLSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 1893db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy) { 1894191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMinOffset = LU.MinOffset; 1895191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMaxOffset = LU.MaxOffset; 1896db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *NewAccessTy = AccessTy; 1897572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched kind. It's tempting to collapse mismatched kinds to 1899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // something conservative, however this can pessimize in the case that one of 1900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the uses will have all its uses outside the loop, for example. 1901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != Kind) 1902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume HasBaseReg is true for now. 1904191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset < LU.MinOffset) { 1905191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(LU.MaxOffset - NewOffset, 0, HasBaseReg, 1906454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 19077979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1908191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMinOffset = NewOffset; 1909191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } else if (NewOffset > LU.MaxOffset) { 1910191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(NewOffset - LU.MinOffset, 0, HasBaseReg, 1911454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 19127979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1913191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMaxOffset = NewOffset; 1914a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched access type, and fall back conservatively as needed. 191674e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // TODO: Be less conservative when the type is similar and can use the same 191774e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // addressing modes. 1918572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Kind == LSRUse::Address && AccessTy != LU.AccessTy) 1919191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewAccessTy = Type::getVoidTy(AccessTy->getContext()); 1920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the use. 1922191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MinOffset = NewMinOffset; 1923191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MaxOffset = NewMaxOffset; 1924191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.AccessTy = NewAccessTy; 1925191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset != LU.Offsets.back()) 1926191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(NewOffset); 1927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1929a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getUse - Return an LSRUse index and an offset value for a fixup which 1931572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needs the given expression, with the given kind and optional access type. 19323f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// Either reuse an existing use or create a new one, as needed. 1933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstd::pair<size_t, int64_t> 1934572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::getUse(const SCEV *&Expr, 1935db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy) { 1936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Copy = Expr; 1937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = ExtractImmediate(Expr, SE); 1938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Basic uses can't accept any offset, for example. 1940454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!isAlwaysFoldable(Offset, 0, /*HasBaseReg=*/true, Kind, AccessTy, TLI)) { 1941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Expr = Copy; 1942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = 0; 1943a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1944a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<UseMapTy::iterator, bool> P = 19461e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMap.insert(std::make_pair(std::make_pair(Expr, Kind), 0)); 1947572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!P.second) { 1948572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // A use already existed with this base. 1949572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = P.first->second; 1950572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 1951191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy)) 1952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Reuse this use. 1953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 1954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1955a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Create a new use. 1957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = Uses.size(); 1958572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman P.first->second = LUIdx; 1959572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses.push_back(LSRUse(Kind, AccessTy)); 1960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 1961a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1962191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // We don't need to track redundant offsets, but we don't need to go out 1963191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // of our way here to avoid them. 1964191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LU.Offsets.empty() || Offset != LU.Offsets.back()) 1965191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(Offset); 1966191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 1967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MinOffset = Offset; 1968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MaxOffset = Offset; 1969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 1970a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1971a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 19725ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteUse - Delete the given use from the Uses list. 1973c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohmanvoid LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) { 1974191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (&LU != &Uses.back()) 19755ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(LU, Uses.back()); 19765ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman Uses.pop_back(); 1977c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 1978c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. 1979c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman RegUses.SwapAndDropUse(LUIdx, Uses.size()); 19805ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 19815ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 1982a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// FindUseWithFormula - Look for a use distinct from OrigLU which is has 1983a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// a formula that has the same registers as the given formula. 1984a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRUse * 1985a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRInstance::FindUseWithSimilarFormula(const Formula &OrigF, 1986191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman const LSRUse &OrigLU) { 1987191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Search all uses for the formula. This could be more clever. 1988a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 1989a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 19906a832715325f740a0e78acd194abad9104389075Dan Gohman // Check whether this use is close enough to OrigLU, to see whether it's 19916a832715325f740a0e78acd194abad9104389075Dan Gohman // worthwhile looking through its formulae. 19926a832715325f740a0e78acd194abad9104389075Dan Gohman // Ignore ICmpZero uses because they may contain formulae generated by 19936a832715325f740a0e78acd194abad9104389075Dan Gohman // GenerateICmpZeroScales, in which case adding fixup offsets may 19946a832715325f740a0e78acd194abad9104389075Dan Gohman // be invalid. 1995a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (&LU != &OrigLU && 1996a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind != LSRUse::ICmpZero && 1997a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy && 1998a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType == OrigLU.WidestFixupType && 1999a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.HasFormulaWithSameRegs(OrigF)) { 20006a832715325f740a0e78acd194abad9104389075Dan Gohman // Scan through this use's formulae. 2001402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 2002402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 2003402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 20046a832715325f740a0e78acd194abad9104389075Dan Gohman // Check to see if this formula has the same registers and symbols 20056a832715325f740a0e78acd194abad9104389075Dan Gohman // as OrigF. 2006a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.BaseRegs == OrigF.BaseRegs && 2007a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.ScaledReg == OrigF.ScaledReg && 2008a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.AM.BaseGV == OrigF.AM.BaseGV && 2009cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.AM.Scale == OrigF.AM.Scale && 2010cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset == OrigF.UnfoldedOffset) { 2011191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (F.AM.BaseOffs == 0) 2012a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return &LU; 20136a832715325f740a0e78acd194abad9104389075Dan Gohman // This is the formula where all the registers and symbols matched; 20146a832715325f740a0e78acd194abad9104389075Dan Gohman // there aren't going to be any others. Since we declined it, we 20156a832715325f740a0e78acd194abad9104389075Dan Gohman // can skip the rest of the formulae and procede to the next LSRUse. 2016a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 2017a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2018a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2019a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2020a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2021a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 20226a832715325f740a0e78acd194abad9104389075Dan Gohman // Nothing looked good. 2023a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return 0; 2024a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 2025a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectInterestingTypesAndFactors() { 2027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> Strides; 2028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 20291b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Collect interesting types and strides. 2030448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman SmallVector<const SCEV *, 4> Worklist; 2031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 2032c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *Expr = IU.getExpr(*UI); 2033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect interesting types. 2035448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Types.insert(SE.getEffectiveSCEVType(Expr->getType())); 2036448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 2037448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // Add strides for mentioned loops. 2038448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(Expr); 2039448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman do { 2040448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman const SCEV *S = Worklist.pop_back_val(); 2041448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 2042448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Strides.insert(AR->getStepRecurrence(SE)); 2043448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(AR->getStart()); 2044448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 2045403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(Add->op_begin(), Add->op_end()); 2046448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 2047448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } while (!Worklist.empty()); 20481b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman } 20491b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman 20501b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Compute interesting factors from the set of interesting strides. 20511b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator 20521b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman I = Strides.begin(), E = Strides.end(); I != E; ++I) 2053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = 2054ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes llvm::next(I); NewStrideIter != E; ++NewStrideIter) { 20551b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman const SCEV *OldStride = *I; 2056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewStride = *NewStrideIter; 2057a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2058572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) != 2059572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) { 2060572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) > 2061572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) 2062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType()); 2063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2064572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType()); 2065572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2066572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *Factor = 2067f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride, 2068f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2069572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2070572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2071572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVConstant *Factor = 2072454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride, 2073454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman NewStride, 2074f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2075572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2076572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2077a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2078a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2079586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2080572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If all uses use the same type, don't bother looking for truncation-based 2081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reuse. 2082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Types.size() == 1) 2083572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 2084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2085572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_factors_and_types(dbgs())); 2086c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen} 2087c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2088572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectFixupsAndInitialFormulae() { 2089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 2090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record the uses. 2091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 2092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst = UI->getUser(); 2093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI->getOperandValToReplace(); 2094448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.PostIncLoops = UI->getPostIncLoops(); 20950f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel 2096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind = LSRUse::Basic; 2097db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = 0; 2098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAddressUse(LF.UserInst, LF.OperandValToReplace)) { 2099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::Address; 2100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = getAccessType(LF.UserInst); 2101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2103c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *S = IU.getExpr(*UI); 2104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Equality (== and !=) ICmps are special. We can rewrite (i == N) as 2106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // (N - i == 0), and this allows (N - i) to be the expression that we work 2107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with rather than just N or i, so we can consider the register 2108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // requirements for both N and i at the same time. Limiting this code to 2109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps is not a problem because all interesting loops use 2110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps, thanks to IndVarSimplify. 2111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpInst *CI = dyn_cast<ICmpInst>(LF.UserInst)) 2112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (CI->isEquality()) { 2113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Swap the operands if needed to put the OperandValToReplace on the 2114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // left, for consistency. 2115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *NV = CI->getOperand(1); 2116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NV == LF.OperandValToReplace) { 2117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, CI->getOperand(0)); 2118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(0, NV); 2119f182b23f8fed014c26da48d94890080c7d916a0cDan Gohman NV = CI->getOperand(1); 21209da1bf4845e670096b9bf9e62c40960af1697ea0Dan Gohman Changed = true; 2121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // x == y --> x - y == 0 2124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *N = SE.getSCEV(NV); 212517ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (SE.isLoopInvariant(N, L)) { 2126673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman // S is normalized, so normalize N before folding it into S 2127673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman // to keep the result normalized. 2128673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman N = TransformForPostIncUse(Normalize, N, CI, 0, 2129673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman LF.PostIncLoops, SE, DT); 2130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::ICmpZero; 2131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman S = SE.getMinusSCEV(N, S); 2132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // -1 and the negations of all interesting strides (except the negation 2135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of -1) are now also interesting. 2136572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Factors.size(); i != e; ++i) 2137572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors[i] != -1) 2138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-(uint64_t)Factors[i]); 2139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-1); 2140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Set up the initial formula for this use. 2143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy); 2144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 2145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 2146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 2147448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 2148a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 2149a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 2150a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 2151a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 2152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is the first use of this LSRUse, give it a formula. 2154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Formulae.empty()) { 2155454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman InsertInitialFormula(S, LU, LF.LUIdx); 2156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), LF.LUIdx); 2157572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2160572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_fixups(dbgs())); 2161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 216376c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertInitialFormula - Insert a formula for the given expression into 216476c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given use, separating out loop-variant portions from loop-invariant 216576c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// and loop-computable portions. 2166572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2167454d26dc43207ec537d843229db6f5e6a302e23dDan GohmanLSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { 2168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2169dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman F.InitialMatch(S, L, SE); 2170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Initial formula already exists!"); (void)Inserted; 2172572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2173572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 217476c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertSupplementalFormula - Insert a simple single-register formula for 217576c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given expression into the given use. 2176572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2177572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::InsertSupplementalFormula(const SCEV *S, 2178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU, size_t LUIdx) { 2179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(S); 2181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.HasBaseReg = true; 2182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2183572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Supplemental formula already exists!"); (void)Inserted; 2184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2186572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CountRegisters - Note which registers are used by the given formula, 2187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// updating RegUses. 2188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) { 2189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) 2190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(F.ScaledReg, LUIdx); 2191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 2192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) 2193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(*I, LUIdx); 2194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 2197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 2198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) { 2199454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!LU.InsertFormula(F)) 22007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 220180b0f8c0628d848d36f04440687b61e3b2d3dff1Dan Gohman 2202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(F, LUIdx); 2203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 2204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectLoopInvariantFixupsAndFormulae - Check for other uses of 2207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-invariant values which we're tracking. These other uses will pin these 2208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// values in registers, making them less profitable for elimination. 2209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: This currently misses non-constant addrec step registers. 2210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Should this give more weight to users inside the loop? 2211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2212572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::CollectLoopInvariantFixupsAndFormulae() { 2213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); 2214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 8> Inserted; 2215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!Worklist.empty()) { 2217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *S = Worklist.pop_back_val(); 2218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S)) 2220403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(N->op_begin(), N->op_end()); 2221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S)) 2222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(C->getOperand()); 2223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) { 2224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getLHS()); 2225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getRHS()); 2226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 2227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Inserted.insert(U)) continue; 2228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Value *V = U->getValue(); 2229a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 2230a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Look for instructions defined outside the loop. 2231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (L->contains(Inst)) continue; 2232a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman } else if (isa<UndefValue>(V)) 2233a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Undef doesn't have a live range, so it doesn't matter. 2234a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman continue; 223560ad781c61815ca5b8dc2a45a102e1c8af65992fGabor Greif for (Value::const_use_iterator UI = V->use_begin(), UE = V->use_end(); 2236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != UE; ++UI) { 2237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Instruction *UserInst = dyn_cast<Instruction>(*UI); 2238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore non-instructions. 2239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!UserInst) 2240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions in other functions (as can happen with 2242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Constants). 2243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UserInst->getParent()->getParent() != L->getHeader()->getParent()) 2244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions not dominated by the loop. 2246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const BasicBlock *UseBB = !isa<PHINode>(UserInst) ? 2247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UserInst->getParent() : 2248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman cast<PHINode>(UserInst)->getIncomingBlock( 2249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode::getIncomingValueNumForOperand(UI.getOperandNo())); 2250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(L->getHeader(), UseBB)) 2251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore uses which are part of other SCEV expressions, to avoid 2253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // analyzing them multiple times. 22544a2a68336638e99a475b09a9278399db6749618fDan Gohman if (SE.isSCEVable(UserInst->getType())) { 22554a2a68336638e99a475b09a9278399db6749618fDan Gohman const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); 22564a2a68336638e99a475b09a9278399db6749618fDan Gohman // If the user is a no-op, look through to its uses. 22574a2a68336638e99a475b09a9278399db6749618fDan Gohman if (!isa<SCEVUnknown>(UserS)) 22584a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 22594a2a68336638e99a475b09a9278399db6749618fDan Gohman if (UserS == U) { 22604a2a68336638e99a475b09a9278399db6749618fDan Gohman Worklist.push_back( 22614a2a68336638e99a475b09a9278399db6749618fDan Gohman SE.getUnknown(const_cast<Instruction *>(UserInst))); 22624a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 22634a2a68336638e99a475b09a9278399db6749618fDan Gohman } 22644a2a68336638e99a475b09a9278399db6749618fDan Gohman } 2265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore icmp instructions which are already being analyzed. 2266572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) { 2267572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned OtherIdx = !UI.getOperandNo(); 2268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx)); 226917ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L)) 2270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 2274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst = const_cast<Instruction *>(UserInst); 2275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI.getUse(); 2276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, LSRUse::Basic, 0); 2277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 2278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 2279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 2280448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 2281a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 2282a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 2283a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 2284a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 2285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman InsertSupplementalFormula(U, LU, LF.LUIdx); 2286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), Uses.size() - 1); 2287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 2288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectSubexprs - Split S into subexpressions which can be pulled out into 2294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// separate registers. If C is non-null, multiply each subexpression by C. 2295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic void CollectSubexprs(const SCEV *S, const SCEVConstant *C, 2296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Ops, 22973e3f15bb09eaca4d259e531c9a1ecafb5710b57bDan Gohman const Loop *L, 2298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE) { 2299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 2300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break out add operands. 2301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 2302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) 23033e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(*I, C, Ops, L, SE); 2304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 2306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Split a non-zero base out of an addrec. 2307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!AR->getStart()->isZero()) { 2308deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman CollectSubexprs(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), 2309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE), 23103228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick AR->getLoop(), 23113228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick //FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 23123228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap), 23133e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman C, Ops, L, SE); 23143e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(AR->getStart(), C, Ops, L, SE); 2315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2316572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 2318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break (C * (a + b + c)) into C*a + C*b + C*c. 2319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mul->getNumOperands() == 2) 2320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *Op0 = 2321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<SCEVConstant>(Mul->getOperand(0))) { 2322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectSubexprs(Mul->getOperand(1), 2323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0, 23243e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman Ops, L, SE); 2325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 23293e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // Otherwise use the value itself, optionally with a scale applied. 23303e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman Ops.push_back(C ? SE.getMulExpr(C, S) : S); 2331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateReassociations - Split out subexpressions from adds and the bases of 2334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrecs. 2335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx, 2336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base, 2337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth) { 2338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Arbitrarily cap recursion to protect compile time. 2339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Depth >= 3) return; 2340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 2342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = Base.BaseRegs[i]; 2343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 23443e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman SmallVector<const SCEV *, 8> AddOps; 23453e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman CollectSubexprs(BaseReg, 0, AddOps, L, SE); 23463e3f15bb09eaca4d259e531c9a1ecafb5710b57bDan Gohman 2347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddOps.size() == 1) continue; 2348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), 2350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = AddOps.end(); J != JE; ++J) { 23513e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 23523e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // Loop-variant "unknown" values are uninteresting; we won't be able to 23533e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // do anything meaningful with them. 235417ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L)) 23553e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman continue; 23563e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 2357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't pull a constant into a register if the constant could be folded 2358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // into an immediate field. 2359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAlwaysFoldable(*J, LU.MinOffset, LU.MaxOffset, 2360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 2361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 2362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect all operands except *J. 2365403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman SmallVector<const SCEV *, 8> InnerAddOps 23664eaee28e347f33b3e23d2fb7275fc5222d5482fdDan Gohman (((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); 2367403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman InnerAddOps.append 2368ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes (llvm::next(J), ((const SmallVector<const SCEV *, 8> &)AddOps).end()); 2369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't leave just a constant behind in a register if the constant could 2371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // be folded into an immediate field. 2372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InnerAddOps.size() == 1 && 2373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isAlwaysFoldable(InnerAddOps[0], LU.MinOffset, LU.MaxOffset, 2374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 2375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 2376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2378fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); 2379fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman if (InnerSum->isZero()) 2380fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman continue; 2381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2382cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 2383cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Add the remaining pieces of the add back into the new formula. 2384cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum); 2385cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (TLI && InnerSumSC && 2386cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 && 2387cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman TLI->isLegalAddImmediate((uint64_t)F.UnfoldedOffset + 2388cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman InnerSumSC->getValue()->getZExtValue())) { 2389cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset + 2390cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman InnerSumSC->getValue()->getZExtValue(); 2391cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs.erase(F.BaseRegs.begin() + i); 2392cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } else 2393cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs[i] = InnerSum; 2394cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 2395cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Add J as its own register, or an unfolded immediate. 2396cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J); 2397cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (TLI && SC && SE.getTypeSizeInBits(SC->getType()) <= 64 && 2398cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman TLI->isLegalAddImmediate((uint64_t)F.UnfoldedOffset + 2399cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SC->getValue()->getZExtValue())) 2400cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset + 2401cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SC->getValue()->getZExtValue(); 2402cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman else 2403cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs.push_back(*J); 2404cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 2405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InsertFormula(LU, LUIdx, F)) 2406572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If that formula hadn't been seen before, recurse to find more like 2407572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // it. 2408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth+1); 2409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCombinations - Generate a formula consisting of all of the 2414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-dominating registers added into a single register. 2415572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx, 2416441a38993e89104c424e00c95cce63c7351f4fc3Dan Gohman Formula Base) { 24173f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // This method is only interesting on a plurality of registers. 2418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.BaseRegs.size() <= 1) return; 2419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.clear(); 2422572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 2423572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 2424572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Base.BaseRegs.begin(), E = Base.BaseRegs.end(); I != E; ++I) { 2425572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 2426dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(BaseReg, L->getHeader()) && 242717ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman !SE.hasComputableLoopEvolution(BaseReg, L)) 2428572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(BaseReg); 2429572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2430572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(BaseReg); 2431572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Ops.size() > 1) { 2433ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman const SCEV *Sum = SE.getAddExpr(Ops); 2434ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // TODO: If Sum is zero, it probably means ScalarEvolution missed an 2435ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // opportunity to fold something. For now, just ignore such cases 24363f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // rather than proceed with zero in a register. 2437ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman if (!Sum->isZero()) { 2438ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman F.BaseRegs.push_back(Sum); 2439ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2440ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman } 2441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateSymbolicOffsets - Generate reuse formulae using symbolic offsets. 2445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, 2446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 2447572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We can't add a symbolic offset if the address already contains one. 2448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 2449572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2450572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 2451572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 2452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *GV = ExtractSymbol(G, SE); 2453572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || !GV) 2454572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2455572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseGV = GV; 2457572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 2458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 2459572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 2461572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets. 2466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, 2467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 2468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: For now, just add the min and max offset, because it usually isn't 2469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // worthwhile looking at everything inbetween. 2470c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman SmallVector<int64_t, 2> Worklist; 2471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MinOffset); 2472572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MaxOffset != LU.MinOffset) 2473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MaxOffset); 2474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 2476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 2477572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Worklist.begin(), 2479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Worklist.end(); I != E; ++I) { 2480572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)Base.AM.BaseOffs - *I; 2482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(F.AM, LU.MinOffset - *I, LU.MaxOffset - *I, 2483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 2484c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // Add the offset to the base register. 24854065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G); 2486c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // If it cancelled out, drop the base register, otherwise update it. 2487c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman if (NewG->isZero()) { 2488c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman std::swap(F.BaseRegs[i], F.BaseRegs.back()); 2489c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs.pop_back(); 2490c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman } else 2491c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs[i] = NewG; 2492572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2494572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2496572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(G, SE); 2498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || Imm == 0) 2499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Imm; 2502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 2503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 2504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 2506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateICmpZeroScales - For ICmpZero, check to see if we can scale up 2511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the comparison. For example, x == y -> x*c == y*c. 2512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, 2513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 2514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != LSRUse::ICmpZero) return; 2515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 2517db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = Base.getType(); 2518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 2519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(IntTy) > 64) return; 2520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't do this if there is more than one offset. 2522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MinOffset != LU.MaxOffset) return; 2523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Base.AM.BaseGV && "ICmpZero use is not legal!"); 2525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 2527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 2528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 2529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 2530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that the multiplication doesn't overflow. 25322ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (Base.AM.BaseOffs == INT64_MIN && Factor == -1) 2533968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 25342ea09e05466613a22e1211f52c30cd01af563983Dan Gohman int64_t NewBaseOffs = (uint64_t)Base.AM.BaseOffs * Factor; 25352ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (NewBaseOffs / Factor != Base.AM.BaseOffs) 2536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2538572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the use offset doesn't overflow. 2539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = LU.MinOffset; 2540968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman if (Offset == INT64_MIN && Factor == -1) 2541968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 2542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = (uint64_t)Offset * Factor; 2543378c0b35a7b9884f1de2a9762825424fbf1813acDan Gohman if (Offset / Factor != LU.MinOffset) 2544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 25462ea09e05466613a22e1211f52c30cd01af563983Dan Gohman Formula F = Base; 25472ea09e05466613a22e1211f52c30cd01af563983Dan Gohman F.AM.BaseOffs = NewBaseOffs; 25482ea09e05466613a22e1211f52c30cd01af563983Dan Gohman 2549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that this scale is legal. 2550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, Offset, Offset, LU.Kind, LU.AccessTy, TLI)) 2551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compensate for the use having MinOffset built into it. 2554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Offset - LU.MinOffset; 2555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2556deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 2557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with each base register doesn't overflow. 2559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) { 2560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS); 2561f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i]) 2562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto next; 2563572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the scaled register doesn't overflow. 2566572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) { 2567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS); 2568f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg) 2569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2572cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Check that multiplying with the unfolded offset doesn't overflow. 2573cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (F.UnfoldedOffset != 0) { 25741b58d4536a561f28bf935dcb29b483c52a6bf4c6Dan Gohman if (F.UnfoldedOffset == INT64_MIN && Factor == -1) 25751b58d4536a561f28bf935dcb29b483c52a6bf4c6Dan Gohman continue; 2576cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset * Factor; 2577cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (F.UnfoldedOffset / Factor != Base.UnfoldedOffset) 2578cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman continue; 2579cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 2580cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 2581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we make it here and it's legal, add it. 2582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman next:; 2584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2587572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateScales - Generate stride factor reuse formulae by making use of 2588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// scaled-offset address modes, for example. 2589ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) { 2590572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 2591db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = Base.getType(); 2592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 2593572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this Formula already has a scaled register, we can't add another one. 2595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.Scale != 0) return; 2596572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2597572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 2598572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 2599572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 2600572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 2601572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.Scale = Factor; 2603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.HasBaseReg = Base.BaseRegs.size() > 1; 2604572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check whether this scale is going to be legal. 2605572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 2606572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 2607572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // As a special-case, handle special out-of-loop Basic users specially. 2608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Reconsider this special case. 2609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::Basic && 2610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 2611572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::Special, LU.AccessTy, TLI) && 2612572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.AllFixupsOutsideLoop) 2613572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind = LSRUse::Special; 2614572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2615572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2617572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For an ICmpZero, negating a solitary base register won't lead to 2618572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // new solutions. 2619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero && 2620572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !Base.AM.HasBaseReg && Base.AM.BaseOffs == 0 && !Base.AM.BaseGV) 2621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For each addrec base reg, apply the scale, if possible. 2623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) 2624572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = 2625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i])) { 2626deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 2627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FactorS->isZero()) 2628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Divide out the factor, ignoring high bits, since we'll be 2630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scaling the value back up in the end. 2631f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) { 2632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This could be optimized to avoid all the copying. 2633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = Quotient; 26355ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman F.DeleteBaseReg(F.BaseRegs[i]); 2636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2637572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2641572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateTruncates - Generate reuse formulae from different IV types. 2643ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) { 2644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This requires TargetLowering to tell us which truncates are free. 2645572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI) return; 2646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't bother truncating symbolic values. 2648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 2649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2650572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 2651db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *DstTy = Base.getType(); 2652572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DstTy) return; 2653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DstTy = SE.getEffectiveSCEVType(DstTy); 2654572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2655db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner for (SmallSetVector<Type *, 4>::const_iterator 2656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 2657db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *SrcTy = *I; 2658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SrcTy != DstTy && TLI->isTruncateFree(SrcTy, DstTy)) { 2659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 2660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, *I); 2662572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(), 2663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = F.BaseRegs.end(); J != JE; ++J) 2664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *J = SE.getAnyExtendExpr(*J, SrcTy); 2665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This assumes we've done basic processing on all uses and 2667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // have an idea what the register usage is. 2668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses)) 2669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 2672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2675572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 2677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 26786020d85c41987b0b7890d91bf66187aac6e8a3a1Dan Gohman/// WorkItem - Helper class for GenerateCrossUseConstantOffsets. It's used to 2679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// defer modifications so that the search phase doesn't have to worry about 2680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the data structures moving underneath it. 2681572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct WorkItem { 2682572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 2683572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm; 2684572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg; 2685572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WorkItem(size_t LI, int64_t I, const SCEV *R) 2687572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : LUIdx(LI), Imm(I), OrigReg(R) {} 2688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2689572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 2690572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 2691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 2692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2695572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::print(raw_ostream &OS) const { 2696572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx 2697572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << " , add offset " << Imm; 2698572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2699572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2700572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::dump() const { 2701572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 2702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2703572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2704572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCrossUseConstantOffsets - Look for registers which are a constant 2705572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// distance apart and try to form reuse opportunities between them. 2706572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCrossUseConstantOffsets() { 2707572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Group the registers by their value without any added constant offset. 2708572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef std::map<int64_t, const SCEV *> ImmMapTy; 2709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy; 2710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegMapTy Map; 2711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; 2712572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Sequence; 2713572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 2714572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 2715572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 2716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(Reg, SE); 2717572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegMapTy::iterator, bool> Pair = 2718572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.insert(std::make_pair(Reg, ImmMapTy())); 2719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 2720572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.push_back(Reg); 2721572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pair.first->second.insert(std::make_pair(Imm, *I)); 2722572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(*I); 2723572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now examine each set of registers with the same base value. Build up 2726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // a list of work to do and do the work in a separate step so that we're 2727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // not adding formulae and register counts while we're searching. 2728191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallVector<WorkItem, 32> WorkItems; 2729191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems; 2730572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(), 2731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Sequence.end(); I != E; ++I) { 2732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 2733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const ImmMapTy &Imms = Map.find(Reg)->second; 2734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2735cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman // It's not worthwhile looking for reuse if there's only one offset. 2736cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (Imms.size() == 1) 2737cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman continue; 2738cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman 2739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':'; 2740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 2741572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 2742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << J->first; 2743572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 2744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2745572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Examine each offset. 2746572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 2747572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) { 2748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = J->second; 2749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t JImm = J->first; 2751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg); 2752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isa<SCEVConstant>(OrigReg) && 2754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg].count() == 1) { 2755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n'); 2756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2757572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2758572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively examine offsets between this orig reg a few selected 2760572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // other orig regs. 2761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator OtherImms[] = { 2762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imms.begin(), prior(Imms.end()), 2763cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Imms.lower_bound((Imms.begin()->first + prior(Imms.end())->first) / 2) 2764572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 2765572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = array_lengthof(OtherImms); i != e; ++i) { 2766572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator M = OtherImms[i]; 2767cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (M == J || M == JE) continue; 2768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compute the difference between the two. 2770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = (uint64_t)JImm - M->first; 2771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (int LUIdx = UsedByIndices.find_first(); LUIdx != -1; 2772191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LUIdx = UsedByIndices.find_next(LUIdx)) 2773572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Make a memo of this use, offset, and register tuple. 2774191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (UniqueItems.insert(std::make_pair(LUIdx, Imm))) 2775191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg)); 2776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2779572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.clear(); 2781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.clear(); 2782572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap.clear(); 2783191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman UniqueItems.clear(); 2784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now iterate through the worklist and add new formulae. 2786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<WorkItem>::const_iterator I = WorkItems.begin(), 2787572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = WorkItems.end(); I != E; ++I) { 2788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const WorkItem &WI = *I; 2789572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = WI.LUIdx; 2790572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2791572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = WI.Imm; 2792572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = WI.OrigReg; 2793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2794db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType()); 2795572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); 2796572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BitWidth = SE.getTypeSizeInBits(IntTy); 2797572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 27983f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // TODO: Use a more targeted data structure. 2799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) { 28009f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const Formula &F = LU.Formulae[L]; 2801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in the scaled register. 2802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg == OrigReg) { 2803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offs = (uint64_t)F.AM.BaseOffs + 2804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imm * (uint64_t)F.AM.Scale; 2805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't create 50 + reg(-50). 2806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.referencesReg(SE.getSCEV( 2807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt::get(IntTy, -(uint64_t)Offs)))) 2808572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 2810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = Offs; 2811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 2812572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 2813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2814572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg); 2815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2816572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new scale is a constant in a register, and adding the constant 2817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // value to the immediate would produce a value closer to zero than the 2818572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // immediate itself, then the formula isn't worthwhile. 2819572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg)) 2820c73b24db5f6226ed44ebc44ce1c25bb357206623Chris Lattner if (C->getValue()->isNegative() != 2821572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (NewF.AM.BaseOffs < 0) && 2822572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (C->getValue()->getValue().abs() * APInt(BitWidth, F.AM.Scale)) 2823e05678132345eb8a632362dbd320ee7d36226e67Dan Gohman .ule(abs64(NewF.AM.BaseOffs))) 2824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // OK, looks good. 2827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 2828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 2829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in a base register. 2830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) { 2831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = F.BaseRegs[N]; 2832572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseReg != OrigReg) 2833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 2835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = (uint64_t)NewF.AM.BaseOffs + Imm; 2836572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 2837cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 2838cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (!TLI || 2839cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman !TLI->isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm)) 2840cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman continue; 2841cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NewF = F; 2842cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NewF.UnfoldedOffset = (uint64_t)NewF.UnfoldedOffset + Imm; 2843cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 2844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg); 2845572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new formula has a constant in a register, and adding the 2847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // constant value to the immediate would produce a value closer to 2848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // zero than the immediate itself, then the formula isn't worthwhile. 2849572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 2850572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J = NewF.BaseRegs.begin(), JE = NewF.BaseRegs.end(); 2851572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 2852572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*J)) 2853360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman if ((C->getValue()->getValue() + NewF.AM.BaseOffs).abs().slt( 2854360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman abs64(NewF.AM.BaseOffs)) && 2855360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman (C->getValue()->getValue() + 2856360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman NewF.AM.BaseOffs).countTrailingZeros() >= 2857360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman CountTrailingZeros_64(NewF.AM.BaseOffs)) 2858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto skip_formula; 2859572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, looks good. 2861572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 2862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 2863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman skip_formula:; 2864572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2865572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2866572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2867572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2868572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2869572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateAllReuseFormulae - Generate formulae for each use. 2871572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2872572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::GenerateAllReuseFormulae() { 2873c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman // This is split into multiple loops so that hasRegsUsedByUsesOtherThan 2874572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // queries are more precise. 2875572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2876572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2878572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae[i]); 2879572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2880572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCombinations(LU, LUIdx, LU.Formulae[i]); 2881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2882572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2884572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2885572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]); 2886572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2887572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]); 2888572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2889572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]); 2890572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2891572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateScales(LU, LUIdx, LU.Formulae[i]); 2892c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman } 2893c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2894c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman LSRUse &LU = Uses[LUIdx]; 2895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 2896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateTruncates(LU, LUIdx, LU.Formulae[i]); 2897572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCrossUseConstantOffsets(); 29003902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman 29013902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman DEBUG(dbgs() << "\n" 29023902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman "After generating reuse formulae:\n"; 29033902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman print_uses(dbgs())); 2904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2905572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2906f63d70f218807d7522e8bbe2d9e887ff3ea87b24Dan Gohman/// If there are multiple formulae with the same set of registers used 2907572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// by other uses, pick the best one and delete the others. 2908572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::FilterOutUndesirableDedicatedRegisters() { 2909fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman DenseSet<const SCEV *> VisitedRegs; 2910fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman SmallPtrSet<const SCEV *, 16> Regs; 2911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 2912c6519f916b5922de81c53547fd21364994195a70Dan Gohman bool ChangedFormulae = false; 2913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 2914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect the best formula for each unique set of shared registers. This 2916572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // is reset for each use. 2917572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<SmallVector<const SCEV *, 2>, size_t, UniquifierDenseMapInfo> 2918572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy; 2919572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy BestFormulae; 2920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2922572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2923ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n'); 2924572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2925b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 2926572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t FIdx = 0, NumForms = LU.Formulae.size(); 2927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FIdx != NumForms; ++FIdx) { 2928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[FIdx]; 2929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> Key; 2931572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(), 2932572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = F.BaseRegs.end(); J != JE; ++J) { 2933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *J; 2934572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx)) 2935572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Key.push_back(Reg); 2936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg && 2938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx)) 2939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Key.push_back(F.ScaledReg); 2940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Unstable sort by host order ok, because this is only used for 2941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // uniquifying. 2942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::sort(Key.begin(), Key.end()); 2943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<BestFormulaeTy::const_iterator, bool> P = 2945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulae.insert(std::make_pair(Key, FIdx)); 2946572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!P.second) { 2947572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &Best = LU.Formulae[P.first->second]; 2948fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman 2949fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Cost CostF; 2950fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman CostF.RateFormula(F, Regs, VisitedRegs, L, LU.Offsets, SE, DT); 2951fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Regs.clear(); 2952fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Cost CostBest; 2953fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman CostBest.RateFormula(Best, Regs, VisitedRegs, L, LU.Offsets, SE, DT); 2954fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Regs.clear(); 2955fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman if (CostF < CostBest) 2956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::swap(F, Best); 29576458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs()); 2958572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 29596458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman " in favor of formula "; Best.print(dbgs()); 2960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 2961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 2962c6519f916b5922de81c53547fd21364994195a70Dan Gohman ChangedFormulae = true; 2963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 2964d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman LU.DeleteFormula(F); 2965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --FIdx; 2966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --NumForms; 2967b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Any = true; 2968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 297059dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman } 297159dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 297257aaa0b264d9d23d9a14235334b15d4117e32a3bDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 2973b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 2974b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 297559dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 297659dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman // Reset this to prepare for the next use. 2977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulae.clear(); 2978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2980c6519f916b5922de81c53547fd21364994195a70Dan Gohman DEBUG(if (ChangedFormulae) { 29819214b82c5446791b280821bbd892dca633130f80Dan Gohman dbgs() << "\n" 29829214b82c5446791b280821bbd892dca633130f80Dan Gohman "After filtering out undesirable candidates:\n"; 2983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs()); 2984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 2985572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2986572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2987d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman// This is a rough guess that seems to work fairly well. 2988d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmanstatic const size_t ComplexityLimit = UINT16_MAX; 2989d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 2990d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// EstimateSearchSpaceComplexity - Estimate the worst-case number of 2991d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// solutions the solver might have to consider. It almost never considers 2992d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// this many solutions because it prune the search space, but the pruning 2993d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// isn't always sufficient. 2994d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmansize_t LSRInstance::EstimateSearchSpaceComplexity() const { 29950d6715a413a23c021d1042719888966bb8d11872Dan Gohman size_t Power = 1; 2996d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 2997d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman E = Uses.end(); I != E; ++I) { 2998d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t FSize = I->Formulae.size(); 2999d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (FSize >= ComplexityLimit) { 3000d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power = ComplexityLimit; 3001d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 3002d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 3003d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power *= FSize; 3004d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (Power >= ComplexityLimit) 3005d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 3006d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 3007d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman return Power; 3008d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman} 3009d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 30104aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByDetectingSupersets - When one formula uses a superset 30114aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of the registers of another formula, it won't help reduce register 30124aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// pressure (though it may not necessarily hurt register pressure); remove 30134aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// it to simplify the system. 30144aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByDetectingSupersets() { 3015a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3016a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3017a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3018a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by eliminating formulae " 3019a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "which use a superset of registers used by other " 3020a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "formulae.\n"); 3021a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3022a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3023a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 3024a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool Any = false; 3025a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 3026a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula &F = LU.Formulae[i]; 3027f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // Look for a formula with a constant or GV in a register. If the use 3028f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // also has a formula with that same value in an immediate field, 3029f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // delete the one that uses a register. 3030a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 3031a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) { 3032a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) { 3033a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 3034a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseOffs += C->getValue()->getSExtValue(); 3035a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 3036a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 3037a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 3038a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 3039a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 3040a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 3041a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 3042a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 3043a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3044a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3045a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) { 3046a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) 3047a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!F.AM.BaseGV) { 3048a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 3049a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseGV = GV; 3050a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 3051a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 3052a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 3053a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 3054a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 3055a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 3056a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 3057a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 3058a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 3059a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3060a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3061a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3062a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3063a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3064a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3065a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (Any) 3066a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.RecomputeRegs(LUIdx, RegUses); 3067a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3068a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3069a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3070a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 3071a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 30724aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 3073a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 30744aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByCollapsingUnrolledCode - When there are many registers 30754aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// for expressions like A, A+1, A+2, etc., allocate a single register for 30764aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// them. 30774aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() { 3078a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3079a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3080a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3081a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming that uses " 3082a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "separated by a constant offset will use the same " 3083a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "registers.\n"); 3084a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3085f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // This is especially useful for unrolled loops. 3086f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman 3087a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3088a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 3089402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 3090402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 3091402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 3092a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.AM.BaseOffs != 0 && F.AM.Scale == 0) { 3093191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU)) { 3094191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(*LUThatHas, F.AM.BaseOffs, 3095a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman /*HasBaseReg=*/false, 3096a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind, LU.AccessTy)) { 3097a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); 3098a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 3099a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3100a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop; 3101a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3102191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Update the relocs to reference the new use. 3103191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman for (SmallVectorImpl<LSRFixup>::iterator I = Fixups.begin(), 3104191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman E = Fixups.end(); I != E; ++I) { 3105191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRFixup &Fixup = *I; 3106191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == LUIdx) { 3107191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUThatHas - &Uses.front(); 3108191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.Offset += F.AM.BaseOffs; 3109dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman // Add the new offset to LUThatHas' offset list. 3110dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (LUThatHas->Offsets.back() != Fixup.Offset) { 3111dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->Offsets.push_back(Fixup.Offset); 3112dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset > LUThatHas->MaxOffset) 3113dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MaxOffset = Fixup.Offset; 3114dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset < LUThatHas->MinOffset) 3115dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MinOffset = Fixup.Offset; 3116dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman } 3117191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman DEBUG(dbgs() << "New fixup has offset " 3118191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman << Fixup.Offset << '\n'); 3119191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3120191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == NumUses-1) 3121191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUIdx; 3122191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3123191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 3124c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman // Delete formulae from the new use which are no longer legal. 3125c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman bool Any = false; 3126c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) { 3127c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Formula &F = LUThatHas->Formulae[i]; 3128c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (!isLegalUse(F.AM, 3129c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->MinOffset, LUThatHas->MaxOffset, 3130c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->Kind, LUThatHas->AccessTy, TLI)) { 3131c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 3132c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman dbgs() << '\n'); 3133c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->DeleteFormula(F); 3134c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --i; 3135c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --e; 3136c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Any = true; 3137c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3138c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3139c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (Any) 3140c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses); 3141c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman 3142a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Delete the old use. 3143c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman DeleteUse(LU, LUIdx); 3144a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --LUIdx; 3145a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --NumUses; 3146a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3147a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3148a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3149a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3150a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3151a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3152a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3153a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3154a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 3155a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 31564aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 3157a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 31583228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick/// NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters - Call 31594f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// FilterOutUndesirableDedicatedRegisters again, if necessary, now that 31604f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// we've done more filtering, as it may be able to find more formulae to 31614f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// eliminate. 31624f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohmanvoid LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){ 31634f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 31644f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 31654f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 31664f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "Narrowing the search space by re-filtering out " 31674f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman "undesirable dedicated registers.\n"); 31684f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 31694f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman FilterOutUndesirableDedicatedRegisters(); 31704f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 31714f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 31724f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman print_uses(dbgs())); 31734f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman } 31744f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman} 31754f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 31764aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByPickingWinnerRegs - Pick a register which seems likely 31774aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// to be profitable, and then in any use which has any reference to that 31784aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// register, delete all formulae which do not reference that register. 31794aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() { 318076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // With all other options exhausted, loop until the system is simple 318176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // enough to handle. 3182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Taken; 3183d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman while (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we have too many of formulae on our hands to conveniently handle. 3185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use a rough heuristic to thin out the list. 31860da751baf72542ab66518e4549e39da5f34216b4Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Pick the register which is used by the most LSRUses, which is likely 3189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to be a good reuse register candidate. 3190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Best = 0; 3191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BestNum = 0; 3192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 3193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 3194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Taken.count(Reg)) 3196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Best) 3198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 3200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Count = RegUses.getUsedByIndices(Reg).count(); 3201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Count > BestNum) { 3202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestNum = Count; 3204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best 32093f46a3abeedba8d517b4182de34c821d752db058Dan Gohman << " will yield profitable reuse.\n"); 3210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Taken.insert(Best); 3211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // In any use with formulae which references this register, delete formulae 3213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which don't reference it. 3214b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3215b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LSRUse &LU = Uses[LUIdx]; 3216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!LU.Regs.count(Best)) continue; 3217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3218b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 3219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 3220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[i]; 3221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.referencesReg(Best)) { 3222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 3223d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman LU.DeleteFormula(F); 3224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --e; 3225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --i; 3226b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Any = true; 322759dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?"); 3228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3231b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 3232b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 3233b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 3234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 3238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 32414aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceUsingHeuristics - If there are an extraordinary number of 32424aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// formulae to choose from, use some rough heuristics to prune down the number 32434aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of formulae. This keeps the main solver from taking an extraordinary amount 32444aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of time in some worst-case scenarios. 32454aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceUsingHeuristics() { 32464aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByDetectingSupersets(); 32474aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByCollapsingUnrolledCode(); 32484f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 32494aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByPickingWinnerRegs(); 32504aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 32514aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman 3252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// SolveRecurse - This is the recursive solver. 3253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 3254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 3255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 3256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 3257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 3258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const { 3259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Some ideas: 3260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - prune more: 3261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - use more aggressive filtering 3262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the formula so that the most profitable solutions are found first 3263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the uses too 3264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - search faster: 32653f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // - don't compute a cost, and then compare. compare while computing a cost 3266572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // and bail early. 3267572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - track register sets with SmallBitVector 3268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = Uses[Workspace.size()]; 3270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this use references any register that's already a part of the 3272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // in-progress solution, consider it a requirement that a formula must 3273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reference that register in order to be considered. This prunes out 3274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // unprofitable searching. 3275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> ReqRegs; 3276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator I = CurRegs.begin(), 3277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = CurRegs.end(); I != E; ++I) 32789214b82c5446791b280821bbd892dca633130f80Dan Gohman if (LU.Regs.count(*I)) 3279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ReqRegs.insert(*I); 3280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 32819214b82c5446791b280821bbd892dca633130f80Dan Gohman bool AnySatisfiedReqRegs = false; 3282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> NewRegs; 3283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost NewCost; 32849214b82c5446791b280821bbd892dca633130f80Dan Gohmanretry: 3285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 3286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 3287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F = *I; 3288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore formulae which do not use any of the required registers. 3290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(), 3291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = ReqRegs.end(); J != JE; ++J) { 3292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *J; 3293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((!F.ScaledReg || F.ScaledReg != Reg) && 3294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(F.BaseRegs.begin(), F.BaseRegs.end(), Reg) == 3295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.end()) 3296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto skip; 3297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 32989214b82c5446791b280821bbd892dca633130f80Dan Gohman AnySatisfiedReqRegs = true; 3299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Evaluate the cost of the current formula. If it's already worse than 3301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the current best, prune the search at that point. 3302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost = CurCost; 3303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs = CurRegs; 3304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost.RateFormula(F, NewRegs, VisitedRegs, L, LU.Offsets, SE, DT); 3305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NewCost < SolutionCost) { 3306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.push_back(&F); 3307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Workspace.size() != Uses.size()) { 3308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, NewCost, 3309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs, VisitedRegs); 3310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.getNumRegs() == 1 && Workspace.size() == 1) 3311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]); 3312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "New best at "; NewCost.print(dbgs()); 3314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ". Regs:"; 3315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator 3316572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = NewRegs.begin(), E = NewRegs.end(); I != E; ++I) 3317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << **I; 3318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 3319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost = NewCost; 3321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution = Workspace; 3322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.pop_back(); 3324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman skip:; 3326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 33279214b82c5446791b280821bbd892dca633130f80Dan Gohman 332880ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (!EnableRetry && !AnySatisfiedReqRegs) 332980ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 333080ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 33319214b82c5446791b280821bbd892dca633130f80Dan Gohman // If none of the formulae had all of the required registers, relax the 33329214b82c5446791b280821bbd892dca633130f80Dan Gohman // constraint so that we don't exclude all formulae. 33339214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!AnySatisfiedReqRegs) { 333459dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman assert(!ReqRegs.empty() && "Solver failed even without required registers"); 33359214b82c5446791b280821bbd892dca633130f80Dan Gohman ReqRegs.clear(); 33369214b82c5446791b280821bbd892dca633130f80Dan Gohman goto retry; 33379214b82c5446791b280821bbd892dca633130f80Dan Gohman } 3338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 334076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solve - Choose one formula from each use. Return the results in the given 334176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solution vector. 3342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const { 3343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Workspace; 3344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost SolutionCost; 3345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost.Loose(); 3346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost CurCost; 3347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> CurRegs; 3348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> VisitedRegs; 3349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.reserve(Uses.size()); 3350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3351f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // SolveRecurse does all the work. 3352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, CurCost, 3353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CurRegs, VisitedRegs); 335480ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (Solution.empty()) { 335580ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick DEBUG(dbgs() << "\nNo Satisfactory Solution\n"); 335680ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 335780ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick } 3358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we've now made all our decisions. 3360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\n" 3361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The chosen solution requires "; SolutionCost.print(dbgs()); 3362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"; 3363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Uses.size(); i != e; ++i) { 3364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 3365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses[i].print(dbgs()); 3366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 3367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman " "; 3368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution[i]->print(dbgs()); 3369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'; 3370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 3371a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman 3372a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman assert(Solution.size() == Uses.size() && "Malformed solution!"); 3373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3375e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// HoistInsertPosition - Helper for AdjustInsertPositionForExpand. Climb up 3376e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// the dominator tree far as we can go while still being dominated by the 3377e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// input positions. This helps canonicalize the insert position, which 3378e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// encourages sharing. 3379e5f76877aee6f33964de105893f0ef338661ecadDan GohmanBasicBlock::iterator 3380e5f76877aee6f33964de105893f0ef338661ecadDan GohmanLSRInstance::HoistInsertPosition(BasicBlock::iterator IP, 3381e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) 3382e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const { 3383e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (;;) { 3384e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IPLoop = LI.getLoopFor(IP->getParent()); 3385e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0; 3386e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3387e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock *IDom; 3388d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) { 33890fe46d9b480ab4851e1fc8bc589d1ed9c8b2a70eDan Gohman if (!Rung) return IP; 3390d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman Rung = Rung->getIDom(); 3391d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman if (!Rung) return IP; 3392d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman IDom = Rung->getBlock(); 3393e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3394e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Don't climb into a loop though. 3395e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IDomLoop = LI.getLoopFor(IDom); 3396e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0; 3397e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDomDepth <= IPLoopDepth && 3398e5f76877aee6f33964de105893f0ef338661ecadDan Gohman (IDomDepth != IPLoopDepth || IDomLoop == IPLoop)) 3399e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 3400e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3401e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3402e5f76877aee6f33964de105893f0ef338661ecadDan Gohman bool AllDominate = true; 3403e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *BetterPos = 0; 3404e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Tentative = IDom->getTerminator(); 3405e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(), 3406e5f76877aee6f33964de105893f0ef338661ecadDan Gohman E = Inputs.end(); I != E; ++I) { 3407e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Inst = *I; 3408e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (Inst == Tentative || !DT.dominates(Inst, Tentative)) { 3409e5f76877aee6f33964de105893f0ef338661ecadDan Gohman AllDominate = false; 3410e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 3411e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3412e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Attempt to find an insert position in the middle of the block, 3413e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // instead of at the end, so that it can be used for other expansions. 3414e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDom == Inst->getParent() && 3415e5f76877aee6f33964de105893f0ef338661ecadDan Gohman (!BetterPos || DT.dominates(BetterPos, Inst))) 34167d9663c70b3300070298d716dba6e6f6ce2d1e3eDouglas Gregor BetterPos = llvm::next(BasicBlock::iterator(Inst)); 3417e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3418e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (!AllDominate) 3419e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 3420e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (BetterPos) 3421e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = BetterPos; 3422e5f76877aee6f33964de105893f0ef338661ecadDan Gohman else 3423e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = Tentative; 3424e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 3425e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3426e5f76877aee6f33964de105893f0ef338661ecadDan Gohman return IP; 3427e5f76877aee6f33964de105893f0ef338661ecadDan Gohman} 3428e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 3429e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// AdjustInsertPositionForExpand - Determine an input position which will be 3430d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman/// dominated by the operands and which will dominate the result. 3431d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanBasicBlock::iterator 3432e5f76877aee6f33964de105893f0ef338661ecadDan GohmanLSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator IP, 3433e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRFixup &LF, 3434e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRUse &LU) const { 3435d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Collect some instructions which must be dominated by the 3436448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // expanding replacement. These must be dominated by any operands that 3437572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // will be required in the expansion. 3438572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Instruction *, 4> Inputs; 3439572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace)) 3440572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 3441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) 3442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = 3443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1))) 3444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 3445448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.PostIncLoops.count(L)) { 3446448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.isUseFullyOutsideLoop(L)) 3447069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(L->getLoopLatch()->getTerminator()); 3448069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman else 3449069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(IVIncInsertPos); 3450069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman } 3451701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // The expansion must also be dominated by the increment positions of any 3452701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // loops it for which it is using post-inc mode. 3453701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (PostIncLoopSet::const_iterator I = LF.PostIncLoops.begin(), 3454701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman E = LF.PostIncLoops.end(); I != E; ++I) { 3455701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman const Loop *PIL = *I; 3456701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (PIL == L) continue; 3457701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman 3458e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Be dominated by the loop exit. 3459701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman SmallVector<BasicBlock *, 4> ExitingBlocks; 3460701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman PIL->getExitingBlocks(ExitingBlocks); 3461701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (!ExitingBlocks.empty()) { 3462701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BasicBlock *BB = ExitingBlocks[0]; 3463701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i) 3464701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]); 3465701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman Inputs.push_back(BB->getTerminator()); 3466701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 3467701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 3468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Then, climb up the immediate dominator tree as far as we can go while 3470572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // still being dominated by the input positions. 3471e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = HoistInsertPosition(IP, Inputs); 3472d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3473d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Don't insert instructions before PHI nodes. 3474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (isa<PHINode>(IP)) ++IP; 3475d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3476a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling // Ignore landingpad instructions. 3477a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling while (isa<LandingPadInst>(IP)) ++IP; 3478a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling 3479d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Ignore debug intrinsics. 3480449f31cb9dbf4762935b63946e8120dbe98808ffDan Gohman while (isa<DbgInfoIntrinsic>(IP)) ++IP; 3481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3482d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman return IP; 3483d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman} 3484d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 348576c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Expand - Emit instructions for the leading candidate expression for this 348676c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// LSRUse (this is called "expanding"). 3487d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanValue *LSRInstance::Expand(const LSRFixup &LF, 3488d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const Formula &F, 3489d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman BasicBlock::iterator IP, 3490d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SCEVExpander &Rewriter, 3491d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const { 3492d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const LSRUse &LU = Uses[LF.LUIdx]; 3493d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3494d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Determine an input position which will be dominated by the operands and 3495d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // which will dominate the result. 3496e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = AdjustInsertPositionForExpand(IP, LF, LU); 3497d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 3498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Inform the Rewriter if we have a post-increment use, so that it can 3499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // perform an advantageous expansion. 3500448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.setPostInc(LF.PostIncLoops); 3501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type that the user actually needs. 3503db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 3504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This will be the type that we'll initially expand to. 3505db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *Ty = F.getType(); 3506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Ty) 3507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // No type known; just expand directly to the ultimate type. 3508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 3509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy)) 3510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand directly to the ultimate type if it's the right size. 3511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 3512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type to do integer arithmetic in. 3513db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = SE.getEffectiveSCEVType(Ty); 3514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Build up a list of operands to add together to form the full base. 3516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Ops; 3517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the BaseRegs portion. 3519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 3520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 3521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Reg->isZero() && "Zero allocated in a base register!"); 3523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3524448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 3525448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 3526448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Reg = TransformForPostIncUse(Denormalize, Reg, 3527448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 3528448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 3529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, 0, IP))); 3531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3533087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 3534087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman if (!Ops.empty()) { 3535087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 3536087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 3537087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 3538087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman } 3539087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the ScaledReg portion. 3541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *ICmpScaledV = 0; 3542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale != 0) { 3543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledS = F.ScaledReg; 3544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3545448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 3546448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 3547448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman ScaledS = TransformForPostIncUse(Denormalize, ScaledS, 3548448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 3549448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 3550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3551572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 3552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An interesting way of "folding" with an icmp is to use a negated 3553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scale, which we'll implement by inserting it into the other operand 3554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of the icmp. 3555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == -1 && 3556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The only scale supported by ICmpZero uses is -1!"); 3557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Rewriter.expandCodeFor(ScaledS, 0, IP); 3558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise just expand the scaled register and an explicit scale, 3560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which is expected to be matched as part of the address. 3561572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, 0, IP)); 3562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getMulExpr(ScaledS, 3563deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(ScaledS->getType(), F.AM.Scale)); 3564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(ScaledS); 3565087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3566087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 3567087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 3568087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 3569087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 3570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3573087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the GV portion. 3574087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman if (F.AM.BaseGV) { 3575087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(F.AM.BaseGV)); 3576087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3577087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Flush the operand list to suppress SCEVExpander hoisting. 3578087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 3579087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 3580087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 3581087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman } 3582087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 3583087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the immediate portion. 3584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)F.AM.BaseOffs + LF.Offset; 3585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) { 3586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 3587572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // The other interesting way of "folding" with an ICmpZero is to use a 3588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // negated immediate. 3589572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!ICmpScaledV) 3590572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = ConstantInt::get(IntTy, -Offset); 3591572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 3592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(ICmpScaledV)); 3593572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = ConstantInt::get(IntTy, Offset); 3594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3596572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Just add the immediate values. These again are expected to be matched 3597572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // as part of the address. 3598087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset))); 3599572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3600572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3601572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3602cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Expand the unfolded offset portion. 3603cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman int64_t UnfoldedOffset = F.UnfoldedOffset; 3604cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (UnfoldedOffset != 0) { 3605cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Just add the immediate values. 3606cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, 3607cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman UnfoldedOffset))); 3608cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 3609cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Emit instructions summing all the operands. 3611572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *FullS = Ops.empty() ? 3612deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(IntTy, 0) : 3613572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getAddExpr(Ops); 3614572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *FullV = Rewriter.expandCodeFor(FullS, Ty, IP); 3615572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We're done expanding now, so reset the rewriter. 3617448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.clearPostInc(); 3618572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An ICmpZero Formula represents an ICmp which we're handling as a 3620572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // comparison against zero. Now that we've expanded an expression for that 3621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // form, update the ICmp's other operand. 3622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 3623572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *CI = cast<ICmpInst>(LF.UserInst); 3624572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(CI->getOperand(1)); 3625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!F.AM.BaseGV && "ICmp does not support folding a global value and " 3626572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 3627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale == -1) { 3628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpScaledV->getType() != OpTy) { 3629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 3630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false, 3631572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 3632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV, OpTy, "tmp", CI); 3633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Cast; 3634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, ICmpScaledV); 3636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3637572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == 0 && 3638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "ICmp does not support folding a global value and " 3639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 3640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy), 3641572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman -(uint64_t)Offset); 3642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getType() != OpTy) 3643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 3644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 3645572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C, OpTy); 3646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, C); 3648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3650572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3651572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return FullV; 3652572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 36543a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// RewriteForPHI - Helper for Rewrite. PHI nodes are special because the use 36553a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// of their operands effectively happens in their predecessor blocks, so the 36563a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// expression may need to be expanded in multiple places. 36573a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohmanvoid LSRInstance::RewriteForPHI(PHINode *PN, 36583a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const LSRFixup &LF, 36593a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 36603a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 36613a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 36623a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const { 36633a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman DenseMap<BasicBlock *, Value *> Inserted; 36643a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 36653a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (PN->getIncomingValue(i) == LF.OperandValToReplace) { 36663a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman BasicBlock *BB = PN->getIncomingBlock(i); 36673a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 36683a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is a critical edge, split the edge so that we do not insert 36693a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // the code on all predecessor/successor paths. We do this unless this 36703a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // is the canonical backedge for this loop, which complicates post-inc 36713a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // users. 36723a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 && 36733ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman !isa<IndirectBrInst>(BB->getTerminator())) { 367489d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling BasicBlock *Parent = PN->getParent(); 367589d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling Loop *PNLoop = LI.getLoopFor(Parent); 367689d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling if (!PNLoop || Parent != PNLoop->getHeader()) { 36773ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Split the critical edge. 36788b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling BasicBlock *NewBB = 0; 36798b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling if (!Parent->isLandingPad()) { 3680f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick NewBB = SplitCriticalEdge(BB, Parent, P, 3681f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick /*MergeIdenticalEdges=*/true, 3682f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick /*DontDeleteUselessPhis=*/true); 36838b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling } else { 36848b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling SmallVector<BasicBlock*, 2> NewBBs; 36858b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling SplitLandingPadPredecessors(Parent, BB, "", "", P, NewBBs); 36868b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling NewBB = NewBBs[0]; 36878b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling } 36883ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 36893ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // If PN is outside of the loop and BB is in the loop, we want to 36903ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // move the block to be immediately before the PHI block, not 36913ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // immediately after BB. 36923ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman if (L->contains(BB) && !L->contains(PN)) 36933ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman NewBB->moveBefore(PN->getParent()); 36943ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 36953ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Splitting the edge can reduce the number of PHI entries we have. 36963ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman e = PN->getNumIncomingValues(); 36973ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman BB = NewBB; 36983ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman i = PN->getBasicBlockIndex(BB); 36993ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman } 37003a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 37013a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 37023a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair = 37033a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Inserted.insert(std::make_pair(BB, static_cast<Value *>(0))); 37043a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (!Pair.second) 37053a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, Pair.first->second); 37063a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman else { 3707454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, BB->getTerminator(), Rewriter, DeadInsts); 37083a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 37093a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 3710db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 37113a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (FullV->getType() != OpTy) 37123a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV = 37133a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, 37143a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman OpTy, false), 37153a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV, LF.OperandValToReplace->getType(), 37163a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman "tmp", BB->getTerminator()); 37173a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 37183a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, FullV); 37193a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pair.first->second = FullV; 37203a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 37213a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 37223a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman} 37233a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 3724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Rewrite - Emit instructions for the leading candidate expression for this 3725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse (this is called "expanding"), and update the UserInst to reference 3726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the newly expanded value. 3727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Rewrite(const LSRFixup &LF, 3728572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 3729572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 3730572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 3731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const { 3732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // First, find an insertion point that dominates UserInst. For PHI nodes, 3733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // find the nearest block which dominates all the relevant uses. 3734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) { 3735454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman RewriteForPHI(PN, LF, F, Rewriter, DeadInsts, P); 3736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3737454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, LF.UserInst, Rewriter, DeadInsts); 3738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 3740db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 3741572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FullV->getType() != OpTy) { 3742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 3743572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false), 3744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV, OpTy, "tmp", LF.UserInst); 3745572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV = Cast; 3746572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3747572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the user. ICmpZero is handled specially here (for now) because 3749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand may have updated one of the operands of the icmp already, and 3750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // its new value may happen to be equal to LF.OperandValToReplace, in 3751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which case doing replaceUsesOfWith leads to replacing both operands 3752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with the same value. TODO: Reorganize this. 3753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Uses[LF.LUIdx].Kind == LSRUse::ICmpZero) 3754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->setOperand(0, FullV); 3755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 3756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV); 3757572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3758572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(LF.OperandValToReplace); 3760572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 376276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// ImplementSolution - Rewrite all the fixup locations with new values, 376376c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// following the chosen solution. 3764572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 3765572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 3766572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) { 3767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Keep track of instructions we may have made dead, so that 3768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // we can remove them after we are done working. 3769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<WeakVH, 16> DeadInsts; 3770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 37715e7645be4c9dd2193add44d30b5fef8036d7a3ceAndrew Trick SCEVExpander Rewriter(SE, "lsr"); 3772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.disableCanonicalMode(); 3773c5701910604cdf65811fabd31d41e38f1d1d4eb1Andrew Trick Rewriter.enableLSRMode(); 3774572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.setIVIncInsertPos(L, IVIncInsertPos); 3775572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the new value definitions and update the users. 3777402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 3778402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Fixups.end(); I != E; ++I) { 3779402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const LSRFixup &Fixup = *I; 3780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3781402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman Rewrite(Fixup, *Solution[Fixup.LUIdx], Rewriter, DeadInsts, P); 3782572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 3784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clean up after ourselves. This must be done before deleting any 3787572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // instructions. 3788572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.clear(); 3789f7912df4cbdb44aeac9ac9907c192dfc1e22646dDan Gohman 3790572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= DeleteTriviallyDeadInstructions(DeadInsts); 3791572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3792010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 3793572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P) 3794572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : IU(P->getAnalysis<IVUsers>()), 3795572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE(P->getAnalysis<ScalarEvolution>()), 3796572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT(P->getAnalysis<DominatorTree>()), 3797e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LI(P->getAnalysis<LoopInfo>()), 3798572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TLI(tli), L(l), Changed(false), IVIncInsertPos(0) { 37995792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 3800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If LoopSimplify form is not available, stay out of trouble. 3801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!L->isLoopSimplifyForm()) return; 3802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If there's no interesting work to be done, bail early. 3804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (IU.empty()) return; 3805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\nLSR on loop "; 3807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(dbgs(), L->getHeader(), /*PrintType=*/false); 3808572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"); 3809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3810402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // First, perform some low-level loop optimizations. 3811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OptimizeShadowIV(); 3812c6519f916b5922de81c53547fd21364994195a70Dan Gohman OptimizeLoopTermCond(); 3813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 381437eb38d3f8531115f17f4e829013ccb513952badAndrew Trick // If loop preparation eliminates all interesting IV users, bail. 381537eb38d3f8531115f17f4e829013ccb513952badAndrew Trick if (IU.empty()) return; 381637eb38d3f8531115f17f4e829013ccb513952badAndrew Trick 38175219f86a0bab11dd6895a31653e371e9871a6734Andrew Trick // Skip nested loops until we can model them better with formulae. 38180c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick if (!EnableNested && !L->empty()) { 38190c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n"); 38205219f86a0bab11dd6895a31653e371e9871a6734Andrew Trick return; 38210c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick } 38220c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick 3823402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // Start collecting data and preparing for the solver. 3824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectInterestingTypesAndFactors(); 3825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectFixupsAndInitialFormulae(); 3826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectLoopInvariantFixupsAndFormulae(); 3827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n"; 3829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 3830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now use the reuse data to generate a bunch of interesting ways 3832572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to formulate the values needed for the uses. 3833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateAllReuseFormulae(); 3834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FilterOutUndesirableDedicatedRegisters(); 3836572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NarrowSearchSpaceUsingHeuristics(); 3837572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3838572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Solution; 3839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solve(Solution); 3840572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Release memory that is no longer needed. 3842572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.clear(); 3843572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 3844572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.clear(); 3845572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 384680ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (Solution.empty()) 384780ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 384880ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 3849572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 3850572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Formulae should be legal. 3851572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 3852572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 3853572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 3854572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 3855572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) 3856572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(isLegalUse(J->AM, LU.MinOffset, LU.MaxOffset, 3857572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI) && 3858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Illegal formula generated!"); 3859572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 3860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 3861010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 3862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now that we've decided what we want, make it so. 3863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImplementSolution(Solution, P); 3864572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3865169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 3866572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_factors_and_types(raw_ostream &OS) const { 3867572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors.empty() && Types.empty()) return; 38681ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 3869572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR has identified the following interesting factors and types: "; 3870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 3871eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 3872572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 3873572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 3874572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 3875572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 3876572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '*' << *I; 3877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 38788b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 3879db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner for (SmallSetVector<Type *, 4>::const_iterator 3880572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 3881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 3882572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 3883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '(' << **I << ')'; 3884572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3885572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3886572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3887c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 3888572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_fixups(raw_ostream &OS) const { 3889572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following fixup sites:\n"; 3890572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 3891572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Fixups.end(); I != E; ++I) { 3892572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 38939f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman I->print(OS); 3894572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3897010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman 3898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_uses(raw_ostream &OS) const { 3899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following uses:\n"; 3900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 3901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 3902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 3903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 3904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.print(OS); 3905572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3906572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 3907572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) { 3908572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << " "; 3909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J->print(OS); 3910572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 3911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 39126bec5bb344fc0374431aed1cb63418de607a1aecDan Gohman } 3913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print(raw_ostream &OS) const { 3916572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_factors_and_types(OS); 3917572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_fixups(OS); 3918572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(OS); 3919572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::dump() const { 3922572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 3923572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3924572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3925572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 3926572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LoopStrengthReduce : public LoopPass { 3928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TLI - Keep a pointer of a TargetLowering to consult for determining 3929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// transformation profitability. 3930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 3931572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3932572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 3933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static char ID; // Pass ID, replacement for typeid 3934572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman explicit LoopStrengthReduce(const TargetLowering *tli = 0); 3935572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 3937572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool runOnLoop(Loop *L, LPPassManager &LPM); 3938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void getAnalysisUsage(AnalysisUsage &AU) const; 3939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 3940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanchar LoopStrengthReduce::ID = 0; 39442ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce", 3945ce665bd2e2b581ab0858d1afe359192bac96b868Owen Anderson "Loop Strength Reduction", false, false) 39462ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(DominatorTree) 39472ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(ScalarEvolution) 39482ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(IVUsers) 3949205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopInfo) 3950205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopSimplify) 39512ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce", 39522ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson "Loop Strength Reduction", false, false) 39532ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson 3954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3955572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanPass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) { 3956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return new LoopStrengthReduce(TLI); 3957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3958572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3959572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLoopStrengthReduce::LoopStrengthReduce(const TargetLowering *tli) 3960081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson : LoopPass(ID), TLI(tli) { 3961081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry()); 3962081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson } 3963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const { 3965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We split critical edges, so we change the CFG. However, we do update 3966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // many analyses if they are around. 39676793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreservedID(LoopSimplifyID); 3968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 39696793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequired<LoopInfo>(); 39706793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreserved<LoopInfo>(); 39716793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequiredID(LoopSimplifyID); 3972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<DominatorTree>(); 3973572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<DominatorTree>(); 3974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<ScalarEvolution>(); 3975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<ScalarEvolution>(); 39762c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // Requiring LoopSimplify a second time here prevents IVUsers from running 39772c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // twice, since LoopSimplify was invalidated by running ScalarEvolution. 39782c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich AU.addRequiredID(LoopSimplifyID); 3979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<IVUsers>(); 3980572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<IVUsers>(); 3981572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) { 3984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 3985572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3986572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Run the main LSR transformation. 3987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= LSRInstance(TLI, L, this).getChanged(); 3988eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 3989afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman // At this point, it is worth checking to see if any recurrence PHIs are also 399035738ac150afafe2359268d4b2169498c6c98c5fDan Gohman // dead, so that we can remove them as well. 39919fff2187a21f765ed87a25c48552a6942450f3e2Dan Gohman Changed |= DeleteDeadPHIs(L->getHeader()); 3992afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman 39931ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng return Changed; 3994eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 3995