12d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// The LLVM Compiler Infrastructure 4eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 9eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 10cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// This transformation analyzes and transforms the induction variables (and 11cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// computations derived from them) into forms suitable for efficient execution 12cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// on the target. 13cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// 14eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// This pass performs a strength reduction on array references inside loops that 15cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// have as one or more of their components the loop induction variable, it 16cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// rewrites expressions to take advantage of scaled-index addressing modes 17cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// available on the target, and it performs a variety of other optimizations 18cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// related to loop induction variables. 19eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 20572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// Terminology note: this code has a lot of handling for "post-increment" or 21572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// "post-inc" users. This is not talking about post-increment addressing modes; 22572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it is instead talking about code like this: 23572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 24572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i = phi [ 0, %entry ], [ %i.next, %latch ] 25572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// ... 26572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %i.next = add %i, 1 27572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// %c = icmp eq %i.next, %n 28572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 29572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however 30572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// it's useful to think about these as the same register, with some uses using 31572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the register before the add and some using // it after. In this 32572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// example, the icmp is a post-increment user, since it uses %i.next, which is 33572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// the value of the induction variable after the increment. The other common 34572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// case of post-increment users is users outside the loop. 35572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 36572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: More sophistication in the way Formulae are generated and filtered. 37572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 38572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Handle multiple loops at a time. 39572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 40572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: Should TargetLowering::AddrMode::BaseGV be changed to a ConstantExpr 41572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// instead of a GlobalValue? 42572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 43572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When truncation is free, truncate ICmp users' operands to make it a 44572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// smaller encoding (on x86 at least). 45572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 46572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// TODO: When a negated register is used by an add (such as in a list of 47572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// multiple base registers, or as the increment expression in an addrec), 48572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// we may not actually need both reg and (-1 * reg) in registers; the 49572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// negation can be implemented by using a sub instead of an add. The 50572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// lack of support for taking this into consideration when making 51572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// register pressure decisions is partly worked around by the "Special" 52572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// use kind. 53572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman// 54eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 55eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 56be3e5212e23edc9210f774fac18d801de252e906Chris Lattner#define DEBUG_TYPE "loop-reduce" 57eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Scalar.h" 58eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Constants.h" 59eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Instructions.h" 60e5b01bea7b9b7dce7c24484d2d915b0c118d4d07Dan Gohman#include "llvm/IntrinsicInst.h" 612f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/DerivedTypes.h" 6281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman#include "llvm/Analysis/IVUsers.h" 63572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/Analysis/Dominators.h" 640f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel#include "llvm/Analysis/LoopPass.h" 65169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Analysis/ScalarEvolutionExpander.h" 669fc5cdf77c812aaa80419036de27576d45894d0dChris Lattner#include "llvm/Assembly/Writer.h" 67e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner#include "llvm/Transforms/Utils/BasicBlockUtils.h" 68eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Utils/Local.h" 69572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SmallBitVector.h" 70572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/SetVector.h" 71572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#include "llvm/ADT/DenseSet.h" 72169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/Debug.h" 7380ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick#include "llvm/Support/CommandLine.h" 74afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman#include "llvm/Support/ValueHandle.h" 75460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar#include "llvm/Support/raw_ostream.h" 76d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng#include "llvm/Target/TargetLowering.h" 77cfb1d4235fe3291028341e6acf4139723b4749e3Jeff Cohen#include <algorithm> 78eaa13851a7fe604363577350c5cf65c257c4d41aNate Begemanusing namespace llvm; 79eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 80b5122635966a980a850c028895e275a43e0f946dAndrew Trick/// MaxIVUsers is an arbitrary threshold that provides an early opportunitiy for 81b5122635966a980a850c028895e275a43e0f946dAndrew Trick/// bail out. This threshold is far beyond the number of users that LSR can 82b5122635966a980a850c028895e275a43e0f946dAndrew Trick/// conceivably solve, so it should not affect generated code, but catches the 83b5122635966a980a850c028895e275a43e0f946dAndrew Trick/// worst cases before LSR burns too much compile time and stack space. 84b5122635966a980a850c028895e275a43e0f946dAndrew Trickstatic const unsigned MaxIVUsers = 200; 85b5122635966a980a850c028895e275a43e0f946dAndrew Trick 86a02bfced06b4cc700e50bc497cc42667653f091aAndrew Trick// Temporary flag to cleanup congruent phis after LSR phi expansion. 87a02bfced06b4cc700e50bc497cc42667653f091aAndrew Trick// It's currently disabled until we can determine whether it's truly useful or 88a02bfced06b4cc700e50bc497cc42667653f091aAndrew Trick// not. The flag should be removed after the v3.0 release. 8924f670f1bae0281b08276229041dfc952a810dbfAndrew Trick// This is now needed for ivchains. 900861f5793a1834f02b522fb86fb037cd592c134fBenjamin Kramerstatic cl::opt<bool> EnablePhiElim( 9124f670f1bae0281b08276229041dfc952a810dbfAndrew Trick "enable-lsr-phielim", cl::Hidden, cl::init(true), 9224f670f1bae0281b08276229041dfc952a810dbfAndrew Trick cl::desc("Enable LSR phi elimination")); 9380ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 9422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick#ifndef NDEBUG 9522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick// Stress test IV chain generation. 9622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic cl::opt<bool> StressIVChain( 9722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick "stress-ivchain", cl::Hidden, cl::init(false), 9822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick cl::desc("Stress test LSR IV chains")); 9922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick#else 10022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic bool StressIVChain = false; 10122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick#endif 10222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegSortData - This class holds data which is used to order reuse candidates. 106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegSortData { 107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UsedByIndices - This represents the set of LSRUse indices which reference 109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// a particular register. 110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallBitVector UsedByIndices; 111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData() {} 113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::print(raw_ostream &OS) const { 121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "[NumUses=" << UsedByIndices.count() << ']'; 122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 123dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 124cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#ifndef NDEBUG 125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegSortData::dump() const { 126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 128cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#endif 129dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 1307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmannamespace { 1317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RegUseTracker - Map register candidates to information about how they are 133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// used. 134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass RegUseTracker { 135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, RegSortData> RegUsesTy; 136d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 13790bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy RegUsesMap; 138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 16> RegSequence; 139d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegister(const SCEV *Reg, size_t LUIdx); 142b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void DropRegister(const SCEV *Reg, size_t LUIdx); 143c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void SwapAndDropUse(size_t LUIdx, size_t LastLUIdx); 144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const; 146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 147572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &getUsedByIndices(const SCEV *Reg) const; 148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void clear(); 150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::iterator iterator; 152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator; 153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator begin() { return RegSequence.begin(); } 154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman iterator end() { return RegSequence.end(); } 155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator begin() const { return RegSequence.begin(); } 156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const_iterator end() const { return RegSequence.end(); } 157572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 160572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 162572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) { 163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegUsesTy::iterator, bool> Pair = 16490bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.insert(std::make_pair(Reg, RegSortData())); 165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSortData &RSD = Pair.first->second; 166572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 167572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.push_back(Reg); 168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1)); 169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RSD.UsedByIndices.set(LUIdx); 170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 172b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid 173b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan GohmanRegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) { 174b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUsesTy::iterator It = RegUsesMap.find(Reg); 175b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(It != RegUsesMap.end()); 176b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegSortData &RSD = It->second; 177b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman assert(RSD.UsedByIndices.size() > LUIdx); 178b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RSD.UsedByIndices.reset(LUIdx); 179b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 180b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 181a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanvoid 182c6897706b7c3796ac24535c9ea1449c0411f8c7aDan GohmanRegUseTracker::SwapAndDropUse(size_t LUIdx, size_t LastLUIdx) { 183c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman assert(LUIdx <= LastLUIdx); 184c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 185c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. The data structure is not optimized for this purpose; 186c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // we must iterate through it and update each of the bit vectors. 187a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (RegUsesTy::iterator I = RegUsesMap.begin(), E = RegUsesMap.end(); 188c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman I != E; ++I) { 189c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman SmallBitVector &UsedByIndices = I->second.UsedByIndices; 190c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman if (LUIdx < UsedByIndices.size()) 191c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices[LUIdx] = 192c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : 0; 193c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx)); 194c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman } 195a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 196a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 198572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanRegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { 19946fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 20046fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman if (I == RegUsesMap.end()) 20146fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman return false; 20246fd7a6ef8766d5d1b5816e9f2ceff51d5ceb006Dan Gohman const SmallBitVector &UsedByIndices = I->second.UsedByIndices; 203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int i = UsedByIndices.find_first(); 204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (i == -1) return false; 205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((size_t)i != LUIdx) return true; 206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return UsedByIndices.find_next(i) != -1; 207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanconst SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { 21090bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesTy::const_iterator I = RegUsesMap.find(Reg); 21190bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman assert(I != RegUsesMap.end() && "Unknown register!"); 212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return I->second.UsedByIndices; 213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid RegUseTracker::clear() { 21690bb355b162ec5a640554ee1dc3dda5ce9e50c50Dan Gohman RegUsesMap.clear(); 217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegSequence.clear(); 218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Formula - This class holds information that describes a formula for 223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// computing satisfying a use. It may include broken-out immediates and scaled 224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// registers. 225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct Formula { 226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AM - This is used to represent complex addressing, as well as other kinds 227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// of interesting uses. 228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// BaseRegs - The list of "base" registers for this use. When this is 231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// non-empty, AM.HasBaseReg should be set to true. 232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> BaseRegs; 233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// ScaledReg - The 'scaled' register for this use. This should be non-null 235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// when AM.Scale is not zero. 236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledReg; 237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 238cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// UnfoldedOffset - An additional constant offset which added near the 239cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// use. This requires a temporary register, but the offset itself can 240cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman /// live in an add immediate field rather than a register. 241cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman int64_t UnfoldedOffset; 242cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 243cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Formula() : ScaledReg(0), UnfoldedOffset(0) {} 244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 245dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE); 246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned getNumRegs() const; 248db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *getType() const; 249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2505ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman void DeleteBaseReg(const SCEV *&S); 2515ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool referencesReg(const SCEV *S) const; 253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool hasRegsUsedByUsesOtherThan(size_t LUIdx, 254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const; 255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2623f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// DoInitialMatch - Recursion helper for InitialMatch. 263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic void DoInitialMatch(const SCEV *S, Loop *L, 264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Good, 265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const SCEV *> &Bad, 266dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman ScalarEvolution &SE) { 267572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect expressions which properly dominate the loop header. 268dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(S, L->getHeader())) { 269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(S); 270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 272d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at add operands. 274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) 277dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(*I, L, Good, Bad, SE); 278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 280169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Look at addrec operands. 282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) 283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!AR->getStart()->isZero()) { 284dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(AR->getStart(), L, Good, Bad, SE); 285deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), 286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE), 2873228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags() 2883228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick AR->getLoop(), SCEV::FlagAnyWrap), 289dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman L, Good, Bad, SE); 290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 2917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 2922d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle a multiplication by -1 (negation) if it didn't fold. 294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) 295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mul->getOperand(0)->isAllOnesValue()) { 296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end()); 297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewMul = SE.getMulExpr(Ops); 298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyGood; 300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> MyBad; 301dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(NewMul, L, MyGood, MyBad, SE); 302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( 303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getEffectiveSCEVType(NewMul->getType()))); 304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(), 305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyGood.end(); I != E; ++I) 306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Good.push_back(SE.getMulExpr(NegOne, *I)); 307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(), 308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = MyBad.end(); I != E; ++I) 309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(SE.getMulExpr(NegOne, *I)); 310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 3117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 312eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we can't do anything interesting. Just stuff the whole thing into a 314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // register and hope for the best. 315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Bad.push_back(S); 316a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 317844731a7f1909f55935e3514c9e713a62d67662eDan Gohman 318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InitialMatch - Incorporate loop-variant parts of S into this Formula, 319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// attempting to keep all loop-invariant and loop-computable values in a 320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// single base register. 321dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohmanvoid Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { 322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Good; 323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Bad; 324dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman DoInitialMatch(S, L, Good, Bad, SE); 325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Good.empty()) { 326e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Good); 327e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 328e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 330572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Bad.empty()) { 332e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman const SCEV *Sum = SE.getAddExpr(Bad); 333e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman if (!Sum->isZero()) 334e60bb15982dc40517e078c3858fdc0716d43abb5Dan Gohman BaseRegs.push_back(Sum); 335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = true; 336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 338eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getNumRegs - Return the total number of register operands used by this 340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// formula. This does not include register uses implied by non-constant 341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrec strides. 342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanunsigned Formula::getNumRegs() const { 343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !!ScaledReg + BaseRegs.size(); 344a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 34556a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getType - Return the type of this formula, if it has one, or null 347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// otherwise. This type is meaningless except for the bit size. 348db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris LattnerType *Formula::getType() const { 349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !BaseRegs.empty() ? BaseRegs.front()->getType() : 350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledReg ? ScaledReg->getType() : 351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV ? AM.BaseGV->getType() : 352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 0; 353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 35456a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 3555ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteBaseReg - Delete the given base reg from the BaseRegs list. 3565ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohmanvoid Formula::DeleteBaseReg(const SCEV *&S) { 3575ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&S != &BaseRegs.back()) 3585ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(S, BaseRegs.back()); 3595ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman BaseRegs.pop_back(); 3605ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 3615ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// referencesReg - Test if this formula references the given register. 363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::referencesReg(const SCEV *S) const { 364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return S == ScaledReg || 365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(BaseRegs.begin(), BaseRegs.end(), S) != BaseRegs.end(); 366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 367eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// hasRegsUsedByUsesOtherThan - Test whether this formula uses registers 369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// which are used by uses other than the use with the given index. 370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx, 371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const RegUseTracker &RegUses) const { 372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx)) 374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) 377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (RegUses.isRegUsedByUsesOtherThan(*I, LUIdx)) 378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::print(raw_ostream &OS) const { 383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 384572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) { 385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, AM.BaseGV, /*PrintType=*/false); 387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.BaseOffs; 391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), 393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = BaseRegs.end(); I != E; ++I) { 394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "reg(" << **I << ')'; 396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 397c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (AM.HasBaseReg && BaseRegs.empty()) { 398c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 399c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: HasBaseReg**"; 400c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } else if (!AM.HasBaseReg && !BaseRegs.empty()) { 401c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman if (!First) OS << " + "; else First = false; 402c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman OS << "**error: !HasBaseReg**"; 403c4cfbaf2174bc0b0aef080b183b048c9f05c4725Dan Gohman } 404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0) { 405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << " + "; else First = false; 406572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << AM.Scale << "*reg("; 407572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ScaledReg) 408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *ScaledReg; 409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "<unknown>"; 411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ')'; 412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 413cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (UnfoldedOffset != 0) { 414cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (!First) OS << " + "; else First = false; 415cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman OS << "imm(" << UnfoldedOffset << ')'; 416cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 419cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#ifndef NDEBUG 420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Formula::dump() const { 421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 422572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 423cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#endif 424572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 425aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddRecSExtable - Return true if the given addrec can be sign-extended 426aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 427aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { 428db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 429ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1); 430aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy)); 431aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 432aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 433aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// isAddSExtable - Return true if the given add can be sign-extended 434aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 435aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohmanstatic bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) { 436db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 437ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1); 438aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy)); 439aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 440aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 441473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman/// isMulSExtable - Return true if the given mul can be sign-extended 442aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman/// without changing its value. 443473e63512acb3751da7dffbb10e0452cb581b265Dan Gohmanstatic bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) { 444db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WideTy = 445473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman IntegerType::get(SE.getContext(), 446473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman SE.getTypeSizeInBits(M->getType()) * M->getNumOperands()); 447473e63512acb3751da7dffbb10e0452cb581b265Dan Gohman return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy)); 448aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman} 449aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman 450f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// getExactSDiv - Return an expression for LHS /s RHS, if it can be determined 451f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// and if the remainder is known to be zero, or null otherwise. If 452f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified 453f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// to Y, ignoring that the multiplication may overflow, which is useful when 454f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// the result will be used in a context where the most significant bits are 455f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman/// ignored. 456f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohmanstatic const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, 457f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman ScalarEvolution &SE, 458f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman bool IgnoreSignificantBits = false) { 459572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Handle the trivial case, which works for any SCEV type. 460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LHS == RHS) 461deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman return SE.getConstant(LHS->getType(), 1); 462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 463d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle a few RHS special cases. 464d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS); 465d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RC) { 466d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 467d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do 468d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // some folding. 469d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA.isAllOnesValue()) 470d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getMulExpr(LHS, RC); 471d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman // Handle x /s 1 as x. 472d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (RA == 1) 473d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return LHS; 474d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman } 475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a division of a constant by a constant. 477572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) { 478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!RC) 479572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 480d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &LA = C->getValue()->getValue(); 481d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman const APInt &RA = RC->getValue()->getValue(); 482d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman if (LA.srem(RA) != 0) 483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 484d42819a07b052e68c6967c10c35a81eb808c5018Dan Gohman return SE.getConstant(LA.sdiv(RA)); 485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 486572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 487aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over addrec operands, if the addrec doesn't overflow. 488572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) { 489aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddRecSExtable(AR, SE)) { 490f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, 491f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 492aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Step) return 0; 493694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, 494694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman IgnoreSignificantBits); 495694a15eabed6af4ba6da14ab4b0b25dceb980d55Dan Gohman if (!Start) return 0; 4963228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FlagNW is independent of the start value, step direction, and is 4973228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // preserved with smaller magnitude steps. 4983228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 4993228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); 500aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 5012ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 504aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman // Distribute the sdiv over add operands, if the add doesn't overflow. 505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) { 506aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isAddSExtable(Add, SE)) { 507aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman SmallVector<const SCEV *, 8> Ops; 508aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 509aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman I != E; ++I) { 510f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman const SCEV *Op = getExactSDiv(*I, RHS, SE, 511f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits); 512aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (!Op) return 0; 513aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman Ops.push_back(Op); 514aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman } 515aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman return SE.getAddExpr(Ops); 516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5172ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a multiply operand that we can pull RHS out of. 5212ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) { 522aae01f17a639f863e73dcadd80ad690fdc37b468Dan Gohman if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) { 523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Found = false; 525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVMulExpr::op_iterator I = Mul->op_begin(), E = Mul->op_end(); 526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 5274766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman const SCEV *S = *I; 528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Found) 5294766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman if (const SCEV *Q = getExactSDiv(S, RHS, SE, 530f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman IgnoreSignificantBits)) { 5314766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman S = Q; 532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Found = true; 533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5344766744072a65558344e6afdd4b42fc196ce47f4Dan Gohman Ops.push_back(S); 535a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Found ? SE.getMulExpr(Ops) : 0; 537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 5382ea09e05466613a22e1211f52c30cd01af563983Dan Gohman return 0; 5392ea09e05466613a22e1211f52c30cd01af563983Dan Gohman } 540a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise we don't know. 542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractImmediate - If S involves the addition of a constant integer value, 546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that integer value, and mutate S to point to a new SCEV with that 547572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { 549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { 550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getValue()->getValue().getMinSignedBits() <= 64) { 551deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(C->getType(), 0); 552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return C->getValue()->getSExtValue(); 553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 557e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 558e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 561572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Result = ExtractImmediate(NewOps.front(), SE); 563e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result != 0) 5643228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5653228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5663228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 56821e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman } 569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// ExtractSymbol - If S involves the addition of a GlobalValue address, 573572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// return that symbol, and mutate S to point to a new SCEV with that 574572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// value excluded. 575572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { 576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) { 578deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman S = SE.getConstant(GV->getType(), 0); 579572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return GV; 580572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); 583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.back(), SE); 584e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 585e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman S = SE.getAddExpr(NewOps); 586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 587572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); 589572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *Result = ExtractSymbol(NewOps.front(), SE); 590e62d58884ad83d0351b9511d58eed4b21ff6e0b1Dan Gohman if (Result) 5913228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick S = SE.getAddRecExpr(NewOps, AR->getLoop(), 5923228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 5933228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick SCEV::FlagAnyWrap); 594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Result; 595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 596572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return 0; 597169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 598169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 5997979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// isAddressUse - Returns true if the specified instruction is using the 6007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// specified value as an address. 6017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohmanstatic bool isAddressUse(Instruction *Inst, Value *OperandVal) { 6027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman bool isAddress = isa<LoadInst>(Inst); 6037979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 6047979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (SI->getOperand(1) == OperandVal) 6057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 6067979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 6077979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 6087979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 6097979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 6107979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 6117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::prefetch: 6127979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 6137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 6147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 6157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 616ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif if (II->getArgOperand(0) == OperandVal) 6177979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman isAddress = true; 6187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 6197979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return isAddress; 622169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 623169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 6247979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// getAccessType - Return the type of the memory being accessed. 625db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattnerstatic Type *getAccessType(const Instruction *Inst) { 626db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = Inst->getType(); 6277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) 6287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AccessTy = SI->getOperand(0)->getType(); 6297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 6307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Addressing modes can also be folded into prefetches and a variety 6317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // of intrinsics. 6327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman switch (II->getIntrinsicID()) { 6337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman default: break; 6347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse_storeu_ps: 6357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_pd: 6367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storeu_dq: 6377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman case Intrinsic::x86_sse2_storel_dq: 638ad72e731366b99aa52d92607d8a4b7a8c26fa632Gabor Greif AccessTy = II->getArgOperand(0)->getType(); 6397979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman break; 6407979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 6417979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // All pointers have the same requirements, so canonicalize them to an 644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // arbitrary pointer type to minimize variation. 645db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner if (PointerType *PTy = dyn_cast<PointerType>(AccessTy)) 646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = PointerType::get(IntegerType::get(PTy->getContext(), 1), 647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PTy->getAddressSpace()); 648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 6497979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return AccessTy; 650a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 65181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 6528a5d792944582de8e63e96440dbd2cde754351adAndrew Trick/// isExistingPhi - Return true if this AddRec is already a phi in its loop. 6538a5d792944582de8e63e96440dbd2cde754351adAndrew Trickstatic bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { 6548a5d792944582de8e63e96440dbd2cde754351adAndrew Trick for (BasicBlock::iterator I = AR->getLoop()->getHeader()->begin(); 6558a5d792944582de8e63e96440dbd2cde754351adAndrew Trick PHINode *PN = dyn_cast<PHINode>(I); ++I) { 6568a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (SE.isSCEVable(PN->getType()) && 6578a5d792944582de8e63e96440dbd2cde754351adAndrew Trick (SE.getEffectiveSCEVType(PN->getType()) == 6588a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SE.getEffectiveSCEVType(AR->getType())) && 6598a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SE.getSCEV(PN) == AR) 6608a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return true; 6618a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 6628a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return false; 6638a5d792944582de8e63e96440dbd2cde754351adAndrew Trick} 6648a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 66564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// Check if expanding this expression is likely to incur significant cost. This 66664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// is tricky because SCEV doesn't track which expressions are actually computed 66764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// by the current IR. 66864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// 66964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// We currently allow expansion of IV increments that involve adds, 67064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// multiplication by constants, and AddRecs from existing phis. 67164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// 67264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// TODO: Allow UDivExpr if we can find an existing IV increment that is an 67364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// obvious multiple of the UDivExpr. 67464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trickstatic bool isHighCostExpansion(const SCEV *S, 67564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick SmallPtrSet<const SCEV*, 8> &Processed, 67664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ScalarEvolution &SE) { 67764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Zero/One operand expressions 67864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick switch (S->getSCEVType()) { 67964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scUnknown: 68064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scConstant: 68164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 68264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scTruncate: 68364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(cast<SCEVTruncateExpr>(S)->getOperand(), 68464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Processed, SE); 68564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scZeroExtend: 68664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(cast<SCEVZeroExtendExpr>(S)->getOperand(), 68764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Processed, SE); 68864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scSignExtend: 68964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(cast<SCEVSignExtendExpr>(S)->getOperand(), 69064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Processed, SE); 69164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 69264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 69364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (!Processed.insert(S)) 69464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 69564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 69664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 69764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 69864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick I != E; ++I) { 69964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isHighCostExpansion(*I, Processed, SE)) 70064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return true; 70164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 70264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 70364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 70464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 70564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 70664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (Mul->getNumOperands() == 2) { 70764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Multiplication by a constant is ok 70864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isa<SCEVConstant>(Mul->getOperand(0))) 70964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return isHighCostExpansion(Mul->getOperand(1), Processed, SE); 71064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 71164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // If we have the value of one operand, check if an existing 71264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // multiplication already generates this expression. 71364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) { 71464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Value *UVal = U->getValue(); 71564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (Value::use_iterator UI = UVal->use_begin(), UE = UVal->use_end(); 71664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick UI != UE; ++UI) { 71705fecbe42e835b30274a7b38af27687a8abbd114Andrew Trick // If U is a constant, it may be used by a ConstantExpr. 71805fecbe42e835b30274a7b38af27687a8abbd114Andrew Trick Instruction *User = dyn_cast<Instruction>(*UI); 71905fecbe42e835b30274a7b38af27687a8abbd114Andrew Trick if (User && User->getOpcode() == Instruction::Mul 72064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick && SE.isSCEVable(User->getType())) { 72164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return SE.getSCEV(User) == Mul; 72264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 72764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 72864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 72964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isExistingPhi(AR, SE)) 73064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 73164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 73264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 73364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Fow now, consider any other type of expression (div/mul/min/max) high cost. 73464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return true; 73564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick} 73664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 737572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DeleteTriviallyDeadInstructions - If any of the instructions is the 738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// specified set are trivially dead, delete them and see if this makes any of 739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// their operands subsequently dead. 740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool 741572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanDeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) { 742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 7432f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!DeadInsts.empty()) { 745875cc5d629697c16fd24662f7cce21766c3d2e4eRichard Smith Value *V = DeadInsts.pop_back_val(); 746875cc5d629697c16fd24662f7cce21766c3d2e4eRichard Smith Instruction *I = dyn_cast_or_null<Instruction>(V); 74781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (I == 0 || !isInstructionTriviallyDead(I)) 749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 750221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) 752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *U = dyn_cast<Instruction>(*OI)) { 753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *OI = 0; 754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (U->use_empty()) 755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(U); 756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 757221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 758572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I->eraseFromParent(); 759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 760572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 7612f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Changed; 7637979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 7642f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 765572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 7662f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Cost - This class is used to measure and compare candidate formulae. 768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass Cost { 769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TODO: Some of these could be merged. Also, a lexical ordering 770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// isn't always optimal. 771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumRegs; 772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned AddRecCost; 773572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumIVMuls; 774572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned NumBaseAdds; 775572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned ImmCost; 776572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned SetupCost; 777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 778572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 779572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost() 780572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : NumRegs(0), AddRecCost(0), NumIVMuls(0), NumBaseAdds(0), ImmCost(0), 781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost(0) {} 782572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool operator<(const Cost &Other) const; 784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Loose(); 786572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 7877d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick#ifndef NDEBUG 7887d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick // Once any of the metrics loses, they must all remain losers. 7897d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick bool isValid() { 7907d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return ((NumRegs | AddRecCost | NumIVMuls | NumBaseAdds 7917d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick | ImmCost | SetupCost) != ~0u) 7927d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick || ((NumRegs & AddRecCost & NumIVMuls & NumBaseAdds 7937d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick & ImmCost & SetupCost) == ~0u); 7947d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 7957d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick#endif 7967d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick 7977d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick bool isLoser() { 7987d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick assert(isValid() && "invalid cost"); 7997d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return NumRegs == ~0u; 8007d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick } 8017d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick 802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateFormula(const Formula &F, 803572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 804572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 8078a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 8088a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs = 0); 809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 812572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 814572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void RateRegister(const SCEV *Reg, 815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 816572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 817572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT); 8189214b82c5446791b280821bbd892dca633130f80Dan Gohman void RatePrimaryRegister(const SCEV *Reg, 8199214b82c5446791b280821bbd892dca633130f80Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 8209214b82c5446791b280821bbd892dca633130f80Dan Gohman const Loop *L, 8218a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 8228a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs); 823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 8240e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 8267979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// RateRegister - Tally up interesting quantities from the given register. 828572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateRegister(const SCEV *Reg, 829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE, DominatorTree &DT) { 8329214b82c5446791b280821bbd892dca633130f80Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) { 8330c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // If this is an addrec for another loop, don't second-guess its addrec phi 8340c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick // nodes. LSR isn't currently smart enough to reason about more than one 835bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick // loop at a time. LSR has already run on inner loops, will not run on outer 836bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick // loops, and cannot be expected to change sibling loops. 837bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick if (AR->getLoop() != L) { 838bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick // If the AddRec exists, consider it's register free and leave it alone. 8398a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (isExistingPhi(AR, SE)) 8408a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return; 8418a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 842bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick // Otherwise, do not consider this formula at all. 843bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick Loose(); 844bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick return; 8459214b82c5446791b280821bbd892dca633130f80Dan Gohman } 846bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick AddRecCost += 1; /// TODO: This should be a function of the stride. 8477979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 8489214b82c5446791b280821bbd892dca633130f80Dan Gohman // Add the step value register, if it needs one. 8499214b82c5446791b280821bbd892dca633130f80Dan Gohman // TODO: The non-affine case isn't precisely modeled here. 85025b689e067697d3b49ae123120703fada030350fAndrew Trick if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) { 85125b689e067697d3b49ae123120703fada030350fAndrew Trick if (!Regs.count(AR->getOperand(1))) { 852572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RateRegister(AR->getOperand(1), Regs, L, SE, DT); 85325b689e067697d3b49ae123120703fada030350fAndrew Trick if (isLoser()) 85425b689e067697d3b49ae123120703fada030350fAndrew Trick return; 85525b689e067697d3b49ae123120703fada030350fAndrew Trick } 85625b689e067697d3b49ae123120703fada030350fAndrew Trick } 857a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 8589214b82c5446791b280821bbd892dca633130f80Dan Gohman ++NumRegs; 8599214b82c5446791b280821bbd892dca633130f80Dan Gohman 8609214b82c5446791b280821bbd892dca633130f80Dan Gohman // Rough heuristic; favor registers which don't require extra setup 8619214b82c5446791b280821bbd892dca633130f80Dan Gohman // instructions in the preheader. 8629214b82c5446791b280821bbd892dca633130f80Dan Gohman if (!isa<SCEVUnknown>(Reg) && 8639214b82c5446791b280821bbd892dca633130f80Dan Gohman !isa<SCEVConstant>(Reg) && 8649214b82c5446791b280821bbd892dca633130f80Dan Gohman !(isa<SCEVAddRecExpr>(Reg) && 8659214b82c5446791b280821bbd892dca633130f80Dan Gohman (isa<SCEVUnknown>(cast<SCEVAddRecExpr>(Reg)->getStart()) || 8669214b82c5446791b280821bbd892dca633130f80Dan Gohman isa<SCEVConstant>(cast<SCEVAddRecExpr>(Reg)->getStart())))) 8679214b82c5446791b280821bbd892dca633130f80Dan Gohman ++SetupCost; 86823c3fde39aa35334e74c26273a8973a872871e34Dan Gohman 86923c3fde39aa35334e74c26273a8973a872871e34Dan Gohman NumIVMuls += isa<SCEVMulExpr>(Reg) && 87017ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman SE.hasComputableLoopEvolution(Reg, L); 8719214b82c5446791b280821bbd892dca633130f80Dan Gohman} 8729214b82c5446791b280821bbd892dca633130f80Dan Gohman 8739214b82c5446791b280821bbd892dca633130f80Dan Gohman/// RatePrimaryRegister - Record this register in the set. If we haven't seen it 8748a5d792944582de8e63e96440dbd2cde754351adAndrew Trick/// before, rate it. Optional LoserRegs provides a way to declare any formula 8758a5d792944582de8e63e96440dbd2cde754351adAndrew Trick/// that refers to one of those regs an instant loser. 8769214b82c5446791b280821bbd892dca633130f80Dan Gohmanvoid Cost::RatePrimaryRegister(const SCEV *Reg, 8777fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 8787fca2294dad873aa7873e338ec3fcc4db49ea074Dan Gohman const Loop *L, 8798a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 8808a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs) { 8818a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (LoserRegs && LoserRegs->count(Reg)) { 8828a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Loose(); 8838a5d792944582de8e63e96440dbd2cde754351adAndrew Trick return; 8848a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 8858a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (Regs.insert(Reg)) { 8869214b82c5446791b280821bbd892dca633130f80Dan Gohman RateRegister(Reg, Regs, L, SE, DT); 8878a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (isLoser()) 8888a5d792944582de8e63e96440dbd2cde754351adAndrew Trick LoserRegs->insert(Reg); 8898a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 8907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 8917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 892572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::RateFormula(const Formula &F, 893572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> &Regs, 894572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const DenseSet<const SCEV *> &VisitedRegs, 895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Loop *L, 896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVectorImpl<int64_t> &Offsets, 8978a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ScalarEvolution &SE, DominatorTree &DT, 8988a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> *LoserRegs) { 899572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the registers. 900572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEV *ScaledReg = F.ScaledReg) { 901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(ScaledReg)) { 902572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 9047979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 9058a5d792944582de8e63e96440dbd2cde754351adAndrew Trick RatePrimaryRegister(ScaledReg, Regs, L, SE, DT, LoserRegs); 9067d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 9077d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 9083821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner } 909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 910572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 912572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (VisitedRegs.count(BaseReg)) { 913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loose(); 914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 9157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 9168a5d792944582de8e63e96440dbd2cde754351adAndrew Trick RatePrimaryRegister(BaseReg, Regs, L, SE, DT, LoserRegs); 9177d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick if (isLoser()) 9187d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick return; 919572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 921cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Determine how many (unfolded) adds we'll need inside the loop. 922cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman size_t NumBaseParts = F.BaseRegs.size() + (F.UnfoldedOffset != 0); 923cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (NumBaseParts > 1) 924cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NumBaseAdds += NumBaseParts - 1; 925572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 926572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Tally up the non-zero immediates. 927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)*I + F.AM.BaseOffs; 930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.BaseGV) 931572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += 64; // Handle symbolic values conservatively. 932572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This should probably be the pointer size. 933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Offset != 0) 934572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost += APInt(64, Offset, true).getMinSignedBits(); 935572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 9367d11bd850f97f499117704cd3e03d6a6cc1adcb3Andrew Trick assert(isValid() && "invalid cost"); 937169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 938169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 9397a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner/// Loose - Set this cost to a losing value. 940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::Loose() { 941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumRegs = ~0u; 942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AddRecCost = ~0u; 943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumIVMuls = ~0u; 944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NumBaseAdds = ~0u; 945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmCost = ~0u; 946572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SetupCost = ~0u; 947572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 94856a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 949572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// operator< - Choose the lower cost. 950572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool Cost::operator<(const Cost &Other) const { 951572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumRegs != Other.NumRegs) 952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumRegs < Other.NumRegs; 953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != Other.AddRecCost) 954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return AddRecCost < Other.AddRecCost; 955572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != Other.NumIVMuls) 956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumIVMuls < Other.NumIVMuls; 957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != Other.NumBaseAdds) 958572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return NumBaseAdds < Other.NumBaseAdds; 959572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != Other.ImmCost) 960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return ImmCost < Other.ImmCost; 961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != Other.SetupCost) 962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return SetupCost < Other.SetupCost; 963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 9657979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::print(raw_ostream &OS) const { 967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << NumRegs << " reg" << (NumRegs == 1 ? "" : "s"); 968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddRecCost != 0) 969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", with addrec cost " << AddRecCost; 970572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumIVMuls != 0) 971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumIVMuls << " IV mul" << (NumIVMuls == 1 ? "" : "s"); 972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NumBaseAdds != 0) 973572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << NumBaseAdds << " base add" 974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << (NumBaseAdds == 1 ? "" : "s"); 975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ImmCost != 0) 976572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << ImmCost << " imm cost"; 977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SetupCost != 0) 978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", plus " << SetupCost << " setup cost"; 979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 98044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 981cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#ifndef NDEBUG 982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid Cost::dump() const { 983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 98444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner} 985cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#endif 98644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 98844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 989572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRFixup - An operand value in an instruction which is to be replaced 990572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// with some equivalent, possibly strength-reduced, replacement. 991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct LSRFixup { 992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// UserInst - The instruction which will be updated. 993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *UserInst; 99456a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// OperandValToReplace - The operand of the instruction which will 996572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// be replaced. The operand may be used more than once; every instance 997572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// will be replaced. 998572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OperandValToReplace; 99956a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1000448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman /// PostIncLoops - If this user is to use the post-incremented value of an 1001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable, this variable is non-null and holds the loop 1002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// associated with the induction variable. 1003448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet PostIncLoops; 100426d91f16464db56283087176a73981048331dd2dChris Lattner 1005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LUIdx - The index of the LSRUse describing the expression which 1006572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// this fixup needs, minus an offset (below). 1007572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 100826d91f16464db56283087176a73981048331dd2dChris Lattner 1009572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Offset - A constant offset to be added to the LSRUse expression. 1010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// This allows multiple fixups to share the same LSRUse with different 1011572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// offsets, for example in an unrolled loop. 1012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset; 1013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1014448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman bool isUseFullyOutsideLoop(const Loop *L) const; 1015448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 1016572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup(); 1017572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1018572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1019572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1020572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1021572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1022572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1023572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1024572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRFixup::LSRFixup() 1025ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman : UserInst(0), OperandValToReplace(0), LUIdx(~size_t(0)), Offset(0) {} 1026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1027448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// isUseFullyOutsideLoop - Test whether this fixup always uses its 1028448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman/// value outside of the given loop. 1029448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohmanbool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const { 1030448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // PHI nodes use their value in their incoming blocks. 1031448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) { 1032448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 1033448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (PN->getIncomingValue(i) == OperandValToReplace && 1034448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman L->contains(PN->getIncomingBlock(i))) 1035448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return false; 1036448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return true; 1037448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 1038448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 1039448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman return !L->contains(UserInst); 1040448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman} 1041448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 1042572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::print(raw_ostream &OS) const { 1043572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "UserInst="; 1044572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Store is common and interesting enough to be worth special-casing. 1045572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) { 1046572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "store "; 1047572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, Store->getOperand(0), /*PrintType=*/false); 1048572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (UserInst->getType()->isVoidTy()) 1049572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << UserInst->getOpcodeName(); 1050572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, UserInst, /*PrintType=*/false); 1052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", OperandValToReplace="; 1054572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(OS, OperandValToReplace, /*PrintType=*/false); 1055572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1056448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman for (PostIncLoopSet::const_iterator I = PostIncLoops.begin(), 1057448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman E = PostIncLoops.end(); I != E; ++I) { 1058572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", PostIncLoop="; 1059448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman WriteAsOperand(OS, (*I)->getHeader(), /*PrintType=*/false); 1060934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1061a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LUIdx != ~size_t(0)) 1063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", LUIdx=" << LUIdx; 1064572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1065572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) 1066572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offset=" << Offset; 10677979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 10687979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1069cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#ifndef NDEBUG 1070572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRFixup::dump() const { 1071572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 10727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 1073cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#endif 10747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1075572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 1076a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1077572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// UniquifierDenseMapInfo - A DenseMapInfo implementation for holding 1078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// DenseMaps and DenseSets of sorted SmallVectors of const SCEV*. 1079572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct UniquifierDenseMapInfo { 1080572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getEmptyKey() { 1081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 1082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-1)); 1083572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 1084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1085934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1086572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static SmallVector<const SCEV *, 2> getTombstoneKey() { 1087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> V; 1088572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman V.push_back(reinterpret_cast<const SCEV *>(-2)); 1089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return V; 1090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static unsigned getHashValue(const SmallVector<const SCEV *, 2> &V) { 1093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Result = 0; 1094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = V.begin(), 1095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = V.end(); I != E; ++I) 1096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Result ^= DenseMapInfo<const SCEV *>::getHashValue(*I); 10977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Result; 1098a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1099a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static bool isEqual(const SmallVector<const SCEV *, 2> &LHS, 1101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallVector<const SCEV *, 2> &RHS) { 1102572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return LHS == RHS; 1103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1105572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse - This class holds the state that LSR keeps for each use in 1107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// IVUsers, as well as uses invented by LSR itself. It includes information 1108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// about what kinds of things can be folded into the user, information about 1109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the user itself, and information about how the use may be satisfied. 1110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Represent multiple users of the same expression in common? 1111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRUse { 1112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<SmallVector<const SCEV *, 2>, UniquifierDenseMapInfo> Uniquifier; 1113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 1115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// KindType - An enum for a kind of use, indicating what types of 1116572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// scaled and immediate operands it might support. 1117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman enum KindType { 1118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Basic, ///< A normal use, with no folding. 1119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Special, ///< A special case of basic, allowing -1 scales. 1120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Address, ///< An address use; folding according to TargetLowering 1121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpZero ///< An equality icmp with both operands folded into one. 1122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Add a generic icmp too? 1123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 11247979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman KindType Kind; 1126db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy; 11277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<int64_t, 8> Offsets; 1129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset; 1130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MaxOffset; 1131203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 1132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// AllFixupsOutsideLoop - This records whether all of the fixups using this 1133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// LSRUse are outside of the loop, in which case some special-case heuristics 1134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// may be used. 1135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool AllFixupsOutsideLoop; 11367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1137a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// WidestFixupType - This records the widest use type for any fixup using 1138a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// this LSRUse. FindUseWithSimilarFormula can't consider uses with different 1139a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// max fixup widths to be equivalent, because the narrower one may be relying 1140a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman /// on the implicit truncation to truncate away bogus bits. 1141db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *WidestFixupType; 1142a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 1143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Formulae - A list of ways to build a value that can satisfy this user. 1144572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// After the list is populated, one of these is selected heuristically and 1145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// used to formulate a replacement for OperandValToReplace in UserInst. 1146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Formula, 12> Formulae; 1147589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 1148572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Regs - The set of register candidates used by all formulae in this LSRUse. 1149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Regs; 1150934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1151db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse(KindType K, Type *T) : Kind(K), AccessTy(T), 1152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MinOffset(INT64_MAX), 1153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman MaxOffset(INT64_MIN), 1154a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman AllFixupsOutsideLoop(true), 1155a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman WidestFixupType(0) {} 115656a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1157a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool HasFormulaWithSameRegs(const Formula &F) const; 1158454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman bool InsertFormula(const Formula &F); 1159d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman void DeleteFormula(Formula &F); 1160b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses); 1161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1164572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 1165572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1166b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman} 1167b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 1168a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// HasFormula - Test whether this use as a formula which has the same 1169a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// registers as the given formula. 1170a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohmanbool LSRUse::HasFormulaWithSameRegs(const Formula &F) const { 1171a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1172a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1173a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1174a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman std::sort(Key.begin(), Key.end()); 1175a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return Uniquifier.count(Key); 1176a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 1177a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 1179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 1180454d26dc43207ec537d843229db6f5e6a302e23dDan Gohmanbool LSRUse::InsertFormula(const Formula &F) { 1181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 2> Key = F.BaseRegs; 1182572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) Key.push_back(F.ScaledReg); 1183572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Unstable sort by host order ok, because this is only used for uniquifying. 1184572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::sort(Key.begin(), Key.end()); 1185572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1186572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Uniquifier.insert(Key).second) 1187572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1188572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1189572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Using a register to hold the value of 0 is not profitable. 1190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert((!F.ScaledReg || !F.ScaledReg->isZero()) && 1191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Zero allocated in a scaled register!"); 1192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 1193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = 1194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) 1195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!(*I)->isZero() && "Zero allocated in a base register!"); 1196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 1197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Add the formula to the list. 1199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formulae.push_back(F); 120056a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record registers now being used by this use. 1202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1207d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman/// DeleteFormula - Remove the given formula from this use's list. 1208d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohmanvoid LSRUse::DeleteFormula(Formula &F) { 12095ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman if (&F != &Formulae.back()) 12105ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(F, Formulae.back()); 1211d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman Formulae.pop_back(); 1212d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman} 1213d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman 1214b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman/// RecomputeRegs - Recompute the Regs field, and update RegUses. 1215b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohmanvoid LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) { 1216b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 1217b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman SmallPtrSet<const SCEV *, 4> OldRegs = Regs; 1218b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.clear(); 1219402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = Formulae.begin(), 1220402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Formulae.end(); I != E; ++I) { 1221402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 1222b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (F.ScaledReg) Regs.insert(F.ScaledReg); 1223b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); 1224b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman } 1225b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1226b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman // Update the RegTracker. 1227b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (SmallPtrSet<const SCEV *, 4>::iterator I = OldRegs.begin(), 1228b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman E = OldRegs.end(); I != E; ++I) 1229b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (!Regs.count(*I)) 1230b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman RegUses.DropRegister(*I, LUIdx); 1231b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman} 1232b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 1233572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::print(raw_ostream &OS) const { 1234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR Use: Kind="; 1235572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1236572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Basic: OS << "Basic"; break; 1237572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Special: OS << "Special"; break; 1238572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case ICmpZero: OS << "ICmpZero"; break; 1239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case Address: 1240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "Address of "; 12411df9859c40492511b8aa4321eb76496005d3b75bDuncan Sands if (AccessTy->isPointerTy()) 1242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "pointer"; // the full pointer type could be really verbose 12437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman else 1244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *AccessTy; 12457979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 12461bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", Offsets={"; 1248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(), 1249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Offsets.end(); I != E; ++I) { 1250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << *I; 1251ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes if (llvm::next(I) != E) 1252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ','; 1253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '}'; 1255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AllFixupsOutsideLoop) 1257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << ", all-fixups-outside-loop"; 1258a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman 1259a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (WidestFixupType) 1260a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman OS << ", widest fixup type: " << *WidestFixupType; 12617979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman} 1262d6b62a572210aff965a55626cf36a68821838844Evan Cheng 1263cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#ifndef NDEBUG 1264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRUse::dump() const { 1265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 1266572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 1267cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#endif 126856a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// isLegalUse - Test whether the use described by AM is "legal", meaning it can 1270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// be completely folded into the user instruction at isel time. This includes 1271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// address-mode folding and special icmp tricks. 1272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(const TargetLowering::AddrMode &AM, 1273db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman switch (Kind) { 1276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Address: 1277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can 1278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // completely fold this address. 1279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) return TLI->isLegalAddressingMode(AM, AccessTy); 1280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise, just guess that reg+reg addressing is legal. 1282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.BaseOffs == 0 && AM.Scale <= 1; 1283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::ICmpZero: 1285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // There's not even a target hook for querying whether it would be legal to 1286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // fold a GV into an ICmp. 1287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseGV) 1288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1289579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner 1290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only has two operands; don't allow more than two non-trivial parts. 1291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.HasBaseReg && AM.BaseOffs != 0) 1292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 12931bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by 1295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // putting the scaled register in the other operand of the icmp. 1296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.Scale != 0 && AM.Scale != -1) 12977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 129881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we have low-level target information, ask the target if it can fold an 1300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // integer immediate on an icmp. 1301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AM.BaseOffs != 0) { 13029243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen if (!TLI) 13039243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen return false; 13049243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen // We have one of: 13059243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen // ICmpZero BaseReg + Offset => ICmp BaseReg, -Offset 13069243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen // ICmpZero -1*ScaleReg + Offset => ICmp ScaleReg, Offset 13079243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen // Offs is the ICmp immediate. 13089243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen int64_t Offs = AM.BaseOffs; 13099243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen if (AM.Scale == 0) 13109243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen Offs = -(uint64_t)Offs; // The cast does the right thing with INT64_MIN. 13119243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen return TLI->isLegalICmpImmediate(Offs); 1312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 13149243c4f7c54b8d0be22a4c9d411b15f462039d06Jakob Stoklund Olesen // ICmpZero BaseReg + -1*ScaleReg => ICmp BaseReg, ScaleReg 13157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return true; 131681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1317572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Basic: 1318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Only handle single-register values. 1319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return !AM.BaseGV && AM.Scale == 0 && AM.BaseOffs == 0; 132081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman case LSRUse::Special: 1322546f2101152f75c8d84115f6a6b2f3c16389b1acAndrew Trick // Special case Basic to handle -1 scales. 1323546f2101152f75c8d84115f6a6b2f3c16389b1acAndrew Trick return !AM.BaseGV && (AM.Scale == 0 || AM.Scale == -1) && AM.BaseOffs == 0; 1324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 1325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 13264d6ccb5f68cd7c6418a209f1fa4dbade569e4493David Blaikie llvm_unreachable("Invalid LSRUse Kind!"); 1327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 132881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1329572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isLegalUse(TargetLowering::AddrMode AM, 1330572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1331db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI) { 1333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MinOffset) > AM.BaseOffs) != 1335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MinOffset > 0)) 1336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MinOffset; 1338572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(AM, Kind, AccessTy, TLI)) { 1339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs - MinOffset; 1340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for overflow. 1341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (((int64_t)((uint64_t)AM.BaseOffs + MaxOffset) > AM.BaseOffs) != 1342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (MaxOffset > 0)) 13437979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 1344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + MaxOffset; 1345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 13467979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 13485f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 13495f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(int64_t BaseOffs, 1351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV, 1352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1353db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1354454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman const TargetLowering *TLI) { 1355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1366a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Canonicalize a scale of 1 to a base register if the formula doesn't 1367a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // already have a base register. 1368a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!AM.HasBaseReg && AM.Scale == 1) { 1369a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.Scale = 0; 1370a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman AM.HasBaseReg = true; 1371a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 1372a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, Kind, AccessTy, TLI); 1374a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1375a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstatic bool isAlwaysFoldable(const SCEV *S, 1377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t MinOffset, int64_t MaxOffset, 1378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool HasBaseReg, 1379db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy, 1380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *TLI, 1381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE) { 1382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (S->isZero()) return true; 1384572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t BaseOffs = ExtractImmediate(S, SE); 1388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *BaseGV = ExtractSymbol(S, SE); 1389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If there's anything else involved, it's not foldable. 1391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!S->isZero()) return false; 1392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Fast-path: zero is always foldable. 1394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseOffs == 0 && !BaseGV) return true; 1395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively, create an address with an immediate and a 1397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // base and a scale. 1398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 1399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseOffs = BaseOffs; 1400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.BaseGV = BaseGV; 1401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.HasBaseReg = HasBaseReg; 1402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1; 1403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return isLegalUse(AM, MinOffset, MaxOffset, Kind, AccessTy, TLI); 1405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 140656a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1407b6211710acdf558b3b45c2d198e74aa602496893Dan Gohmannamespace { 1408b6211710acdf558b3b45c2d198e74aa602496893Dan Gohman 14091e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// UseMapDenseMapInfo - A DenseMapInfo implementation for holding 14101e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman/// DenseMaps and DenseSets of pairs of const SCEV* and LSRUse::Kind. 14111e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohmanstruct UseMapDenseMapInfo { 14121e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getEmptyKey() { 14131e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-1), LSRUse::Basic); 14141e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14151e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14161e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static std::pair<const SCEV *, LSRUse::KindType> getTombstoneKey() { 14171e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return std::make_pair(reinterpret_cast<const SCEV *>(-2), LSRUse::Basic); 14181e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14191e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14201e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static unsigned 14211e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman getHashValue(const std::pair<const SCEV *, LSRUse::KindType> &V) { 14221e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman unsigned Result = DenseMapInfo<const SCEV *>::getHashValue(V.first); 14231e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman Result ^= DenseMapInfo<unsigned>::getHashValue(unsigned(V.second)); 14241e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return Result; 14251e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14261e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14271e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman static bool isEqual(const std::pair<const SCEV *, LSRUse::KindType> &LHS, 14281e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman const std::pair<const SCEV *, LSRUse::KindType> &RHS) { 14291e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman return LHS == RHS; 14301e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman } 14311e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman}; 14321e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman 14336c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// IVInc - An individual increment in a Chain of IV increments. 14346c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Relate an IV user to an expression that computes the IV it uses from the IV 14356c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// used by the previous link in the Chain. 14366c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 14376c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// For the head of a chain, IncExpr holds the absolute SCEV expression for the 14386c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// original IVOperand. The head of the chain's IVOperand is only valid during 14396c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// chain collection, before LSR replaces IV users. During chain generation, 14406c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// IncExpr can be used to find the new IVOperand that computes the same 14416c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// expression. 14426c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstruct IVInc { 14436c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *UserInst; 14446c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Value* IVOperand; 14456c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick const SCEV *IncExpr; 14466c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 14476c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick IVInc(Instruction *U, Value *O, const SCEV *E): 14486c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick UserInst(U), IVOperand(O), IncExpr(E) {} 14496c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick}; 14506c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 14516c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick// IVChain - The list of IV increments in program order. 14526c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick// We typically add the head of a chain without finding subsequent links. 145370a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesenstruct IVChain { 145470a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen SmallVector<IVInc,1> Incs; 1455f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *ExprBase; 1456f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen 1457f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen IVChain() : ExprBase(0) {} 1458f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen 1459f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen IVChain(const IVInc &Head, const SCEV *Base) 1460f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen : Incs(1, Head), ExprBase(Base) {} 146170a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen 146270a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen typedef SmallVectorImpl<IVInc>::const_iterator const_iterator; 146370a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen 146470a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen // begin - return the first increment in the chain. 146570a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen const_iterator begin() const { 146670a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen assert(!Incs.empty()); 146770a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen return llvm::next(Incs.begin()); 146870a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen } 146970a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen const_iterator end() const { 147070a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen return Incs.end(); 147170a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen } 147270a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen 147370a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen // hasIncs - Returns true if this chain contains any increments. 147470a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen bool hasIncs() const { return Incs.size() >= 2; } 147570a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen 147670a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen // add - Add an IVInc to the end of this chain. 147770a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen void add(const IVInc &X) { Incs.push_back(X); } 147870a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen 147970a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen // tailUserInst - Returns the last UserInst in the chain. 148070a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen Instruction *tailUserInst() const { return Incs.back().UserInst; } 1481f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen 1482f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // isProfitableIncrement - Returns true if IncExpr can be profitably added to 1483f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // this chain. 1484f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen bool isProfitableIncrement(const SCEV *OperExpr, 1485f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *IncExpr, 1486f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen ScalarEvolution&); 148770a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen}; 14886c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 14896c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// ChainUsers - Helper for CollectChains to track multiple IV increment uses. 14906c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Distinguish between FarUsers that definitely cross IV increments and 14916c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// NearUsers that may be used between IV increments. 14926c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstruct ChainUsers { 14936c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*, 4> FarUsers; 14946c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*, 4> NearUsers; 14956c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick}; 14966c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 1497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRInstance - This class holds state for the main loop strength reduction 1498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// logic. 1499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LSRInstance { 1500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers &IU; 1501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScalarEvolution &SE; 1502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DominatorTree &DT; 1503e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LoopInfo &LI; 1504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 1505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Loop *const L; 1506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed; 1507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// IVIncInsertPos - This is the insert position that the current loop's 1509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// induction variable increment should be placed. In simple loops, this is 1510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// the latch block's terminator. But in more complicated cases, this is a 1511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// position which will dominate all the in-loop post-increment users. 1512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *IVIncInsertPos; 1513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1514572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Factors - Interesting factors between use strides. 1515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<int64_t, 8> Factors; 1516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Types - Interesting use types, to facilitate truncation reuse. 1518db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner SmallSetVector<Type *, 4> Types; 1519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Fixups - The list of operands which are to be replaced. 1521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRFixup, 16> Fixups; 1522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// Uses - The list of interesting uses. 1524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<LSRUse, 16> Uses; 1525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// RegUses - Track which uses use which register candidates. 1527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUseTracker RegUses; 1528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 15296c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Limit the number of chains to avoid quadratic behavior. We don't expect to 15306c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // have more than a few IV increment chains in a loop. Missing a Chain falls 15316c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // back to normal LSR behavior for those uses. 15326c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick static const unsigned MaxChains = 8; 15336c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 15346c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick /// IVChainVec - IV users can form a chain of IV increments. 15356c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVector<IVChain, MaxChains> IVChainVec; 15366c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 153722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick /// IVIncSet - IV users that belong to profitable IVChains. 153822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SmallPtrSet<Use*, MaxChains> IVIncSet; 153922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 1540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void OptimizeShadowIV(); 1541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse); 1542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse); 1543c6519f916b5922de81c53547fd21364994195a70Dan Gohman void OptimizeLoopTermCond(); 1544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 15456c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick void ChainInstruction(Instruction *UserInst, Instruction *IVOper, 15466c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVectorImpl<ChainUsers> &ChainUsersVec); 154722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick void FinalizeChain(IVChain &Chain); 15486c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick void CollectChains(); 154922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick void GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter, 155022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SmallVectorImpl<WeakVH> &DeadInsts); 15516c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 1552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectInterestingTypesAndFactors(); 1553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectFixupsAndInitialFormulae(); 1554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &getNewFixup() { 1556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Fixups.push_back(LSRFixup()); 1557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return Fixups.back(); 1558a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 155932e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng 1560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Support for sharing of LSRUses between LSRFixups. 15611e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman typedef DenseMap<std::pair<const SCEV *, LSRUse::KindType>, 15621e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman size_t, 15631e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMapDenseMapInfo> UseMapTy; 1564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UseMapTy UseMap; 1565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1566191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 1567db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy); 1568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> getUse(const SCEV *&Expr, 1570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind, 1571db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy); 1572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1573c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman void DeleteUse(LSRUse &LU, size_t LUIdx); 15745ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 1575191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU); 1576a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 1577454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1578572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); 1579572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CountRegisters(const Formula &F, size_t LUIdx); 1580572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F); 1581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void CollectLoopInvariantFixupsAndFormulae(); 1583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base, 1585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth = 0); 1586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base); 1587572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); 1589572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1590572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base); 1591572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base); 1592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateCrossUseConstantOffsets(); 1593572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void GenerateAllReuseFormulae(); 1594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void FilterOutUndesirableDedicatedRegisters(); 1596d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 1597d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t EstimateSearchSpaceComplexity() const; 15984aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByDetectingSupersets(); 15994aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByCollapsingUnrolledCode(); 16004f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 16014aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman void NarrowSearchSpaceByPickingWinnerRegs(); 1602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void NarrowSearchSpaceUsingHeuristics(); 1603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1604572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 1605572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 1606572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 1607572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 1608572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 1609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const; 1610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Solve(SmallVectorImpl<const Formula *> &Solution) const; 1611572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1612e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock::iterator 1613e5f76877aee6f33964de105893f0ef338661ecadDan Gohman HoistInsertPosition(BasicBlock::iterator IP, 1614e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) const; 1615b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick BasicBlock::iterator 1616b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick AdjustInsertPositionForExpand(BasicBlock::iterator IP, 1617b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick const LSRFixup &LF, 1618b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick const LSRUse &LU, 1619b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick SCEVExpander &Rewriter) const; 1620d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 1621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *Expand(const LSRFixup &LF, 1622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1623454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman BasicBlock::iterator IP, 1624572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1625454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const; 16263a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman void RewriteForPHI(PHINode *PN, const LSRFixup &LF, 16273a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 16283a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 16293a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 16303a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const; 1631572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void Rewrite(const LSRFixup &LF, 1632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 1633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 1634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 1635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const; 1636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 1637572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P); 1638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1639d56ef8d709c3dd5735debc23e082722718a7d58aAndrew Trickpublic: 1640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRInstance(const TargetLowering *tli, Loop *l, Pass *P); 1641572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool getChanged() const { return Changed; } 1643572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1644572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_factors_and_types(raw_ostream &OS) const; 1645572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_fixups(raw_ostream &OS) const; 1646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print_uses(raw_ostream &OS) const; 1647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 1648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 1649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 16507979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1651a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1652c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// OptimizeShadowIV - If IV is used in a int-to-float cast 16543f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// inside the loop then try to eliminate the cast operation. 1655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::OptimizeShadowIV() { 1656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 1657572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 1658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return; 1659c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); 1661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != E; /* empty */) { 1662572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVUsers::const_iterator CandidateUI = UI; 1663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ++UI; 1664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *ShadowUse = CandidateUI->getUser(); 1665db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *DestTy = NULL; 1666c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick bool IsSigned = false; 1667c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* If shadow use is a int->float cast then insert a second IV 1669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman to eliminate this cast. 1670c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i) 1672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo((double)i); 1673c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman is transformed into 1675c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman double d = 0.0; 1677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (unsigned i = 0; i < n; ++i, ++d) 1678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman foo(d); 1679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman */ 1680c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) { 1681c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick IsSigned = false; 1682572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = UCast->getDestTy(); 1683c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick } 1684c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) { 1685c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick IsSigned = true; 1686572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DestTy = SCast->getDestTy(); 1687c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick } 1688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DestTy) continue; 16895792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1690572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (TLI) { 1691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If target does not support DestTy natively then do not apply 1692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // this transformation. 1693572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman EVT DVT = TLI->getValueType(DestTy); 1694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI->isTypeLegal(DVT)) continue; 16957e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 16967e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1697572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); 1698572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!PH) continue; 1699572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getNumIncomingValues() != 2) continue; 1700a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1701db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *SrcTy = PH->getType(); 1702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int Mantissa = DestTy->getFPMantissaWidth(); 1703572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Mantissa == -1) continue; 1704572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa) 1705572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 17062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 1707572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Entry, Latch; 1708572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { 1709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 0; 1710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 1; 1711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 1712572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Entry = 1; 1713572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Latch = 0; 1714a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1715eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 1716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); 1717572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Init) continue; 1718c2c988e5e0b15408f790c96fd7ad2d86a6a93a08Andrew Trick Constant *NewInit = ConstantFP::get(DestTy, IsSigned ? 1719c205a094bd5019773c98adcfbdc21c07c9da1888Andrew Trick (double)Init->getSExtValue() : 1720c205a094bd5019773c98adcfbdc21c07c9da1888Andrew Trick (double)Init->getZExtValue()); 1721a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1722572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *Incr = 1723572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); 1724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Incr) continue; 1725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOpcode() != Instruction::Add 1726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman && Incr->getOpcode() != Instruction::Sub) 1727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1728aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 1729572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Initialize new IV, double d = 0.0 in above example. */ 1730572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt *C = NULL; 1731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Incr->getOperand(0) == PH) 1732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(1)); 1733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (Incr->getOperand(1) == PH) 1734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = dyn_cast<ConstantInt>(Incr->getOperand(0)); 1735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 1736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 1737bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman 1738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C) continue; 1739a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore negative constants, as the code below doesn't handle them 1741572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // correctly. TODO: Remove this restriction. 1742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!C->getValue().isStrictlyPositive()) continue; 1743a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Add new PHINode. */ 17453ecfc861b4365f341c5c969b40e1afccde676e6fJay Foad PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH); 1746a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1747572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* create new increment. '++d' in above example. */ 1748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue()); 1749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator *NewIncr = 1750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ? 1751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction::FAdd : Instruction::FSub, 1752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH, CFP, "IV.S.next.", Incr); 1753a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 1754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); 1755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); 17568b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 1757572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /* Remove cast operation */ 1758572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->replaceAllUsesWith(NewPH); 1759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ShadowUse->eraseFromParent(); 1760c6519f916b5922de81c53547fd21364994195a70Dan Gohman Changed = true; 1761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 17627979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman } 1763a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 1764cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 17657979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// FindIVUserForCond - If Cond has an operand that is an expression of an IV, 17667979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// set the IV user and stride information and return true, otherwise return 17677979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// false. 1768ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanbool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) { 1769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UI->getUser() == Cond) { 1771572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // NOTE: we could handle setcc instructions with multiple uses here, but 1772572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // InstCombine does it as well for simple uses, it's not clear that it 1773572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // occurs enough in real life to handle. 1774572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CondUse = UI; 1775572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 1776a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 1777572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 1778cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 1779cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 17807979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// OptimizeMax - Rewrite the loop's terminating condition if it uses 17817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a max computation. 17827979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17837979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This is a narrow solution to a specific, but acute, problem. For loops 17847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// like this: 17857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 17877979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 17887979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 17897979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 17907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 17917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the trip count isn't just 'n', because 'n' might not be positive. And 17927979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// unfortunately this can come up even for loops where the user didn't use 17937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// a C do-while loop. For example, seemingly well-behaved top-test loops 17947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// will commonly be lowered like this: 17957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman// 17967979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// if (n > 0) { 17977979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 17987979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 17997979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 18007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i < n); 18017979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } 18027979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 18037979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// and then it's possible for subsequent optimization to obscure the if 18047979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// test in such a way that indvars can't find it. 18057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 18067979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// When indvars can't find the if test in loops like this, it creates a 18077979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max expression, which allows it to give the loop a canonical 18087979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// induction variable: 18097979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 18107979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// i = 0; 18117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// max = n < 1 ? 1 : n; 18127979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// do { 18137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// p[i] = 0.0; 18147979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// } while (++i != max); 18157979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 18167979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// Canonical induction variables are necessary because the loop passes 18177979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// are designed around them. The most obvious example of this is the 18187979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// LoopInfo analysis, which doesn't remember trip count values. It 18197979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// expects to be able to rediscover the trip count each time it is 1820572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needed, and it does this using a simple analysis that only succeeds if 18217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the loop has a canonical induction variable. 18227979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 18237979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// However, when it comes time to generate code, the maximum operation 18247979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// can be quite costly, especially if it's inside of an outer loop. 18257979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 18267979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// This function solves this problem by detecting this type of loop and 18277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting 18287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// the instructions for the maximum computation. 18297979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman/// 1830572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { 18317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check that the loop matches the pattern we're looking for. 18327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() != CmpInst::ICMP_EQ && 18337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->getPredicate() != CmpInst::ICMP_NE) 18347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1835ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); 18377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!Sel || !Sel->hasOneUse()) return Cond; 1838ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 1839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 18407979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 18417979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1842deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); 1843ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18447979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Add one to the backedge-taken count to get the trip count. 18454065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); 18461d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (IterationCount != SE.getSCEV(Sel)) return Cond; 18471d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 18481d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Check for a max calculation that matches the pattern. There's no check 18491d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for ICMP_ULE here because the comparison would be with zero, which 18501d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // isn't interesting. 18511d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; 18521d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman const SCEVNAryExpr *Max = 0; 18531d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) { 18541d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLE; 18551d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 18561d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) { 18571d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_SLT; 18581d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = S; 18591d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) { 18601d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Pred = ICmpInst::ICMP_ULT; 18611d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman Max = U; 18621d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else { 18631d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // No match; bail. 18647979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 18651d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } 1866ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18677979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // To handle a max with more than two operands, this optimization would 18687979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // require additional checking and setup. 18697979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Max->getNumOperands() != 2) 18707979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 18717979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 18727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxLHS = Max->getOperand(0); 18737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEV *MaxRHS = Max->getOperand(1); 18741d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 18751d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // ScalarEvolution canonicalizes constants to the left. For < and >, look 18761d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // for a comparison with 1. For <= and >=, a comparison with zero. 18771d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!MaxLHS || 18781d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One))) 18791d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 18801d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman 18817979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the relevant induction variable for conformance to 18827979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // the pattern. 1883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); 18847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 18857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (!AR || !AR->isAffine() || 18867979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman AR->getStart() != One || 1887572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AR->getStepRecurrence(SE) != One) 18887979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return Cond; 1889ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 18907979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman assert(AR->getLoop() == L && 18917979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman "Loop condition operand is an addrec in a different loop!"); 1892bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman 18937979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Check the right operand of the select, and remember it, as it will 18947979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // be used in the new comparison instruction. 18957979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Value *NewRHS = 0; 18961d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (ICmpInst::isTrueWhenEqual(Pred)) { 18971d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman // Look for n+1, and grab n. 18981d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1))) 18991d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 19001d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 19011d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 19021d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 19031d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2))) 19041d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (isa<ConstantInt>(BO->getOperand(1)) && 19051d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman cast<ConstantInt>(BO->getOperand(1))->isOne() && 19061d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman SE.getSCEV(BO->getOperand(0)) == MaxRHS) 19071d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman NewRHS = BO->getOperand(0); 19081d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman if (!NewRHS) 19091d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman return Cond; 19101d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS) 19117979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(1); 1912572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS) 19137979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman NewRHS = Sel->getOperand(2); 1914caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS)) 1915caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman NewRHS = SU->getValue(); 19161d367988e21bb1b4e8346877f8fa377dff194c29Dan Gohman else 1917caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman // Max doesn't match expected pattern. 1918caf71ab4735d6b80300cf123a8c3c0759d2e2873Dan Gohman return Cond; 1919ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 19207979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Determine the new comparison opcode. It may be signed or unsigned, 19217979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // and the original comparison may be either equality or inequality. 19227979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cond->getPredicate() == CmpInst::ICMP_EQ) 19237979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Pred = CmpInst::getInversePredicate(Pred); 19242781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman 19257979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Ok, everything looks ok to change the condition into an SLT or SGE and 19267979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // delete the max calculation. 19277979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman ICmpInst *NewCond = 19287979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp"); 1929ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 19307979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // Delete the max calculation instructions. 19317979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->replaceAllUsesWith(NewCond); 19327979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman CondUse->setUser(NewCond); 19337979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); 19347979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cond->eraseFromParent(); 19357979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Sel->eraseFromParent(); 19367979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman if (Cmp->use_empty()) 19377979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman Cmp->eraseFromParent(); 19387979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return NewCond; 1939ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman} 1940ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 194156a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach/// OptimizeLoopTermCond - Change loop terminating condition to use the 19422d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng/// postinc iv when possible. 1943c6519f916b5922de81c53547fd21364994195a70Dan Gohmanvoid 1944572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::OptimizeLoopTermCond() { 1945572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<Instruction *, 4> PostIncs; 1946572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 19475792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng BasicBlock *LatchBlock = L->getLoopLatch(); 1948076e085698c484354c9e131f1bd8fd001581397bEvan Cheng SmallVector<BasicBlock*, 8> ExitingBlocks; 1949076e085698c484354c9e131f1bd8fd001581397bEvan Cheng L->getExitingBlocks(ExitingBlocks); 195056a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 1951076e085698c484354c9e131f1bd8fd001581397bEvan Cheng for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 1952076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BasicBlock *ExitingBlock = ExitingBlocks[i]; 1953010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 1954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Get the terminating condition for the loop if possible. If we 1955076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // can, we want to change it to use a post-incremented version of its 1956076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // induction variable, to allow coalescing the live ranges for the IV into 1957076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // one register value. 1958cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1959076e085698c484354c9e131f1bd8fd001581397bEvan Cheng BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); 1960076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (!TermBr) 1961076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1962076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // FIXME: Overly conservative, termination condition could be an 'or' etc.. 1963076e085698c484354c9e131f1bd8fd001581397bEvan Cheng if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition())) 1964076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1965076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 1966076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Search IVUsesByStride to find Cond's IVUse if there is one. 1967076e085698c484354c9e131f1bd8fd001581397bEvan Cheng IVStrideUse *CondUse = 0; 1968076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); 1969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!FindIVUserForCond(Cond, CondUse)) 1970076e085698c484354c9e131f1bd8fd001581397bEvan Cheng continue; 1971076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 19727979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // If the trip count is computed in terms of a max (due to ScalarEvolution 19737979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // being unable to find a sufficient guard, for example), change the loop 19747979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman // comparison to use SLT or ULT instead of NE. 1975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // One consequence of doing this now is that it disrupts the count-down 1976572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // optimization. That's not always a bad thing though, because in such 1977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // cases it may still be worthwhile to avoid a max. 1978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cond = OptimizeMax(Cond, CondUse); 1979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 1980572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this exiting block dominates the latch block, it may also use 1981572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the post-inc value if it won't be shared with other uses. 1982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for dominance. 1983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(ExitingBlock, LatchBlock)) 19847979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman continue; 19857979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman 1986572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively avoid trying to use the post-inc value in non-latch 1987572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // exits if there may be pre-inc users in intervening blocks. 1988590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (LatchBlock != ExitingBlock) 1989572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) 1990572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Test if the use is reachable from the exiting block. This dominator 1991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // query is a conservative approximation of reachability. 1992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*UI != CondUse && 1993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) { 1994572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume there may be reuse if the quotient of their 1995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // strides could be a legal scale. 1996c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *A = IU.getStride(*CondUse, L); 1997c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *B = IU.getStride(*UI, L); 1998448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (!A || !B) continue; 1999572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) != 2000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) { 2001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(A->getType()) > 2002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(B->getType())) 2003572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman B = SE.getSignExtendExpr(B, A->getType()); 2004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2005572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman A = SE.getSignExtendExpr(A, B->getType()); 2006572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2007572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *D = 2008f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) { 20099f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const ConstantInt *C = D->getValue(); 2010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Stride of one or negative one can have reuse with non-addresses. 20119f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->isOne() || C->isAllOnesValue()) 2012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 2013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Avoid weird situations. 20149f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman if (C->getValue().getMinSignedBits() >= 64 || 20159f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman C->getValue().isMinSignedValue()) 2016572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 2017590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // Without TLI, assume that any stride might be valid, and so any 2018590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman // use might be shared. 2019590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman if (!TLI) 2020590bfe8641631c136160fed7a9df9fe805938cbeDan Gohman goto decline_post_inc; 2021572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for possible scaled-address reuse. 2022db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = getAccessType(UI->getUser()); 2023572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TargetLowering::AddrMode AM; 20249f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman AM.Scale = C->getSExtValue(); 20252763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 2026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 2027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AM.Scale = -AM.Scale; 20282763dfdc701964a01c4a0386fadd1c4e92be9021Dan Gohman if (TLI->isLegalAddressingMode(AM, AccessTy)) 2029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto decline_post_inc; 2030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 203363c9463c62fce8cbe02176dfa2d73f375a06f1f2David Greene DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: " 2034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << *Cond << '\n'); 203556a1f806aff00a8be6db4eb5c9cd5c751ff0f4c1Jim Grosbach 2036076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // It's possible for the setcc instruction to be anywhere in the loop, and 2037076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // possible for it to have multiple users. If it is not immediately before 2038076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // the exiting block branch, move it. 2039572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (&*++BasicBlock::iterator(Cond) != TermBr) { 2040572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Cond->hasOneUse()) { 2041076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->moveBefore(TermBr); 2042076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } else { 2043572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clone the terminating condition and insert into the loopend. 2044572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *OldCond = Cond; 2045076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond = cast<ICmpInst>(Cond->clone()); 2046076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Cond->setName(L->getHeader()->getName() + ".termcond"); 2047076e085698c484354c9e131f1bd8fd001581397bEvan Cheng ExitingBlock->getInstList().insert(TermBr, Cond); 2048076e085698c484354c9e131f1bd8fd001581397bEvan Cheng 2049076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // Clone the IVUse, as the old use still exists! 20504417e537b65c14b378aeca75b2773582dd102f63Andrew Trick CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace()); 2051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TermBr->replaceUsesOfWith(OldCond, Cond); 2052076e085698c484354c9e131f1bd8fd001581397bEvan Cheng } 2053010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2054010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2055076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // If we get to here, we know that we can transform the setcc instruction to 2056076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // use the post-incremented version of the IV, allowing us to coalesce the 2057076e085698c484354c9e131f1bd8fd001581397bEvan Cheng // live ranges for the IV correctly. 2058448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman CondUse->transformToPostInc(L); 2059076e085698c484354c9e131f1bd8fd001581397bEvan Cheng Changed = true; 20605792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 2061572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PostIncs.insert(Cond); 2062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman decline_post_inc:; 2063c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen } 2064c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2065572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine an insertion point for the loop induction variable increment. It 2066572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // must dominate all the post-inc comparisons we just set up, and it must 2067572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // dominate the loop latch edge. 2068572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = L->getLoopLatch()->getTerminator(); 2069572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<Instruction *, 4>::const_iterator I = PostIncs.begin(), 2070572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = PostIncs.end(); I != E; ++I) { 2071572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BasicBlock *BB = 2072572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT.findNearestCommonDominator(IVIncInsertPos->getParent(), 2073572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (*I)->getParent()); 2074572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BB == (*I)->getParent()) 2075572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = *I; 2076572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (BB != IVIncInsertPos->getParent()) 2077572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman IVIncInsertPos = BB->getTerminator(); 2078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2079572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2080a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 20817a2bdde0a0eebcd2125055e0eacaca040f0b766cChris Lattner/// reconcileNewOffset - Determine if the given use can accommodate a fixup 208276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// at the given offset and other details. If so, update the use and 208376c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// return true. 2084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool 2085191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan GohmanLSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, 2086db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy) { 2087191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMinOffset = LU.MinOffset; 2088191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman int64_t NewMaxOffset = LU.MaxOffset; 2089db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *NewAccessTy = AccessTy; 2090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2091572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched kind. It's tempting to collapse mismatched kinds to 2092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // something conservative, however this can pessimize in the case that one of 2093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the uses will have all its uses outside the loop, for example. 2094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != Kind) 2095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return false; 2096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively assume HasBaseReg is true for now. 2097191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset < LU.MinOffset) { 2098191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(LU.MaxOffset - NewOffset, 0, HasBaseReg, 2099454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 21007979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 2101191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMinOffset = NewOffset; 2102191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } else if (NewOffset > LU.MaxOffset) { 2103191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (!isAlwaysFoldable(NewOffset - LU.MinOffset, 0, HasBaseReg, 2104454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Kind, AccessTy, TLI)) 21057979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 2106191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewMaxOffset = NewOffset; 2107a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check for a mismatched access type, and fall back conservatively as needed. 210974e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // TODO: Be less conservative when the type is similar and can use the same 211074e5ef096ee53586ae8798c35d64e5974345c187Dan Gohman // addressing modes. 2111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Kind == LSRUse::Address && AccessTy != LU.AccessTy) 2112191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman NewAccessTy = Type::getVoidTy(AccessTy->getContext()); 2113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the use. 2115191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MinOffset = NewMinOffset; 2116191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.MaxOffset = NewMaxOffset; 2117191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.AccessTy = NewAccessTy; 2118191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (NewOffset != LU.Offsets.back()) 2119191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(NewOffset); 2120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 2121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2122a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// getUse - Return an LSRUse index and an offset value for a fixup which 2124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// needs the given expression, with the given kind and optional access type. 21253f46a3abeedba8d517b4182de34c821d752db058Dan Gohman/// Either reuse an existing use or create a new one, as needed. 2126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstd::pair<size_t, int64_t> 2127572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::getUse(const SCEV *&Expr, 2128db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner LSRUse::KindType Kind, Type *AccessTy) { 2129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Copy = Expr; 2130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = ExtractImmediate(Expr, SE); 2131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Basic uses can't accept any offset, for example. 2133454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!isAlwaysFoldable(Offset, 0, /*HasBaseReg=*/true, Kind, AccessTy, TLI)) { 2134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Expr = Copy; 2135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = 0; 2136a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2137a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<UseMapTy::iterator, bool> P = 21391e3121c80a02359fda87cf77ce1fd7bbd5066991Dan Gohman UseMap.insert(std::make_pair(std::make_pair(Expr, Kind), 0)); 2140572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!P.second) { 2141572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // A use already existed with this base. 2142572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = P.first->second; 2143572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2144191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy)) 2145572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Reuse this use. 2146572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 2147572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2148a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2149572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Create a new use. 2150572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = Uses.size(); 2151572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman P.first->second = LUIdx; 2152572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses.push_back(LSRUse(Kind, AccessTy)); 2153572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 2154a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2155191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // We don't need to track redundant offsets, but we don't need to go out 2156191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // of our way here to avoid them. 2157191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LU.Offsets.empty() || Offset != LU.Offsets.back()) 2158191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LU.Offsets.push_back(Offset); 2159191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 2160572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MinOffset = Offset; 2161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.MaxOffset = Offset; 2162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return std::make_pair(LUIdx, Offset); 2163a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman} 2164a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 21655ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman/// DeleteUse - Delete the given use from the Uses list. 2166c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohmanvoid LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) { 2167191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (&LU != &Uses.back()) 21685ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman std::swap(LU, Uses.back()); 21695ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman Uses.pop_back(); 2170c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman 2171c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman // Update RegUses. 2172c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman RegUses.SwapAndDropUse(LUIdx, Uses.size()); 21735ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman} 21745ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman 2175a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// FindUseWithFormula - Look for a use distinct from OrigLU which is has 2176a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman/// a formula that has the same registers as the given formula. 2177a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRUse * 2178a2086b3483b88b5b64f098b6644e450f94933f49Dan GohmanLSRInstance::FindUseWithSimilarFormula(const Formula &OrigF, 2179191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman const LSRUse &OrigLU) { 2180191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Search all uses for the formula. This could be more clever. 2181a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 2182a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 21836a832715325f740a0e78acd194abad9104389075Dan Gohman // Check whether this use is close enough to OrigLU, to see whether it's 21846a832715325f740a0e78acd194abad9104389075Dan Gohman // worthwhile looking through its formulae. 21856a832715325f740a0e78acd194abad9104389075Dan Gohman // Ignore ICmpZero uses because they may contain formulae generated by 21866a832715325f740a0e78acd194abad9104389075Dan Gohman // GenerateICmpZeroScales, in which case adding fixup offsets may 21876a832715325f740a0e78acd194abad9104389075Dan Gohman // be invalid. 2188a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (&LU != &OrigLU && 2189a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind != LSRUse::ICmpZero && 2190a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy && 2191a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType == OrigLU.WidestFixupType && 2192a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.HasFormulaWithSameRegs(OrigF)) { 21936a832715325f740a0e78acd194abad9104389075Dan Gohman // Scan through this use's formulae. 2194402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 2195402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 2196402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 21976a832715325f740a0e78acd194abad9104389075Dan Gohman // Check to see if this formula has the same registers and symbols 21986a832715325f740a0e78acd194abad9104389075Dan Gohman // as OrigF. 2199a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.BaseRegs == OrigF.BaseRegs && 2200a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.ScaledReg == OrigF.ScaledReg && 2201a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman F.AM.BaseGV == OrigF.AM.BaseGV && 2202cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.AM.Scale == OrigF.AM.Scale && 2203cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset == OrigF.UnfoldedOffset) { 2204191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (F.AM.BaseOffs == 0) 2205a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return &LU; 22066a832715325f740a0e78acd194abad9104389075Dan Gohman // This is the formula where all the registers and symbols matched; 22076a832715325f740a0e78acd194abad9104389075Dan Gohman // there aren't going to be any others. Since we declined it, we 2208d9b0b025612992a0b724eeca8bdf10b1d7a5c355Benjamin Kramer // can skip the rest of the formulae and proceed to the next LSRUse. 2209a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 2210a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2211a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2212a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2213a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 2214a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 22156a832715325f740a0e78acd194abad9104389075Dan Gohman // Nothing looked good. 2216a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman return 0; 2217a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman} 2218a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 2219572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectInterestingTypesAndFactors() { 2220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> Strides; 2221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 22221b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Collect interesting types and strides. 2223448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman SmallVector<const SCEV *, 4> Worklist; 2224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 2225c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *Expr = IU.getExpr(*UI); 2226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect interesting types. 2228448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Types.insert(SE.getEffectiveSCEVType(Expr->getType())); 2229448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman 2230448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // Add strides for mentioned loops. 2231448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(Expr); 2232448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman do { 2233448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman const SCEV *S = Worklist.pop_back_val(); 2234448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 2235bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick if (AR->getLoop() == L) 2236fa1948a40f14d98c1a31a2ec19035a2d5254e854Andrew Trick Strides.insert(AR->getStepRecurrence(SE)); 2237448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Worklist.push_back(AR->getStart()); 2238448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 2239403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(Add->op_begin(), Add->op_end()); 2240448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } 2241448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman } while (!Worklist.empty()); 22421b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman } 22431b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman 22441b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman // Compute interesting factors from the set of interesting strides. 22451b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator 22461b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman I = Strides.begin(), E = Strides.end(); I != E; ++I) 2247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = 2248ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes llvm::next(I); NewStrideIter != E; ++NewStrideIter) { 22491b7bf18def8db328bb4efa02c0958ea399e8d8cbDan Gohman const SCEV *OldStride = *I; 2250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NewStride = *NewStrideIter; 2251a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman 2252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) != 2253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) { 2254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(OldStride->getType()) > 2255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getTypeSizeInBits(NewStride->getType())) 2256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType()); 2257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 2258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType()); 2259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *Factor = 2261f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride, 2262f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVConstant *Factor = 2266454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride, 2267454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman NewStride, 2268f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman SE, true))) { 2269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factor->getValue()->getValue().getMinSignedBits() <= 64) 2270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(Factor->getValue()->getValue().getSExtValue()); 2271a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2272a10756ee657a4d43a48cca5c166919093930ed6bDan Gohman } 2273586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If all uses use the same type, don't bother looking for truncation-based 2275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reuse. 2276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Types.size() == 1) 2277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 2278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_factors_and_types(dbgs())); 2280c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen} 2281c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 22826c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// findIVOperand - Helper for CollectChains that finds an IV operand (computed 22836c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// by an AddRec in this loop) within [OI,OE) or returns OE. If IVUsers mapped 22846c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Instructions to IVStrideUses, we could partially skip this. 22856c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstatic User::op_iterator 22866c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew TrickfindIVOperand(User::op_iterator OI, User::op_iterator OE, 22876c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Loop *L, ScalarEvolution &SE) { 22886c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for(; OI != OE; ++OI) { 22896c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (Instruction *Oper = dyn_cast<Instruction>(*OI)) { 22906c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!SE.isSCEVable(Oper->getType())) 22916c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 22926c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 22936c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (const SCEVAddRecExpr *AR = 22946c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Oper))) { 22956c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (AR->getLoop() == L) 22966c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick break; 22976c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 22986c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 22996c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 23006c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return OI; 23016c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 23026c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 23036c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// getWideOperand - IVChain logic must consistenctly peek base TruncInst 23046c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// operands, so wrap it in a convenient helper. 23056c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstatic Value *getWideOperand(Value *Oper) { 23066c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (TruncInst *Trunc = dyn_cast<TruncInst>(Oper)) 23076c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return Trunc->getOperand(0); 23086c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return Oper; 23096c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 23106c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 23116c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// isCompatibleIVType - Return true if we allow an IV chain to include both 23126c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// types. 23136c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickstatic bool isCompatibleIVType(Value *LVal, Value *RVal) { 23146c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Type *LType = LVal->getType(); 23156c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Type *RType = RVal->getType(); 23166c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return (LType == RType) || (LType->isPointerTy() && RType->isPointerTy()); 23176c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 23186c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 231964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// getExprBase - Return an approximation of this SCEV expression's "base", or 232064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// NULL for any constant. Returning the expression itself is 232164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// conservative. Returning a deeper subexpression is more precise and valid as 232264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// long as it isn't less complex than another subexpression. For expressions 232364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// involving multiple unscaled values, we need to return the pointer-type 232464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// SCEVUnknown. This avoids forming chains across objects, such as: 232564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// PrevOper==a[i], IVOper==b[i], IVInc==b-a. 232664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// 232764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost 232864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick/// SCEVUnknown, we simply return the rightmost SCEV operand. 232964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trickstatic const SCEV *getExprBase(const SCEV *S) { 233064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick switch (S->getSCEVType()) { 233164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick default: // uncluding scUnknown. 233264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return S; 233364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scConstant: 233464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return 0; 233564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scTruncate: 233664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVTruncateExpr>(S)->getOperand()); 233764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scZeroExtend: 233864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVZeroExtendExpr>(S)->getOperand()); 233964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scSignExtend: 234064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVSignExtendExpr>(S)->getOperand()); 234164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scAddExpr: { 234264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Skip over scaled operands (scMulExpr) to follow add operands as long as 234364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // there's nothing more complex. 234464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // FIXME: not sure if we want to recognize negation. 234564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEVAddExpr *Add = cast<SCEVAddExpr>(S); 234664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (std::reverse_iterator<SCEVAddExpr::op_iterator> I(Add->op_end()), 234764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick E(Add->op_begin()); I != E; ++I) { 234864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *SubExpr = *I; 234964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (SubExpr->getSCEVType() == scAddExpr) 235064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(SubExpr); 235164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 235264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (SubExpr->getSCEVType() != scMulExpr) 235364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return SubExpr; 235464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 235564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return S; // all operands are scaled, be conservative. 235664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 235764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick case scAddRecExpr: 235864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return getExprBase(cast<SCEVAddRecExpr>(S)->getStart()); 235964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 236064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick} 236164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 236222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Return true if the chain increment is profitable to expand into a loop 236322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// invariant value, which may require its own register. A profitable chain 236422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// increment will be an offset relative to the same base. We allow such offsets 236522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// to potentially be used as chain increment as long as it's not obviously 236622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// expensive to expand using real instructions. 2367f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesenbool IVChain::isProfitableIncrement(const SCEV *OperExpr, 2368f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *IncExpr, 2369f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen ScalarEvolution &SE) { 2370f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // Aggressively form chains when -stress-ivchain. 237122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (StressIVChain) 2372f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen return true; 237322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 237464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Do not replace a constant offset from IV head with a nonconstant IV 237564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // increment. 237664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (!isa<SCEVConstant>(IncExpr)) { 2377f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand)); 237864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr))) 237964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return 0; 238064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 238164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 238264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick SmallPtrSet<const SCEV*, 8> Processed; 2383f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen return !isHighCostExpansion(IncExpr, Processed, SE); 238422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 238522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 238622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Return true if the number of registers needed for the chain is estimated to 238722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// be less than the number required for the individual IV users. First prohibit 238822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// any IV users that keep the IV live across increments (the Users set should 238922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// be empty). Next count the number and type of increments in the chain. 239022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// 239122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Chaining IVs can lead to considerable code bloat if ISEL doesn't 239222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// effectively use postinc addressing modes. Only consider it profitable it the 239322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// increments can be computed in fewer registers when chained. 239422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// 239522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// TODO: Consider IVInc free if it's already used in another chains. 239622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic bool 239722d20c218aeb14af388bff2346d6d4cc131e8449Andrew TrickisProfitableChain(IVChain &Chain, SmallPtrSet<Instruction*, 4> &Users, 239822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ScalarEvolution &SE, const TargetLowering *TLI) { 239922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (StressIVChain) 240022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return true; 240122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 240270a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen if (!Chain.hasIncs()) 240364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 240464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 240564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (!Users.empty()) { 240670a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " users:\n"; 240764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (SmallPtrSet<Instruction*, 4>::const_iterator I = Users.begin(), 240864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick E = Users.end(); I != E; ++I) { 240964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick dbgs() << " " << **I << "\n"; 241064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick }); 241164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return false; 241264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 241370a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen assert(!Chain.Incs.empty() && "empty IV chains are not allowed"); 241464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 241564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // The chain itself may require a register, so intialize cost to 1. 241664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick int cost = 1; 241764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 241864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // A complete chain likely eliminates the need for keeping the original IV in 241964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // a register. LSR does not currently know how to form a complete chain unless 242064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // the header phi already exists. 242170a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen if (isa<PHINode>(Chain.tailUserInst()) 242270a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen && SE.getSCEV(Chain.tailUserInst()) == Chain.Incs[0].IncExpr) { 242364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick --cost; 242464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 242564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick const SCEV *LastIncExpr = 0; 242664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick unsigned NumConstIncrements = 0; 242764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick unsigned NumVarIncrements = 0; 242864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick unsigned NumReusedIncrements = 0; 242970a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen for (IVChain::const_iterator I = Chain.begin(), E = Chain.end(); 243064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick I != E; ++I) { 243164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 243264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (I->IncExpr->isZero()) 243364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick continue; 243464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 243564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Incrementing by zero or some constant is neutral. We assume constants can 243664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // be folded into an addressing mode or an add's immediate operand. 243764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (isa<SCEVConstant>(I->IncExpr)) { 243864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ++NumConstIncrements; 243964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick continue; 244064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 244164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 244264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (I->IncExpr == LastIncExpr) 244364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ++NumReusedIncrements; 244464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick else 244564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ++NumVarIncrements; 244664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 244764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick LastIncExpr = I->IncExpr; 244864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 244964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // An IV chain with a single increment is handled by LSR's postinc 245064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // uses. However, a chain with multiple increments requires keeping the IV's 245164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // value live longer than it needs to be if chained. 245264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick if (NumConstIncrements > 1) 245364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick --cost; 245464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 245564925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Materializing increment expressions in the preheader that didn't exist in 245664925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // the original code may cost a register. For example, sign-extended array 245764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // indices can produce ridiculous increments like this: 245864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64))) 245964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick cost += NumVarIncrements; 246064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 246164925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Reusing variable increments likely saves a register to hold the multiple of 246264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // the stride. 246364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick cost -= NumReusedIncrements; 246464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 246570a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " Cost: " << cost 246670a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen << "\n"); 246764925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 246864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick return cost < 0; 246922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 247022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 24716c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// ChainInstruction - Add this IV user to an existing chain or make it the head 24726c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// of a new chain. 24736c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickvoid LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper, 24746c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVectorImpl<ChainUsers> &ChainUsersVec) { 24756c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // When IVs are used as types of varying widths, they are generally converted 24766c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // to a wider type with some uses remaining narrow under a (free) trunc. 2477f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen Value *const NextIV = getWideOperand(IVOper); 2478f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *const OperExpr = SE.getSCEV(NextIV); 2479f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *const OperExprBase = getExprBase(OperExpr); 24806c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 24816c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Visit all existing chains. Check if its IVOper can be computed as a 24826c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // profitable loop invariant increment from the last link in the Chain. 24836c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick unsigned ChainIdx = 0, NChains = IVChainVec.size(); 24846c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick const SCEV *LastIncExpr = 0; 24856c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (; ChainIdx < NChains; ++ChainIdx) { 2486f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen IVChain &Chain = IVChainVec[ChainIdx]; 2487f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen 2488f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // Prune the solution space aggressively by checking that both IV operands 2489f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // are expressions that operate on the same unscaled SCEVUnknown. This 2490f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // "base" will be canceled by the subsequent getMinusSCEV call. Checking 2491f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // first avoids creating extra SCEV expressions. 2492f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen if (!StressIVChain && Chain.ExprBase != OperExprBase) 2493f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen continue; 2494f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen 2495f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen Value *PrevIV = getWideOperand(Chain.Incs.back().IVOperand); 24966c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!isCompatibleIVType(PrevIV, NextIV)) 24976c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 24986c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 2499d4e46a6316c8e1dc4623ead64b4cdd3dfb103180Andrew Trick // A phi node terminates a chain. 2500f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen if (isa<PHINode>(UserInst) && isa<PHINode>(Chain.tailUserInst())) 2501f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen continue; 2502f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen 2503f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // The increment must be loop-invariant so it can be kept in a register. 2504f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *PrevExpr = SE.getSCEV(PrevIV); 2505f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr); 2506f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen if (!SE.isLoopInvariant(IncExpr, L)) 25076c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 25086c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 2509f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) { 25106c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick LastIncExpr = IncExpr; 25116c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick break; 25126c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25136c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25146c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // If we haven't found a chain, create a new one, unless we hit the max. Don't 25156c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // bother for phi nodes, because they must be last in the chain. 25166c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (ChainIdx == NChains) { 25176c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (isa<PHINode>(UserInst)) 25186c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return; 251922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (NChains >= MaxChains && !StressIVChain) { 25206c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick DEBUG(dbgs() << "IV Chain Limit\n"); 25216c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick return; 25226c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 2523f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen LastIncExpr = OperExpr; 25240041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick // IVUsers may have skipped over sign/zero extensions. We don't currently 25250041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick // attempt to form chains involving extensions unless they can be hoisted 25260041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick // into this loop's AddRec. 25270041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick if (!isa<SCEVAddRecExpr>(LastIncExpr)) 25280041d4d447c26825e566ba38a4fe301471fda1ebAndrew Trick return; 25296c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ++NChains; 2530f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen IVChainVec.push_back(IVChain(IVInc(UserInst, IVOper, LastIncExpr), 2531f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen OperExprBase)); 25326c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec.resize(NChains); 2533165324cd7da5d4c534c52f7db51e6ef0c185cf5bJakob Stoklund Olesen DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst 2534165324cd7da5d4c534c52f7db51e6ef0c185cf5bJakob Stoklund Olesen << ") IV=" << *LastIncExpr << "\n"); 2535f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen } else { 2536165324cd7da5d4c534c52f7db51e6ef0c185cf5bJakob Stoklund Olesen DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst 2537165324cd7da5d4c534c52f7db51e6ef0c185cf5bJakob Stoklund Olesen << ") IV+" << *LastIncExpr << "\n"); 2538f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen // Add this IV user to the end of the chain. 2539f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen IVChainVec[ChainIdx].add(IVInc(UserInst, IVOper, LastIncExpr)); 2540f9f1c7aa89c87a9c8d6b8b317957b24e44f66570Jakob Stoklund Olesen } 25416c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25426c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers; 25436c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // This chain's NearUsers become FarUsers. 25446c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!LastIncExpr->isZero()) { 25456c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec[ChainIdx].FarUsers.insert(NearUsers.begin(), 25466c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick NearUsers.end()); 25476c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick NearUsers.clear(); 25486c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25496c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25506c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // All other uses of IVOperand become near uses of the chain. 25516c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // We currently ignore intermediate values within SCEV expressions, assuming 25526c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // they will eventually be used be the current chain, or can be computed 25536c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // from one of the chain increments. To be more precise we could 25546c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // transitively follow its user and only add leaf IV users to the set. 25556c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (Value::use_iterator UseIter = IVOper->use_begin(), 25566c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick UseEnd = IVOper->use_end(); UseIter != UseEnd; ++UseIter) { 25576c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *OtherUse = dyn_cast<Instruction>(*UseIter); 255881748bc32092c75c61bb7cb34bfb5a616924e535Andrew Trick if (!OtherUse || OtherUse == UserInst) 255981748bc32092c75c61bb7cb34bfb5a616924e535Andrew Trick continue; 25606c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (SE.isSCEVable(OtherUse->getType()) 25616c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick && !isa<SCEVUnknown>(SE.getSCEV(OtherUse)) 25626c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick && IU.isIVUserOrOperand(OtherUse)) { 25636c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 25646c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 256581748bc32092c75c61bb7cb34bfb5a616924e535Andrew Trick NearUsers.insert(OtherUse); 25666c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 25676c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25686c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Since this user is part of the chain, it's no longer considered a use 25696c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // of the chain. 25706c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec[ChainIdx].FarUsers.erase(UserInst); 25716c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 25726c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25736c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// CollectChains - Populate the vector of Chains. 25746c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25756c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// This decreases ILP at the architecture level. Targets with ample registers, 25766c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// multiple memory ports, and no register renaming probably don't want 25776c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// this. However, such targets should probably disable LSR altogether. 25786c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25796c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// The job of LSR is to make a reasonable choice of induction variables across 25806c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// the loop. Subsequent passes can easily "unchain" computation exposing more 25816c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// ILP *within the loop* if the target wants it. 25826c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25836c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// Finding the best IV chain is potentially a scheduling problem. Since LSR 25846c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// will not reorder memory operations, it will recognize this as a chain, but 25856c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// will generate redundant IV increments. Ideally this would be corrected later 25866c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// by a smart scheduler: 25876c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// = A[i] 25886c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// = A[i+x] 25896c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// A[i] = 25906c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// A[i+x] = 25916c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// 25926c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// TODO: Walk the entire domtree within this loop, not just the path to the 25936c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// loop latch. This will discover chains on side paths, but requires 25946c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick/// maintaining multiple copies of the Chains state. 25956c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trickvoid LSRInstance::CollectChains() { 2596165324cd7da5d4c534c52f7db51e6ef0c185cf5bJakob Stoklund Olesen DEBUG(dbgs() << "Collecting IV Chains.\n"); 25976c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVector<ChainUsers, 8> ChainUsersVec; 25986c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 25996c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallVector<BasicBlock *,8> LatchPath; 26006c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick BasicBlock *LoopHeader = L->getHeader(); 26016c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (DomTreeNode *Rung = DT.getNode(L->getLoopLatch()); 26026c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) { 26036c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick LatchPath.push_back(Rung->getBlock()); 26046c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 26056c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick LatchPath.push_back(LoopHeader); 26066c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 26076c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Walk the instruction stream from the loop header to the loop latch. 26086c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (SmallVectorImpl<BasicBlock *>::reverse_iterator 26096c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick BBIter = LatchPath.rbegin(), BBEnd = LatchPath.rend(); 26106c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick BBIter != BBEnd; ++BBIter) { 26116c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (BasicBlock::iterator I = (*BBIter)->begin(), E = (*BBIter)->end(); 26126c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick I != E; ++I) { 26136c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Skip instructions that weren't seen by IVUsers analysis. 26146c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (isa<PHINode>(I) || !IU.isIVUserOrOperand(I)) 26156c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 26166c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 26176c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Ignore users that are part of a SCEV expression. This way we only 26186c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // consider leaf IV Users. This effectively rediscovers a portion of 26196c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // IVUsers analysis but in program order this time. 26206c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (SE.isSCEVable(I->getType()) && !isa<SCEVUnknown>(SE.getSCEV(I))) 26216c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 26226c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 26236c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Remove this instruction from any NearUsers set it may be in. 26246c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (unsigned ChainIdx = 0, NChains = IVChainVec.size(); 26256c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainIdx < NChains; ++ChainIdx) { 26266c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainUsersVec[ChainIdx].NearUsers.erase(I); 26276c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 26286c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Search for operands that can be chained. 26296c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick SmallPtrSet<Instruction*, 4> UniqueOperands; 26306c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick User::op_iterator IVOpEnd = I->op_end(); 26316c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick User::op_iterator IVOpIter = findIVOperand(I->op_begin(), IVOpEnd, L, SE); 26326c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick while (IVOpIter != IVOpEnd) { 26336c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *IVOpInst = cast<Instruction>(*IVOpIter); 26346c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (UniqueOperands.insert(IVOpInst)) 26356c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainInstruction(I, IVOpInst, ChainUsersVec); 26366c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick IVOpIter = findIVOperand(llvm::next(IVOpIter), IVOpEnd, L, SE); 26376c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 26386c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } // Continue walking down the instructions. 26396c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } // Continue walking down the domtree. 26406c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick // Visit phi backedges to determine if the chain can generate the IV postinc. 26416c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick for (BasicBlock::iterator I = L->getHeader()->begin(); 26426c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick PHINode *PN = dyn_cast<PHINode>(I); ++I) { 26436c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (!SE.isSCEVable(PN->getType())) 26446c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick continue; 26456c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 26466c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick Instruction *IncV = 26476c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick dyn_cast<Instruction>(PN->getIncomingValueForBlock(L->getLoopLatch())); 26486c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick if (IncV) 26496c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick ChainInstruction(PN, IncV, ChainUsersVec); 26506c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick } 265122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Remove any unprofitable chains. 265222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick unsigned ChainIdx = 0; 265322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (unsigned UsersIdx = 0, NChains = IVChainVec.size(); 265422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick UsersIdx < NChains; ++UsersIdx) { 265522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!isProfitableChain(IVChainVec[UsersIdx], 265622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ChainUsersVec[UsersIdx].FarUsers, SE, TLI)) 265722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 265822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Preserve the chain at UsesIdx. 265922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (ChainIdx != UsersIdx) 266022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVChainVec[ChainIdx] = IVChainVec[UsersIdx]; 266122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick FinalizeChain(IVChainVec[ChainIdx]); 266222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ++ChainIdx; 266322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 266422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVChainVec.resize(ChainIdx); 266522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 266622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 266722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickvoid LSRInstance::FinalizeChain(IVChain &Chain) { 266870a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen assert(!Chain.Incs.empty() && "empty IV chains are not allowed"); 266970a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n"); 267022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 267170a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen for (IVChain::const_iterator I = Chain.begin(), E = Chain.end(); 267222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick I != E; ++I) { 267322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DEBUG(dbgs() << " Inc: " << *I->UserInst << "\n"); 267422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator UseI = 267522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick std::find(I->UserInst->op_begin(), I->UserInst->op_end(), I->IVOperand); 267622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(UseI != I->UserInst->op_end() && "cannot find IV operand"); 267722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVIncSet.insert(UseI); 267822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 267922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 268022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 268122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// Return true if the IVInc can be folded into an addressing mode. 268222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickstatic bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst, 268322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *Operand, const TargetLowering *TLI) { 268422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(IncExpr); 268522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!IncConst || !isAddressUse(UserInst, Operand)) 268622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return false; 268722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 268822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IncConst->getValue()->getValue().getMinSignedBits() > 64) 268922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return false; 269022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 269122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick int64_t IncOffset = IncConst->getValue()->getSExtValue(); 269222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!isAlwaysFoldable(IncOffset, /*BaseGV=*/0, /*HaseBaseReg=*/false, 269322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LSRUse::Address, getAccessType(UserInst), TLI)) 269422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return false; 269522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 269622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return true; 269722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick} 269822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 269922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// GenerateIVChains - Generate an add or subtract for each IVInc in a chain to 270022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick/// materialize the IV user's operand from the previous IV user's operand. 270122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trickvoid LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter, 270222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SmallVectorImpl<WeakVH> &DeadInsts) { 270322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Find the new IVOperand for the head of the chain. It may have been replaced 270422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // by LSR. 270570a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen const IVInc &Head = Chain.Incs[0]; 270622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator IVOpEnd = Head.UserInst->op_end(); 270722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator IVOpIter = findIVOperand(Head.UserInst->op_begin(), 270822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOpEnd, L, SE); 270922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IVSrc = 0; 271022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick while (IVOpIter != IVOpEnd) { 271122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVSrc = getWideOperand(*IVOpIter); 271222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 271322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // If this operand computes the expression that the chain needs, we may use 271422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // it. (Check this after setting IVSrc which is used below.) 271522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // 271622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Note that if Head.IncExpr is wider than IVSrc, then this phi is too 271722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // narrow for the chain, so we can no longer use it. We do allow using a 271822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // wider phi, assuming the LSR checked for free truncation. In that case we 271922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // should already have a truncate on this operand such that 272022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // getSCEV(IVSrc) == IncExpr. 272122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (SE.getSCEV(*IVOpIter) == Head.IncExpr 272222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick || SE.getSCEV(IVSrc) == Head.IncExpr) { 272322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick break; 272422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 272522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOpIter = findIVOperand(llvm::next(IVOpIter), IVOpEnd, L, SE); 272622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 272722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVOpIter == IVOpEnd) { 272822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Gracefully give up on this chain. 272922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n"); 273022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick return; 273122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 273222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 273322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n"); 273422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *IVTy = IVSrc->getType(); 273522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *IntTy = SE.getEffectiveSCEVType(IVTy); 273622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEV *LeftOverExpr = 0; 273770a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen for (IVChain::const_iterator IncI = Chain.begin(), 273822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IncE = Chain.end(); IncI != IncE; ++IncI) { 273922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 274022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Instruction *InsertPt = IncI->UserInst; 274122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (isa<PHINode>(InsertPt)) 274222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick InsertPt = L->getLoopLatch()->getTerminator(); 274322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 274422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // IVOper will replace the current IV User's operand. IVSrc is the IV 274522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // value currently held in a register. 274622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IVOper = IVSrc; 274722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!IncI->IncExpr->isZero()) { 274822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // IncExpr was the result of subtraction of two narrow values, so must 274922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // be signed. 275022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEV *IncExpr = SE.getNoopOrSignExtend(IncI->IncExpr, IntTy); 275122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LeftOverExpr = LeftOverExpr ? 275222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr; 275322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 275422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (LeftOverExpr && !LeftOverExpr->isZero()) { 275522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Expand the IV increment. 275622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Rewriter.clearPostInc(); 275722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IncV = Rewriter.expandCodeFor(LeftOverExpr, IntTy, InsertPt); 275822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc), 275922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick SE.getUnknown(IncV)); 276022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt); 276122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 276222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // If an IV increment can't be folded, use it as the next IV value. 276322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!canFoldIVIncExpr(LeftOverExpr, IncI->UserInst, IncI->IVOperand, 276422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick TLI)) { 276522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(IVTy == IVOper->getType() && "inconsistent IV increment type"); 276622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVSrc = IVOper; 276722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LeftOverExpr = 0; 276822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 276922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 277022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *OperTy = IncI->IVOperand->getType(); 277122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVTy != OperTy) { 277222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) && 277322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick "cannot extend a chained IV"); 277422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IRBuilder<> Builder(InsertPt); 277522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain"); 277622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 277722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IncI->UserInst->replaceUsesOfWith(IncI->IVOperand, IVOper); 277822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DeadInsts.push_back(IncI->IVOperand); 277922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 278022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // If LSR created a new, wider phi, we may also replace its postinc. We only 278122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // do this if we also found a wide value for the head of the chain. 278270a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen if (isa<PHINode>(Chain.tailUserInst())) { 278322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (BasicBlock::iterator I = L->getHeader()->begin(); 278422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick PHINode *Phi = dyn_cast<PHINode>(I); ++I) { 278522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!isCompatibleIVType(Phi, IVSrc)) 278622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 278722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Instruction *PostIncV = dyn_cast<Instruction>( 278822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Phi->getIncomingValueForBlock(L->getLoopLatch())); 278922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc))) 279022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 279122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Value *IVOper = IVSrc; 279222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Type *PostIncTy = PostIncV->getType(); 279322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVTy != PostIncTy) { 279422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types"); 279522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IRBuilder<> Builder(L->getLoopLatch()->getTerminator()); 279622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc()); 279722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain"); 279822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 279922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Phi->replaceUsesOfWith(PostIncV, IVOper); 280022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick DeadInsts.push_back(PostIncV); 280122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 280222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 28036c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick} 28046c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick 2805572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CollectFixupsAndInitialFormulae() { 2806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 280722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Instruction *UserInst = UI->getUser(); 280822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick // Skip IV users that are part of profitable IV Chains. 280922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick User::op_iterator UseI = std::find(UserInst->op_begin(), UserInst->op_end(), 281022d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick UI->getOperandValToReplace()); 281122d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(UseI != UserInst->op_end() && "cannot find IV operand"); 281222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick if (IVIncSet.count(UseI)) 281322d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick continue; 281422d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick 2815572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Record the uses. 2816572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 281722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick LF.UserInst = UserInst; 2818572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI->getOperandValToReplace(); 2819448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.PostIncLoops = UI->getPostIncLoops(); 28200f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel 2821572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::KindType Kind = LSRUse::Basic; 2822db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *AccessTy = 0; 2823572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAddressUse(LF.UserInst, LF.OperandValToReplace)) { 2824572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::Address; 2825572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AccessTy = getAccessType(LF.UserInst); 2826572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2827572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2828c056454ecfe66f7c646fedef594f4ed48a9f3bf0Dan Gohman const SCEV *S = IU.getExpr(*UI); 2829572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2830572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Equality (== and !=) ICmps are special. We can rewrite (i == N) as 2831572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // (N - i == 0), and this allows (N - i) to be the expression that we work 2832572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with rather than just N or i, so we can consider the register 2833572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // requirements for both N and i at the same time. Limiting this code to 2834572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps is not a problem because all interesting loops use 2835572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // equality icmps, thanks to IndVarSimplify. 2836572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpInst *CI = dyn_cast<ICmpInst>(LF.UserInst)) 2837572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (CI->isEquality()) { 2838572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Swap the operands if needed to put the OperandValToReplace on the 2839572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // left, for consistency. 2840572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *NV = CI->getOperand(1); 2841572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NV == LF.OperandValToReplace) { 2842572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, CI->getOperand(0)); 2843572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(0, NV); 2844f182b23f8fed014c26da48d94890080c7d916a0cDan Gohman NV = CI->getOperand(1); 28459da1bf4845e670096b9bf9e62c40960af1697ea0Dan Gohman Changed = true; 2846572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2847572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2848572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // x == y --> x - y == 0 2849572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *N = SE.getSCEV(NV); 2850e08c32249fca32cd7b122024a4ca252fcb235694Andrew Trick if (SE.isLoopInvariant(N, L) && isSafeToExpand(N)) { 2851673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman // S is normalized, so normalize N before folding it into S 2852673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman // to keep the result normalized. 2853673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman N = TransformForPostIncUse(Normalize, N, CI, 0, 2854673968ae78f26bd78337d8bbb212fd280839fc12Dan Gohman LF.PostIncLoops, SE, DT); 2855572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Kind = LSRUse::ICmpZero; 2856572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman S = SE.getMinusSCEV(N, S); 2857572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2858572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2859572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // -1 and the negations of all interesting strides (except the negation 2860572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of -1) are now also interesting. 2861572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Factors.size(); i != e; ++i) 2862572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors[i] != -1) 2863572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-(uint64_t)Factors[i]); 2864572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.insert(-1); 2865572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2866572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2867572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Set up the initial formula for this use. 2868572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy); 2869572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 2870572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 2871572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 2872448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 2873a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 2874a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 2875a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 2876a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 2877572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2878572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is the first use of this LSRUse, give it a formula. 2879572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Formulae.empty()) { 2880454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman InsertInitialFormula(S, LU, LF.LUIdx); 2881572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), LF.LUIdx); 2882572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2883572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2884572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2885572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(print_fixups(dbgs())); 2886572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2887572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 288876c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertInitialFormula - Insert a formula for the given expression into 288976c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given use, separating out loop-variant portions from loop-invariant 289076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// and loop-computable portions. 2891572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2892454d26dc43207ec537d843229db6f5e6a302e23dDan GohmanLSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { 2893572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2894dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman F.InitialMatch(S, L, SE); 2895572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2896572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Initial formula already exists!"); (void)Inserted; 2897572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2898572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 289976c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// InsertSupplementalFormula - Insert a simple single-register formula for 290076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// the given expression into the given use. 2901572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2902572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::InsertSupplementalFormula(const SCEV *S, 2903572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU, size_t LUIdx) { 2904572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F; 2905572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(S); 2906572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.HasBaseReg = true; 2907572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Inserted = InsertFormula(LU, LUIdx, F); 2908572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(Inserted && "Supplemental formula already exists!"); (void)Inserted; 2909572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2910572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2911572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CountRegisters - Note which registers are used by the given formula, 2912572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// updating RegUses. 2913572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) { 2914572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) 2915572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(F.ScaledReg, LUIdx); 2916572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 2917572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) 2918572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.CountRegister(*I, LUIdx); 2919572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2920572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2921572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// InsertFormula - If the given formula has not yet been inserted, add it to 2922572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the list, and return true. Return false otherwise. 2923572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) { 2924454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman if (!LU.InsertFormula(F)) 29257979b72febb73f7bb1d1ed095a68f210822b2e7cDan Gohman return false; 292680b0f8c0628d848d36f04440687b61e3b2d3dff1Dan Gohman 2927572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(F, LUIdx); 2928572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return true; 2929572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 2930572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2931572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectLoopInvariantFixupsAndFormulae - Check for other uses of 2932572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-invariant values which we're tracking. These other uses will pin these 2933572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// values in registers, making them less profitable for elimination. 2934572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: This currently misses non-constant addrec step registers. 2935572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// TODO: Should this give more weight to users inside the loop? 2936572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 2937572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::CollectLoopInvariantFixupsAndFormulae() { 2938572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); 2939572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 8> Inserted; 2940572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2941572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (!Worklist.empty()) { 2942572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *S = Worklist.pop_back_val(); 2943572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2944572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S)) 2945403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman Worklist.append(N->op_begin(), N->op_end()); 2946572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S)) 2947572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(C->getOperand()); 2948572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) { 2949572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getLHS()); 2950572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(D->getRHS()); 2951572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 2952572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Inserted.insert(U)) continue; 2953572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Value *V = U->getValue(); 2954a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 2955a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Look for instructions defined outside the loop. 2956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (L->contains(Inst)) continue; 2957a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman } else if (isa<UndefValue>(V)) 2958a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman // Undef doesn't have a live range, so it doesn't matter. 2959a15ec5dfccace52bcfb31733e0412801a1ae3915Dan Gohman continue; 296060ad781c61815ca5b8dc2a45a102e1c8af65992fGabor Greif for (Value::const_use_iterator UI = V->use_begin(), UE = V->use_end(); 2961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UI != UE; ++UI) { 2962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Instruction *UserInst = dyn_cast<Instruction>(*UI); 2963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore non-instructions. 2964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!UserInst) 2965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions in other functions (as can happen with 2967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Constants). 2968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (UserInst->getParent()->getParent() != L->getHeader()->getParent()) 2969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2970572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore instructions not dominated by the loop. 2971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const BasicBlock *UseBB = !isa<PHINode>(UserInst) ? 2972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UserInst->getParent() : 2973572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman cast<PHINode>(UserInst)->getIncomingBlock( 2974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman PHINode::getIncomingValueNumForOperand(UI.getOperandNo())); 2975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DT.dominates(L->getHeader(), UseBB)) 2976572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore uses which are part of other SCEV expressions, to avoid 2978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // analyzing them multiple times. 29794a2a68336638e99a475b09a9278399db6749618fDan Gohman if (SE.isSCEVable(UserInst->getType())) { 29804a2a68336638e99a475b09a9278399db6749618fDan Gohman const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); 29814a2a68336638e99a475b09a9278399db6749618fDan Gohman // If the user is a no-op, look through to its uses. 29824a2a68336638e99a475b09a9278399db6749618fDan Gohman if (!isa<SCEVUnknown>(UserS)) 29834a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 29844a2a68336638e99a475b09a9278399db6749618fDan Gohman if (UserS == U) { 29854a2a68336638e99a475b09a9278399db6749618fDan Gohman Worklist.push_back( 29864a2a68336638e99a475b09a9278399db6749618fDan Gohman SE.getUnknown(const_cast<Instruction *>(UserInst))); 29874a2a68336638e99a475b09a9278399db6749618fDan Gohman continue; 29884a2a68336638e99a475b09a9278399db6749618fDan Gohman } 29894a2a68336638e99a475b09a9278399db6749618fDan Gohman } 2990572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore icmp instructions which are already being analyzed. 2991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) { 2992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned OtherIdx = !UI.getOperandNo(); 2993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx)); 299417ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L)) 2995572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 2996572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 2997572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 2998572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRFixup &LF = getNewFixup(); 2999572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst = const_cast<Instruction *>(UserInst); 3000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.OperandValToReplace = UI.getUse(); 3001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<size_t, int64_t> P = getUse(S, LSRUse::Basic, 0); 3002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.LUIdx = P.first; 3003572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.Offset = P.second; 3004572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LF.LUIdx]; 3005448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); 3006a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman if (!LU.WidestFixupType || 3007a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LU.WidestFixupType) < 3008a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) 3009a9db12973005881a1bd1c5eddc31001d9c3189c8Dan Gohman LU.WidestFixupType = LF.OperandValToReplace->getType(); 3010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman InsertSupplementalFormula(U, LU, LF.LUIdx); 3011572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CountRegisters(LU.Formulae.back(), Uses.size() - 1); 3012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 3013572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3014572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3015572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3016572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3017572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3018572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// CollectSubexprs - Split S into subexpressions which can be pulled out into 3019572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// separate registers. If C is non-null, multiply each subexpression by C. 302006a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick/// 302106a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick/// Return remainder expression after factoring the subexpressions captured by 302206a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick/// Ops. If Ops is complete, return NULL. 302306a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trickstatic const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C, 302406a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick SmallVectorImpl<const SCEV *> &Ops, 302506a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick const Loop *L, 302606a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick ScalarEvolution &SE, 302706a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick unsigned Depth = 0) { 302806a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick // Arbitrarily cap recursion to protect compile time. 302906a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (Depth >= 3) 303006a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick return S; 303106a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick 3032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 3033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break out add operands. 3034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); 303506a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick I != E; ++I) { 303606a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick const SCEV *Remainder = CollectSubexprs(*I, C, Ops, L, SE, Depth+1); 303706a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (Remainder) 303806a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); 303906a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick } 304006a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick return NULL; 3041572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { 3042572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Split a non-zero base out of an addrec. 304306a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (AR->getStart()->isZero()) 304406a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick return S; 304506a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick 304606a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick const SCEV *Remainder = CollectSubexprs(AR->getStart(), 304706a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick C, Ops, L, SE, Depth+1); 304806a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick // Split the non-zero AddRec unless it is part of a nested recurrence that 304906a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick // does not pertain to this loop. 305006a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (Remainder && (AR->getLoop() == L || !isa<SCEVAddRecExpr>(Remainder))) { 305106a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); 305206a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick Remainder = NULL; 305306a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick } 305406a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (Remainder != AR->getStart()) { 305506a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (!Remainder) 305606a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick Remainder = SE.getConstant(AR->getType(), 0); 305706a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick return SE.getAddRecExpr(Remainder, 305806a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick AR->getStepRecurrence(SE), 305906a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick AR->getLoop(), 306006a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick //FIXME: AR->getNoWrapFlags(SCEV::FlagNW) 306106a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick SCEV::FlagAnyWrap); 3062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3064572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Break (C * (a + b + c)) into C*a + C*b + C*c. 306506a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (Mul->getNumOperands() != 2) 306606a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick return S; 306706a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (const SCEVConstant *Op0 = 306806a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick dyn_cast<SCEVConstant>(Mul->getOperand(0))) { 306906a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0; 307006a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick const SCEV *Remainder = 307106a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1); 307206a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (Remainder) 307306a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick Ops.push_back(SE.getMulExpr(C, Remainder)); 307406a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick return NULL; 307506a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick } 3076572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 307706a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick return S; 3078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3079572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3080572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateReassociations - Split out subexpressions from adds and the bases of 3081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// addrecs. 3082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx, 3083572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base, 3084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Depth) { 3085572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Arbitrarily cap recursion to protect compile time. 3086572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Depth >= 3) return; 3087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3088572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 3089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = Base.BaseRegs[i]; 3090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 30913e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman SmallVector<const SCEV *, 8> AddOps; 309206a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick const SCEV *Remainder = CollectSubexprs(BaseReg, 0, AddOps, L, SE); 309306a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick if (Remainder) 309406a27cc1aae20e3a2c985dd8dadab3b4ef5d032aAndrew Trick AddOps.push_back(Remainder); 30953e3f15bb09eaca4d259e531c9a1ecafb5710b57bDan Gohman 3096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (AddOps.size() == 1) continue; 3097572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), 3099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = AddOps.end(); J != JE; ++J) { 31003e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 31013e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // Loop-variant "unknown" values are uninteresting; we won't be able to 31023e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman // do anything meaningful with them. 310317ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L)) 31043e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman continue; 31053e22b7c91698f55b9053e88a168bd9e2eed71c9bDan Gohman 3106572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't pull a constant into a register if the constant could be folded 3107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // into an immediate field. 3108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isAlwaysFoldable(*J, LU.MinOffset, LU.MaxOffset, 3109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 3110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 3111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect all operands except *J. 3114403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman SmallVector<const SCEV *, 8> InnerAddOps 31154eaee28e347f33b3e23d2fb7275fc5222d5482fdDan Gohman (((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); 3116403a8cdda5e76ea689693de16474650b4b0df818Dan Gohman InnerAddOps.append 3117ee56c42168f6c4271593f6018c4409b6a5910302Oscar Fuentes (llvm::next(J), ((const SmallVector<const SCEV *, 8> &)AddOps).end()); 3118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3119572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't leave just a constant behind in a register if the constant could 3120572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // be folded into an immediate field. 3121572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InnerAddOps.size() == 1 && 3122572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isAlwaysFoldable(InnerAddOps[0], LU.MinOffset, LU.MaxOffset, 3123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.getNumRegs() > 1, 3124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI, SE)) 3125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3127fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); 3128fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman if (InnerSum->isZero()) 3129fafb890ee204d60456d0780ff55a149fa082eaeaDan Gohman continue; 3130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3131cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3132cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Add the remaining pieces of the add back into the new formula. 3133cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum); 3134cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (TLI && InnerSumSC && 3135cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 && 3136cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman TLI->isLegalAddImmediate((uint64_t)F.UnfoldedOffset + 3137cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman InnerSumSC->getValue()->getZExtValue())) { 3138cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset + 3139cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman InnerSumSC->getValue()->getZExtValue(); 3140cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs.erase(F.BaseRegs.begin() + i); 3141cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } else 3142cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs[i] = InnerSum; 3143cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3144cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Add J as its own register, or an unfolded immediate. 3145cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J); 3146cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (TLI && SC && SE.getTypeSizeInBits(SC->getType()) <= 64 && 3147cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman TLI->isLegalAddImmediate((uint64_t)F.UnfoldedOffset + 3148cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SC->getValue()->getZExtValue())) 3149cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset + 3150cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman SC->getValue()->getZExtValue(); 3151cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman else 3152cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.BaseRegs.push_back(*J); 3153cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3154572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (InsertFormula(LU, LUIdx, F)) 3155572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If that formula hadn't been seen before, recurse to find more like 3156572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // it. 3157572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth+1); 3158572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3159572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3160572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3161572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3162572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCombinations - Generate a formula consisting of all of the 3163572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// loop-dominating registers added into a single register. 3164572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx, 3165441a38993e89104c424e00c95cce63c7351f4fc3Dan Gohman Formula Base) { 31663f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // This method is only interesting on a plurality of registers. 3167572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.BaseRegs.size() <= 1) return; 3168572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3169572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3170572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.clear(); 3171572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 4> Ops; 3172572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 3173572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Base.BaseRegs.begin(), E = Base.BaseRegs.end(); I != E; ++I) { 3174572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = *I; 3175dc0e8fb9f9512622f55f73e1a434caa5c0915694Dan Gohman if (SE.properlyDominates(BaseReg, L->getHeader()) && 317617ead4ff4baceb2c5503f233d0288d363ae44165Dan Gohman !SE.hasComputableLoopEvolution(BaseReg, L)) 3177572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(BaseReg); 3178572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 3179572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs.push_back(BaseReg); 3180572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3181572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Ops.size() > 1) { 3182ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman const SCEV *Sum = SE.getAddExpr(Ops); 3183ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // TODO: If Sum is zero, it probably means ScalarEvolution missed an 3184ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman // opportunity to fold something. For now, just ignore such cases 31853f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // rather than proceed with zero in a register. 3186ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman if (!Sum->isZero()) { 3187ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman F.BaseRegs.push_back(Sum); 3188ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3189ce947366ec07ed3e9b017f0f4a07fa668a799119Dan Gohman } 3190572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3191572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3192572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3193572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateSymbolicOffsets - Generate reuse formulae using symbolic offsets. 3194572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, 3195572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 3196572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We can't add a symbolic offset if the address already contains one. 3197572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 3198572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3199572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 3200572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 3201572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GlobalValue *GV = ExtractSymbol(G, SE); 3202572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || !GV) 3203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseGV = GV; 3206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 3207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 3208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 3210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3211572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3212572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3213572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3214572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets. 3215572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, 3216572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 3217572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: For now, just add the min and max offset, because it usually isn't 3218572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // worthwhile looking at everything inbetween. 3219c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman SmallVector<int64_t, 2> Worklist; 3220572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MinOffset); 3221572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MaxOffset != LU.MinOffset) 3222572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Worklist.push_back(LU.MaxOffset); 3223572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3224572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { 3225572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *G = Base.BaseRegs[i]; 3226572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3227572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<int64_t>::const_iterator I = Worklist.begin(), 3228572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Worklist.end(); I != E; ++I) { 3229572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3230572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)Base.AM.BaseOffs - *I; 3231572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (isLegalUse(F.AM, LU.MinOffset - *I, LU.MaxOffset - *I, 3232572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 3233c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // Add the offset to the base register. 32344065f609128ea4cdfa575f48b816d1cc64e710e0Dan Gohman const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G); 3235c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman // If it cancelled out, drop the base register, otherwise update it. 3236c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman if (NewG->isZero()) { 3237c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman std::swap(F.BaseRegs[i], F.BaseRegs.back()); 3238c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs.pop_back(); 3239c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman } else 3240c88c1a4581a4c73657892ef4ed72f1c9f72ed7ffDan Gohman F.BaseRegs[i] = NewG; 3241572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3242572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3243572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3245572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3246572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(G, SE); 3247572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (G->isZero() || Imm == 0) 3248572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3249572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3250572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Imm; 3251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, LU.MinOffset, LU.MaxOffset, 3252572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 3253572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3254572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = G; 3255572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3256572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3257572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3258572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3259572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateICmpZeroScales - For ICmpZero, check to see if we can scale up 3260572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the comparison. For example, x == y -> x*c == y*c. 3261572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, 3262572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula Base) { 3263572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind != LSRUse::ICmpZero) return; 3264572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3265572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 3266db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = Base.getType(); 3267572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 3268572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SE.getTypeSizeInBits(IntTy) > 64) return; 3269572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3270572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't do this if there is more than one offset. 3271572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.MinOffset != LU.MaxOffset) return; 3272572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Base.AM.BaseGV && "ICmpZero use is not legal!"); 3274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3275572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 3276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 3277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 3278572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 3279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3280572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that the multiplication doesn't overflow. 32812ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (Base.AM.BaseOffs == INT64_MIN && Factor == -1) 3282968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 32832ea09e05466613a22e1211f52c30cd01af563983Dan Gohman int64_t NewBaseOffs = (uint64_t)Base.AM.BaseOffs * Factor; 32842ea09e05466613a22e1211f52c30cd01af563983Dan Gohman if (NewBaseOffs / Factor != Base.AM.BaseOffs) 3285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the use offset doesn't overflow. 3288572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = LU.MinOffset; 3289968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman if (Offset == INT64_MIN && Factor == -1) 3290968cb939e5a00cb06aefafc89581645790c590b3Dan Gohman continue; 3291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Offset = (uint64_t)Offset * Factor; 3292378c0b35a7b9884f1de2a9762825424fbf1813acDan Gohman if (Offset / Factor != LU.MinOffset) 3293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 32952ea09e05466613a22e1211f52c30cd01af563983Dan Gohman Formula F = Base; 32962ea09e05466613a22e1211f52c30cd01af563983Dan Gohman F.AM.BaseOffs = NewBaseOffs; 32972ea09e05466613a22e1211f52c30cd01af563983Dan Gohman 3298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that this scale is legal. 3299572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(F.AM, Offset, Offset, LU.Kind, LU.AccessTy, TLI)) 3300572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3301572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3302572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compensate for the use having MinOffset built into it. 3303572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs + Offset - LU.MinOffset; 3304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3305deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 3306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with each base register doesn't overflow. 3308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) { 3309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS); 3310f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i]) 3311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto next; 3312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3313572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3314572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check that multiplying with the scaled register doesn't overflow. 3315572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) { 3316572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS); 3317f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg) 3318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3321cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Check that multiplying with the unfolded offset doesn't overflow. 3322cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (F.UnfoldedOffset != 0) { 33231b58d4536a561f28bf935dcb29b483c52a6bf4c6Dan Gohman if (F.UnfoldedOffset == INT64_MIN && Factor == -1) 33241b58d4536a561f28bf935dcb29b483c52a6bf4c6Dan Gohman continue; 3325cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset * Factor; 3326cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (F.UnfoldedOffset / Factor != Base.UnfoldedOffset) 3327cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman continue; 3328cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 3329cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 3330572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If we make it here and it's legal, add it. 3331572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3332572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman next:; 3333572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3334572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3335572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateScales - Generate stride factor reuse formulae by making use of 3337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// scaled-offset address modes, for example. 3338ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) { 3339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 3340db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = Base.getType(); 3341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!IntTy) return; 3342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3343572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this Formula already has a scaled register, we can't add another one. 3344572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.Scale != 0) return; 3345572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3346572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check each interesting stride. 3347572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 3348572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 3349572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Factor = *I; 3350572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3351572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.Scale = Factor; 3352572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Base.AM.HasBaseReg = Base.BaseRegs.size() > 1; 3353572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Check whether this scale is going to be legal. 3354572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 3355572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 3356572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // As a special-case, handle special out-of-loop Basic users specially. 3357572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: Reconsider this special case. 3358572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::Basic && 3359572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman isLegalUse(Base.AM, LU.MinOffset, LU.MaxOffset, 3360572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse::Special, LU.AccessTy, TLI) && 3361572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.AllFixupsOutsideLoop) 3362572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind = LSRUse::Special; 3363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 3364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For an ICmpZero, negating a solitary base register won't lead to 3367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // new solutions. 3368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero && 3369572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman !Base.AM.HasBaseReg && Base.AM.BaseOffs == 0 && !Base.AM.BaseGV) 3370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // For each addrec base reg, apply the scale, if possible. 3372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) 3373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVAddRecExpr *AR = 3374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i])) { 3375deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman const SCEV *FactorS = SE.getConstant(IntTy, Factor); 3376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FactorS->isZero()) 3377572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Divide out the factor, ignoring high bits, since we'll be 3379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scaling the value back up in the end. 3380f09b71233bc43cfcca5199e355cc1129f7c4f89aDan Gohman if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) { 3381572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This could be optimized to avoid all the copying. 3382572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3383572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman F.ScaledReg = Quotient; 33845ce6d05ad6b500a6262211674e51b8d0598714d3Dan Gohman F.DeleteBaseReg(F.BaseRegs[i]); 3385572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3386572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3387572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3388572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3391572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateTruncates - Generate reuse formulae from different IV types. 3392ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohmanvoid LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) { 3393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This requires TargetLowering to tell us which truncates are free. 3394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!TLI) return; 3395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3396572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't bother truncating symbolic values. 3397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Base.AM.BaseGV) return; 3398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Determine the integer type for the base formula. 3400db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *DstTy = Base.getType(); 3401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!DstTy) return; 3402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DstTy = SE.getEffectiveSCEVType(DstTy); 3403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3404db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner for (SmallSetVector<Type *, 4>::const_iterator 3405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 3406db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *SrcTy = *I; 3407572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (SrcTy != DstTy && TLI->isTruncateFree(SrcTy, DstTy)) { 3408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula F = Base; 3409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, *I); 3411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(), 3412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = F.BaseRegs.end(); J != JE; ++J) 3413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman *J = SE.getAnyExtendExpr(*J, SrcTy); 3414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3415572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // TODO: This assumes we've done basic processing on all uses and 3416572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // have an idea what the register usage is. 3417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses)) 3418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, F); 3421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3422572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3423572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3424572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3425572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 3426572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 34276020d85c41987b0b7890d91bf66187aac6e8a3a1Dan Gohman/// WorkItem - Helper class for GenerateCrossUseConstantOffsets. It's used to 3428572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// defer modifications so that the search phase doesn't have to worry about 3429572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the data structures moving underneath it. 3430572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanstruct WorkItem { 3431572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx; 3432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm; 3433572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg; 3434572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3435572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WorkItem(size_t LI, int64_t I, const SCEV *R) 3436572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : LUIdx(LI), Imm(I), OrigReg(R) {} 3437572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3438572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void print(raw_ostream &OS) const; 3439572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void dump() const; 3440572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 3441572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3442572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3443572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3444572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::print(raw_ostream &OS) const { 3445572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx 3446572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman << " , add offset " << Imm; 3447572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3448572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3449cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#ifndef NDEBUG 3450572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid WorkItem::dump() const { 3451572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 3452572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3453cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#endif 3454572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3455572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateCrossUseConstantOffsets - Look for registers which are a constant 3456572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// distance apart and try to form reuse opportunities between them. 3457572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::GenerateCrossUseConstantOffsets() { 3458572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Group the registers by their value without any added constant offset. 3459572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef std::map<int64_t, const SCEV *> ImmMapTy; 3460572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy; 3461572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegMapTy Map; 3462572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; 3463572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Sequence; 3464572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 3465572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 3466572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3467572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = ExtractImmediate(Reg, SE); 3468572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::pair<RegMapTy::iterator, bool> Pair = 3469572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.insert(std::make_pair(Reg, ImmMapTy())); 3470572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Pair.second) 3471572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.push_back(Reg); 3472572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pair.first->second.insert(std::make_pair(Imm, *I)); 3473572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(*I); 3474572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3475572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3476572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now examine each set of registers with the same base value. Build up 3477572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // a list of work to do and do the work in a separate step so that we're 3478572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // not adding formulae and register counts while we're searching. 3479191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallVector<WorkItem, 32> WorkItems; 3480191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems; 3481572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(), 3482572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Sequence.end(); I != E; ++I) { 3483572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3484572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const ImmMapTy &Imms = Map.find(Reg)->second; 3485572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3486cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman // It's not worthwhile looking for reuse if there's only one offset. 3487cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (Imms.size() == 1) 3488cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman continue; 3489cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman 3490572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':'; 3491572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 3492572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 3493572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << J->first; 3494572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 3495572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3496572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Examine each offset. 3497572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); 3498572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) { 3499572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = J->second; 3500572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3501572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t JImm = J->first; 3502572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg); 3503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isa<SCEVConstant>(OrigReg) && 3505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap[Reg].count() == 1) { 3506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n'); 3507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Conservatively examine offsets between this orig reg a few selected 3511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // other orig regs. 3512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator OtherImms[] = { 3513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imms.begin(), prior(Imms.end()), 3514cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Imms.lower_bound((Imms.begin()->first + prior(Imms.end())->first) / 2) 3515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 3516572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = array_lengthof(OtherImms); i != e; ++i) { 3517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImmMapTy::const_iterator M = OtherImms[i]; 3518cd045c08cad9bc3e1e3e234453f5f4464b705e02Dan Gohman if (M == J || M == JE) continue; 3519572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Compute the difference between the two. 3521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = (uint64_t)JImm - M->first; 3522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (int LUIdx = UsedByIndices.find_first(); LUIdx != -1; 3523191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LUIdx = UsedByIndices.find_next(LUIdx)) 3524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Make a memo of this use, offset, and register tuple. 3525191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (UniqueItems.insert(std::make_pair(LUIdx, Imm))) 3526191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg)); 3527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Map.clear(); 3532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Sequence.clear(); 3533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman UsedByIndicesMap.clear(); 3534191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman UniqueItems.clear(); 3535572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now iterate through the worklist and add new formulae. 3537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<WorkItem>::const_iterator I = WorkItems.begin(), 3538572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = WorkItems.end(); I != E; ++I) { 3539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const WorkItem &WI = *I; 3540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman size_t LUIdx = WI.LUIdx; 3541572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3542572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Imm = WI.Imm; 3543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *OrigReg = WI.OrigReg; 3544572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3545db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType()); 3546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); 3547572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BitWidth = SE.getTypeSizeInBits(IntTy); 3548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 35493f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // TODO: Use a more targeted data structure. 3550572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) { 35519f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman const Formula &F = LU.Formulae[L]; 3552572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in the scaled register. 3553572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.ScaledReg == OrigReg) { 3554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offs = (uint64_t)F.AM.BaseOffs + 3555572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Imm * (uint64_t)F.AM.Scale; 3556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Don't create 50 + reg(-50). 3557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.referencesReg(SE.getSCEV( 3558572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ConstantInt::get(IntTy, -(uint64_t)Offs)))) 3559572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3560572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 3561572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = Offs; 3562572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 3563572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI)) 3564572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg); 3566572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3567572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new scale is a constant in a register, and adding the constant 3568572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // value to the immediate would produce a value closer to zero than the 3569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // immediate itself, then the formula isn't worthwhile. 3570572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg)) 3571c73b24db5f6226ed44ebc44ce1c25bb357206623Chris Lattner if (C->getValue()->isNegative() != 3572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (NewF.AM.BaseOffs < 0) && 3573572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (C->getValue()->getValue().abs() * APInt(BitWidth, F.AM.Scale)) 3574e05678132345eb8a632362dbd320ee7d36226e67Dan Gohman .ule(abs64(NewF.AM.BaseOffs))) 3575572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3576572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3577572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // OK, looks good. 3578572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 3579572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 3580572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use the immediate in a base register. 3581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) { 3582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *BaseReg = F.BaseRegs[N]; 3583572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (BaseReg != OrigReg) 3584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula NewF = F; 3586572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.AM.BaseOffs = (uint64_t)NewF.AM.BaseOffs + Imm; 3587572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!isLegalUse(NewF.AM, LU.MinOffset, LU.MaxOffset, 3588cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman LU.Kind, LU.AccessTy, TLI)) { 3589cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (!TLI || 3590cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman !TLI->isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm)) 3591cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman continue; 3592cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NewF = F; 3593cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman NewF.UnfoldedOffset = (uint64_t)NewF.UnfoldedOffset + Imm; 3594cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 3595572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg); 3596572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3597572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If the new formula has a constant in a register, and adding the 3598572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // constant value to the immediate would produce a value closer to 3599572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // zero than the immediate itself, then the formula isn't worthwhile. 3600572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 3601572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J = NewF.BaseRegs.begin(), JE = NewF.BaseRegs.end(); 3602572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J != JE; ++J) 3603572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*J)) 3604360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman if ((C->getValue()->getValue() + NewF.AM.BaseOffs).abs().slt( 3605360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman abs64(NewF.AM.BaseOffs)) && 3606360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman (C->getValue()->getValue() + 3607360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman NewF.AM.BaseOffs).countTrailingZeros() >= 3608360026f07fcf35eee9fcfbf7b9c1afda41d4b148Dan Gohman CountTrailingZeros_64(NewF.AM.BaseOffs)) 3609572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman goto skip_formula; 3610572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3611572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, looks good. 3612572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman (void)InsertFormula(LU, LUIdx, NewF); 3613572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman break; 3614572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman skip_formula:; 3615572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3616572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3617572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3618572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3619572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3620572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3621572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// GenerateAllReuseFormulae - Generate formulae for each use. 3622572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 3623572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::GenerateAllReuseFormulae() { 3624c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman // This is split into multiple loops so that hasRegsUsedByUsesOtherThan 3625572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // queries are more precise. 3626572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3627572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateReassociations(LU, LUIdx, LU.Formulae[i]); 3630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3631572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCombinations(LU, LUIdx, LU.Formulae[i]); 3632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3633572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3635572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]); 3637572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3638572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]); 3639572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3640572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]); 3641572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3642572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateScales(LU, LUIdx, LU.Formulae[i]); 3643c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman } 3644c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3645c2385a0741c43bd93eb2033e2f11eaae83cdb1cbDan Gohman LSRUse &LU = Uses[LUIdx]; 3646572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) 3647572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateTruncates(LU, LUIdx, LU.Formulae[i]); 3648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3650572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateCrossUseConstantOffsets(); 36513902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman 36523902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman DEBUG(dbgs() << "\n" 36533902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman "After generating reuse formulae:\n"; 36543902f9fca8dd69ca3c5fce564deff817d7db3a8bDan Gohman print_uses(dbgs())); 3655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3657f63d70f218807d7522e8bbe2d9e887ff3ea87b24Dan Gohman/// If there are multiple formulae with the same set of registers used 3658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// by other uses, pick the best one and delete the others. 3659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::FilterOutUndesirableDedicatedRegisters() { 3660fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman DenseSet<const SCEV *> VisitedRegs; 3661fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman SmallPtrSet<const SCEV *, 16> Regs; 36628a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallPtrSet<const SCEV *, 16> LoserRegs; 3663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 3664c6519f916b5922de81c53547fd21364994195a70Dan Gohman bool ChangedFormulae = false; 3665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 3666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Collect the best formula for each unique set of shared registers. This 3668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // is reset for each use. 3669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman typedef DenseMap<SmallVector<const SCEV *, 2>, size_t, UniquifierDenseMapInfo> 3670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy; 3671572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulaeTy BestFormulae; 3672572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3673572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LSRUse &LU = Uses[LUIdx]; 3675ea507f5c28e4c21addb4022a6a2ab85f49f172e3Dan Gohman DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n'); 3676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3677b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 3678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t FIdx = 0, NumForms = LU.Formulae.size(); 3679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FIdx != NumForms; ++FIdx) { 3680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[FIdx]; 3681572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 36828a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // Some formulas are instant losers. For example, they may depend on 36838a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // nonexistent AddRecs from other loops. These need to be filtered 36848a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // immediately, otherwise heuristics could choose them over others leading 36858a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // to an unsatisfactory solution. Passing LoserRegs into RateFormula here 36868a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // avoids the need to recompute this information across formulae using the 36878a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // same bad AddRec. Passing LoserRegs is also essential unless we remove 36888a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // the corresponding bad register from the Regs set. 36898a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Cost CostF; 36908a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Regs.clear(); 36918a5d792944582de8e63e96440dbd2cde754351adAndrew Trick CostF.RateFormula(F, Regs, VisitedRegs, L, LU.Offsets, SE, DT, 36928a5d792944582de8e63e96440dbd2cde754351adAndrew Trick &LoserRegs); 36938a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (CostF.isLoser()) { 36948a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // During initial formula generation, undesirable formulae are generated 36958a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // by uses within other loops that have some non-trivial address mode or 36968a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // use the postinc form of the IV. LSR needs to provide these formulae 36978a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // as the basis of rediscovering the desired formula that uses an AddRec 36988a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // corresponding to the existing phi. Once all formulae have been 36998a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // generated, these initial losers may be pruned. 37008a5d792944582de8e63e96440dbd2cde754351adAndrew Trick DEBUG(dbgs() << " Filtering loser "; F.print(dbgs()); 37018a5d792944582de8e63e96440dbd2cde754351adAndrew Trick dbgs() << "\n"); 3702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 37038a5d792944582de8e63e96440dbd2cde754351adAndrew Trick else { 37048a5d792944582de8e63e96440dbd2cde754351adAndrew Trick SmallVector<const SCEV *, 2> Key; 37058a5d792944582de8e63e96440dbd2cde754351adAndrew Trick for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(), 37068a5d792944582de8e63e96440dbd2cde754351adAndrew Trick JE = F.BaseRegs.end(); J != JE; ++J) { 37078a5d792944582de8e63e96440dbd2cde754351adAndrew Trick const SCEV *Reg = *J; 37088a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx)) 37098a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Key.push_back(Reg); 37108a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 37118a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (F.ScaledReg && 37128a5d792944582de8e63e96440dbd2cde754351adAndrew Trick RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx)) 37138a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Key.push_back(F.ScaledReg); 37148a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // Unstable sort by host order ok, because this is only used for 37158a5d792944582de8e63e96440dbd2cde754351adAndrew Trick // uniquifying. 37168a5d792944582de8e63e96440dbd2cde754351adAndrew Trick std::sort(Key.begin(), Key.end()); 37178a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 37188a5d792944582de8e63e96440dbd2cde754351adAndrew Trick std::pair<BestFormulaeTy::const_iterator, bool> P = 37198a5d792944582de8e63e96440dbd2cde754351adAndrew Trick BestFormulae.insert(std::make_pair(Key, FIdx)); 37208a5d792944582de8e63e96440dbd2cde754351adAndrew Trick if (P.second) 37218a5d792944582de8e63e96440dbd2cde754351adAndrew Trick continue; 37228a5d792944582de8e63e96440dbd2cde754351adAndrew Trick 3723572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &Best = LU.Formulae[P.first->second]; 3724fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman 3725fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Cost CostBest; 3726fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman Regs.clear(); 37278a5d792944582de8e63e96440dbd2cde754351adAndrew Trick CostBest.RateFormula(Best, Regs, VisitedRegs, L, LU.Offsets, SE, DT); 3728fc7744b12d7de51f0cda300d939820d06bc8d087Dan Gohman if (CostF < CostBest) 3729572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::swap(F, Best); 37306458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs()); 3731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 37326458ff9230a53bb9c442559074fac2611fb70bbfDan Gohman " in favor of formula "; Best.print(dbgs()); 3733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 37348a5d792944582de8e63e96440dbd2cde754351adAndrew Trick } 3735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 37368a5d792944582de8e63e96440dbd2cde754351adAndrew Trick ChangedFormulae = true; 3737572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 37388a5d792944582de8e63e96440dbd2cde754351adAndrew Trick LU.DeleteFormula(F); 37398a5d792944582de8e63e96440dbd2cde754351adAndrew Trick --FIdx; 37408a5d792944582de8e63e96440dbd2cde754351adAndrew Trick --NumForms; 37418a5d792944582de8e63e96440dbd2cde754351adAndrew Trick Any = true; 374259dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman } 374359dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 374457aaa0b264d9d23d9a14235334b15d4117e32a3bDan Gohman // Now that we've filtered out some formulae, recompute the Regs set. 3745b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 3746b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 374759dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman 374859dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman // Reset this to prepare for the next use. 3749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestFormulae.clear(); 3750572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3752c6519f916b5922de81c53547fd21364994195a70Dan Gohman DEBUG(if (ChangedFormulae) { 37539214b82c5446791b280821bbd892dca633130f80Dan Gohman dbgs() << "\n" 37549214b82c5446791b280821bbd892dca633130f80Dan Gohman "After filtering out undesirable candidates:\n"; 3755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs()); 3756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 3757572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 3758572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3759d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman// This is a rough guess that seems to work fairly well. 3760d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmanstatic const size_t ComplexityLimit = UINT16_MAX; 3761d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 3762d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// EstimateSearchSpaceComplexity - Estimate the worst-case number of 3763d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// solutions the solver might have to consider. It almost never considers 3764d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// this many solutions because it prune the search space, but the pruning 3765d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman/// isn't always sufficient. 3766d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohmansize_t LSRInstance::EstimateSearchSpaceComplexity() const { 37670d6715a413a23c021d1042719888966bb8d11872Dan Gohman size_t Power = 1; 3768d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 3769d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman E = Uses.end(); I != E; ++I) { 3770d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman size_t FSize = I->Formulae.size(); 3771d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (FSize >= ComplexityLimit) { 3772d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power = ComplexityLimit; 3773d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 3774d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 3775d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman Power *= FSize; 3776d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman if (Power >= ComplexityLimit) 3777d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman break; 3778d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman } 3779d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman return Power; 3780d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman} 3781d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman 37824aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByDetectingSupersets - When one formula uses a superset 37834aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of the registers of another formula, it won't help reduce register 37844aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// pressure (though it may not necessarily hurt register pressure); remove 37854aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// it to simplify the system. 37864aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByDetectingSupersets() { 3787a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3788a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3789a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3790a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by eliminating formulae " 3791a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "which use a superset of registers used by other " 3792a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "formulae.\n"); 3793a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3794a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3795a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 3796a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman bool Any = false; 3797a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 3798a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula &F = LU.Formulae[i]; 3799f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // Look for a formula with a constant or GV in a register. If the use 3800f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // also has a formula with that same value in an immediate field, 3801f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // delete the one that uses a register. 3802a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator 3803a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) { 3804a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) { 3805a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 3806a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseOffs += C->getValue()->getSExtValue(); 3807a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 3808a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 3809a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 3810a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 3811a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 3812a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 3813a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 3814a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 3815a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3816a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3817a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) { 3818a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) 3819a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (!F.AM.BaseGV) { 3820a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Formula NewF = F; 3821a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.AM.BaseGV = GV; 3822a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman NewF.BaseRegs.erase(NewF.BaseRegs.begin() + 3823a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman (I - F.BaseRegs.begin())); 3824a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (LU.HasFormulaWithSameRegs(NewF)) { 3825a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 3826a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 3827a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.DeleteFormula(F); 3828a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --i; 3829a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --e; 3830a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman Any = true; 3831a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3832a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3833a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3834a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3835a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3836a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3837a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (Any) 3838a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.RecomputeRegs(LUIdx, RegUses); 3839a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3840a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3841a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3842a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 3843a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 38444aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 3845a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 38464aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByCollapsingUnrolledCode - When there are many registers 38474aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// for expressions like A, A+1, A+2, etc., allocate a single register for 38484aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// them. 38494aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() { 3850a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3851a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3852a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3853a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming that uses " 3854a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "separated by a constant offset will use the same " 3855a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman "registers.\n"); 3856a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3857f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // This is especially useful for unrolled loops. 3858f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman 3859a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3860a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LSRUse &LU = Uses[LUIdx]; 3861402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 3862402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 3863402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const Formula &F = *I; 3864a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman if (F.AM.BaseOffs != 0 && F.AM.Scale == 0) { 3865191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU)) { 3866191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (reconcileNewOffset(*LUThatHas, F.AM.BaseOffs, 3867a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman /*HasBaseReg=*/false, 3868a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LU.Kind, LU.AccessTy)) { 3869a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); 3870a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman dbgs() << '\n'); 3871a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3872a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop; 3873a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3874191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman // Update the relocs to reference the new use. 3875191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman for (SmallVectorImpl<LSRFixup>::iterator I = Fixups.begin(), 3876191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman E = Fixups.end(); I != E; ++I) { 3877191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman LSRFixup &Fixup = *I; 3878191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == LUIdx) { 3879191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUThatHas - &Uses.front(); 3880191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.Offset += F.AM.BaseOffs; 3881dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman // Add the new offset to LUThatHas' offset list. 3882dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (LUThatHas->Offsets.back() != Fixup.Offset) { 3883dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->Offsets.push_back(Fixup.Offset); 3884dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset > LUThatHas->MaxOffset) 3885dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MaxOffset = Fixup.Offset; 3886dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman if (Fixup.Offset < LUThatHas->MinOffset) 3887dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman LUThatHas->MinOffset = Fixup.Offset; 3888dd3db0e0c33958328cc709f01aba45c8d9e57f1fDan Gohman } 3889191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman DEBUG(dbgs() << "New fixup has offset " 3890191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman << Fixup.Offset << '\n'); 3891191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3892191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman if (Fixup.LUIdx == NumUses-1) 3893191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman Fixup.LUIdx = LUIdx; 3894191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman } 3895191bd64a39490fa75d35b9aaecdd57b00c7a8b5fDan Gohman 3896c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman // Delete formulae from the new use which are no longer legal. 3897c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman bool Any = false; 3898c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) { 3899c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Formula &F = LUThatHas->Formulae[i]; 3900c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (!isLegalUse(F.AM, 3901c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->MinOffset, LUThatHas->MaxOffset, 3902c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->Kind, LUThatHas->AccessTy, TLI)) { 3903c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); 3904c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman dbgs() << '\n'); 3905c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->DeleteFormula(F); 3906c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --i; 3907c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman --e; 3908c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman Any = true; 3909c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3910c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman } 3911c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman if (Any) 3912c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses); 3913c2921ea840a0c87f6c4f065bc775a9060e2aa626Dan Gohman 3914a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman // Delete the old use. 3915c6897706b7c3796ac24535c9ea1449c0411f8c7aDan Gohman DeleteUse(LU, LUIdx); 3916a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --LUIdx; 3917a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman --NumUses; 3918a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman break; 3919a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3920a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3921a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3922a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3923a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 3924a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 3925a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 3926a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman print_uses(dbgs())); 3927a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman } 39284aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 3929a2086b3483b88b5b64f098b6644e450f94933f49Dan Gohman 39303228cc259b5ca00e46af36da369a451f5736cbf4Andrew Trick/// NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters - Call 39314f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// FilterOutUndesirableDedicatedRegisters again, if necessary, now that 39324f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// we've done more filtering, as it may be able to find more formulae to 39334f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman/// eliminate. 39344f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohmanvoid LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){ 39354f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 39364f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 39374f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 39384f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "Narrowing the search space by re-filtering out " 39394f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman "undesirable dedicated registers.\n"); 39404f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 39414f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman FilterOutUndesirableDedicatedRegisters(); 39424f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 39434f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 39444f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman print_uses(dbgs())); 39454f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman } 39464f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman} 39474f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman 39484aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceByPickingWinnerRegs - Pick a register which seems likely 39494aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// to be profitable, and then in any use which has any reference to that 39504aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// register, delete all formulae which do not reference that register. 39514aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() { 395276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // With all other options exhausted, loop until the system is simple 395376c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman // enough to handle. 3954572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 4> Taken; 3955d079c300ed138fb391b2d1c955f36311c92aeaeaDan Gohman while (EstimateSearchSpaceComplexity() >= ComplexityLimit) { 3956572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we have too many of formulae on our hands to conveniently handle. 3957572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Use a rough heuristic to thin out the list. 39580da751baf72542ab66518e4549e39da5f34216b4Dan Gohman DEBUG(dbgs() << "The search space is too complex.\n"); 3959572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3960572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Pick the register which is used by the most LSRUses, which is likely 3961572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to be a good reuse register candidate. 3962572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Best = 0; 3963572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned BestNum = 0; 3964572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end(); 3965572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I != E; ++I) { 3966572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 3967572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Taken.count(Reg)) 3968572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 3969572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Best) 3970572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3971572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 3972572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman unsigned Count = RegUses.getUsedByIndices(Reg).count(); 3973572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Count > BestNum) { 3974572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Best = Reg; 3975572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman BestNum = Count; 3976572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3977572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3978572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 3979572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3980572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best 39813f46a3abeedba8d517b4182de34c821d752db058Dan Gohman << " will yield profitable reuse.\n"); 3982572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Taken.insert(Best); 3983572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3984572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // In any use with formulae which references this register, delete formulae 3985572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which don't reference it. 3986b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { 3987b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LSRUse &LU = Uses[LUIdx]; 3988572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!LU.Regs.count(Best)) continue; 3989572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 3990b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman bool Any = false; 3991572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { 3992572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Formula &F = LU.Formulae[i]; 3993572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!F.referencesReg(Best)) { 3994572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n'); 3995d69d62833a66691e96ad2998450cf8c9f75a2e8cDan Gohman LU.DeleteFormula(F); 3996572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --e; 3997572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman --i; 3998b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman Any = true; 399959dc60337fb9c02c4fbec3b44d7275a32bafa775Dan Gohman assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?"); 4000572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman continue; 4001572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4002572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4003b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman 4004b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman if (Any) 4005b2df433f713c5ae9ddf95bd9d47cb3e7b0c6c8baDan Gohman LU.RecomputeRegs(LUIdx, RegUses); 4006572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4007572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4008572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "After pre-selection:\n"; 4009572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 4010572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4011572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4012572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 40134aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// NarrowSearchSpaceUsingHeuristics - If there are an extraordinary number of 40144aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// formulae to choose from, use some rough heuristics to prune down the number 40154aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of formulae. This keeps the main solver from taking an extraordinary amount 40164aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman/// of time in some worst-case scenarios. 40174aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohmanvoid LSRInstance::NarrowSearchSpaceUsingHeuristics() { 40184aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByDetectingSupersets(); 40194aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByCollapsingUnrolledCode(); 40204f7e18dee35c55e4e719f5fb73726ab035a7e5dbDan Gohman NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); 40214aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman NarrowSearchSpaceByPickingWinnerRegs(); 40224aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman} 40234aa5c2e90f70c7032a075bee06b8a08e731d99ecDan Gohman 4024572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// SolveRecurse - This is the recursive solver. 4025572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution, 4026572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost &SolutionCost, 4027572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<const Formula *> &Workspace, 4028572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Cost &CurCost, 4029572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SmallPtrSet<const SCEV *, 16> &CurRegs, 4030572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> &VisitedRegs) const { 4031572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Some ideas: 4032572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - prune more: 4033572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - use more aggressive filtering 4034572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the formula so that the most profitable solutions are found first 4035572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - sort the uses too 4036572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - search faster: 40373f46a3abeedba8d517b4182de34c821d752db058Dan Gohman // - don't compute a cost, and then compare. compare while computing a cost 4038572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // and bail early. 4039572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // - track register sets with SmallBitVector 4040572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4041572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = Uses[Workspace.size()]; 4042572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4043572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this use references any register that's already a part of the 4044572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // in-progress solution, consider it a requirement that a formula must 4045572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // reference that register in order to be considered. This prunes out 4046572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // unprofitable searching. 4047572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallSetVector<const SCEV *, 4> ReqRegs; 4048572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator I = CurRegs.begin(), 4049572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = CurRegs.end(); I != E; ++I) 40509214b82c5446791b280821bbd892dca633130f80Dan Gohman if (LU.Regs.count(*I)) 4051572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ReqRegs.insert(*I); 4052572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4053572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> NewRegs; 4054572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost NewCost; 4055572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(), 4056572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = LU.Formulae.end(); I != E; ++I) { 4057572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F = *I; 4058572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4059572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ignore formulae which do not use any of the required registers. 4060d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick bool SatisfiedReqReg = true; 4061572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(), 4062572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = ReqRegs.end(); J != JE; ++J) { 4063572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *J; 4064572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if ((!F.ScaledReg || F.ScaledReg != Reg) && 4065572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman std::find(F.BaseRegs.begin(), F.BaseRegs.end(), Reg) == 4066d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick F.BaseRegs.end()) { 4067d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick SatisfiedReqReg = false; 4068d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick break; 4069d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick } 4070d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick } 4071d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick if (!SatisfiedReqReg) { 4072d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick // If none of the formulae satisfied the required registers, then we could 4073d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick // clear ReqRegs and try again. Currently, we simply give up in this case. 4074d1944547b73ee7a4ed840a14d164929f1f7c7f12Andrew Trick continue; 4075572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4076572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4077572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Evaluate the cost of the current formula. If it's already worse than 4078572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // the current best, prune the search at that point. 4079572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost = CurCost; 4080572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs = CurRegs; 4081572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewCost.RateFormula(F, NewRegs, VisitedRegs, L, LU.Offsets, SE, DT); 4082572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (NewCost < SolutionCost) { 4083572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.push_back(&F); 4084572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Workspace.size() != Uses.size()) { 4085572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, NewCost, 4086572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NewRegs, VisitedRegs); 4087572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.getNumRegs() == 1 && Workspace.size() == 1) 4088572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]); 4089572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4090572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "New best at "; NewCost.print(dbgs()); 40918bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick dbgs() << ".\n Regs:"; 4092572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallPtrSet<const SCEV *, 16>::const_iterator 4093572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = NewRegs.begin(), E = NewRegs.end(); I != E; ++I) 4094572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ' ' << **I; 4095572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'); 4096572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4097572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost = NewCost; 4098572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution = Workspace; 4099572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4100572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.pop_back(); 4101572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 41029214b82c5446791b280821bbd892dca633130f80Dan Gohman } 4103572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4104572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 410576c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solve - Choose one formula from each use. Return the results in the given 410676c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Solution vector. 4107572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const { 4108572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Workspace; 4109572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost SolutionCost; 4110572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolutionCost.Loose(); 4111572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Cost CurCost; 4112572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallPtrSet<const SCEV *, 16> CurRegs; 4113572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DenseSet<const SCEV *> VisitedRegs; 4114572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Workspace.reserve(Uses.size()); 4115572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4116f7ff37d6745bfe170458e1165a5109cd2432d99dDan Gohman // SolveRecurse does all the work. 4117572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SolveRecurse(Solution, SolutionCost, Workspace, CurCost, 4118572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CurRegs, VisitedRegs); 411980ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (Solution.empty()) { 412080ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick DEBUG(dbgs() << "\nNo Satisfactory Solution\n"); 412180ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 412280ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick } 4123572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4124572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Ok, we've now made all our decisions. 4125572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\n" 4126572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The chosen solution requires "; SolutionCost.print(dbgs()); 4127572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"; 4128572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (size_t i = 0, e = Uses.size(); i != e; ++i) { 4129572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 4130572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Uses[i].print(dbgs()); 4131572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << "\n" 4132572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman " "; 4133572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solution[i]->print(dbgs()); 4134572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << '\n'; 4135572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }); 4136a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman 4137a5528785089bfd093a36cbc2eddcc35980c5340eDan Gohman assert(Solution.size() == Uses.size() && "Malformed solution!"); 4138572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4139572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4140e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// HoistInsertPosition - Helper for AdjustInsertPositionForExpand. Climb up 4141e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// the dominator tree far as we can go while still being dominated by the 4142e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// input positions. This helps canonicalize the insert position, which 4143e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// encourages sharing. 4144e5f76877aee6f33964de105893f0ef338661ecadDan GohmanBasicBlock::iterator 4145e5f76877aee6f33964de105893f0ef338661ecadDan GohmanLSRInstance::HoistInsertPosition(BasicBlock::iterator IP, 4146e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const SmallVectorImpl<Instruction *> &Inputs) 4147e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const { 4148e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (;;) { 4149e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IPLoop = LI.getLoopFor(IP->getParent()); 4150e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0; 4151e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4152e5f76877aee6f33964de105893f0ef338661ecadDan Gohman BasicBlock *IDom; 4153d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) { 41540fe46d9b480ab4851e1fc8bc589d1ed9c8b2a70eDan Gohman if (!Rung) return IP; 4155d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman Rung = Rung->getIDom(); 4156d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman if (!Rung) return IP; 4157d974a0e9d682e627f100340a8021e02ca991f965Dan Gohman IDom = Rung->getBlock(); 4158e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4159e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Don't climb into a loop though. 4160e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const Loop *IDomLoop = LI.getLoopFor(IDom); 4161e5f76877aee6f33964de105893f0ef338661ecadDan Gohman unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0; 4162e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDomDepth <= IPLoopDepth && 4163e5f76877aee6f33964de105893f0ef338661ecadDan Gohman (IDomDepth != IPLoopDepth || IDomLoop == IPLoop)) 4164e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 4165e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4166e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4167e5f76877aee6f33964de105893f0ef338661ecadDan Gohman bool AllDominate = true; 4168e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *BetterPos = 0; 4169e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Tentative = IDom->getTerminator(); 4170e5f76877aee6f33964de105893f0ef338661ecadDan Gohman for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(), 4171e5f76877aee6f33964de105893f0ef338661ecadDan Gohman E = Inputs.end(); I != E; ++I) { 4172e5f76877aee6f33964de105893f0ef338661ecadDan Gohman Instruction *Inst = *I; 4173e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (Inst == Tentative || !DT.dominates(Inst, Tentative)) { 4174e5f76877aee6f33964de105893f0ef338661ecadDan Gohman AllDominate = false; 4175e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 4176e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4177e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Attempt to find an insert position in the middle of the block, 4178e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // instead of at the end, so that it can be used for other expansions. 4179e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (IDom == Inst->getParent() && 41809719cf329bc398191c65cd1c8cb1161d11c5e947Rafael Espindola (!BetterPos || !DT.dominates(Inst, BetterPos))) 41817d9663c70b3300070298d716dba6e6f6ce2d1e3eDouglas Gregor BetterPos = llvm::next(BasicBlock::iterator(Inst)); 4182e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4183e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (!AllDominate) 4184e5f76877aee6f33964de105893f0ef338661ecadDan Gohman break; 4185e5f76877aee6f33964de105893f0ef338661ecadDan Gohman if (BetterPos) 4186e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = BetterPos; 4187e5f76877aee6f33964de105893f0ef338661ecadDan Gohman else 4188e5f76877aee6f33964de105893f0ef338661ecadDan Gohman IP = Tentative; 4189e5f76877aee6f33964de105893f0ef338661ecadDan Gohman } 4190e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4191e5f76877aee6f33964de105893f0ef338661ecadDan Gohman return IP; 4192e5f76877aee6f33964de105893f0ef338661ecadDan Gohman} 4193e5f76877aee6f33964de105893f0ef338661ecadDan Gohman 4194e5f76877aee6f33964de105893f0ef338661ecadDan Gohman/// AdjustInsertPositionForExpand - Determine an input position which will be 4195d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman/// dominated by the operands and which will dominate the result. 4196d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanBasicBlock::iterator 4197b5c26ef9da16052597d59a412eaae32098aa1be0Andrew TrickLSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator LowestIP, 4198e5f76877aee6f33964de105893f0ef338661ecadDan Gohman const LSRFixup &LF, 4199b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick const LSRUse &LU, 4200b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick SCEVExpander &Rewriter) const { 4201d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Collect some instructions which must be dominated by the 4202448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // expanding replacement. These must be dominated by any operands that 4203572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // will be required in the expansion. 4204572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<Instruction *, 4> Inputs; 4205572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace)) 4206572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 4207572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) 4208572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Instruction *I = 4209572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1))) 4210572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Inputs.push_back(I); 4211448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.PostIncLoops.count(L)) { 4212448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman if (LF.isUseFullyOutsideLoop(L)) 4213069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(L->getLoopLatch()->getTerminator()); 4214069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman else 4215069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman Inputs.push_back(IVIncInsertPos); 4216069d6f3396856655d5d4ba155ee16eb0209d38b0Dan Gohman } 4217701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // The expansion must also be dominated by the increment positions of any 4218701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman // loops it for which it is using post-inc mode. 4219701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (PostIncLoopSet::const_iterator I = LF.PostIncLoops.begin(), 4220701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman E = LF.PostIncLoops.end(); I != E; ++I) { 4221701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman const Loop *PIL = *I; 4222701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (PIL == L) continue; 4223701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman 4224e5f76877aee6f33964de105893f0ef338661ecadDan Gohman // Be dominated by the loop exit. 4225701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman SmallVector<BasicBlock *, 4> ExitingBlocks; 4226701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman PIL->getExitingBlocks(ExitingBlocks); 4227701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman if (!ExitingBlocks.empty()) { 4228701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BasicBlock *BB = ExitingBlocks[0]; 4229701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i) 4230701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]); 4231701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman Inputs.push_back(BB->getTerminator()); 4232701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 4233701a4aef7fa0ece4dc1fdbc88b981820564cb4e4Dan Gohman } 4234572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4235b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick assert(!isa<PHINode>(LowestIP) && !isa<LandingPadInst>(LowestIP) 4236b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick && !isa<DbgInfoIntrinsic>(LowestIP) && 4237b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick "Insertion point must be a normal instruction"); 4238b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick 4239572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Then, climb up the immediate dominator tree as far as we can go while 4240572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // still being dominated by the input positions. 4241b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick BasicBlock::iterator IP = HoistInsertPosition(LowestIP, Inputs); 4242d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4243d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Don't insert instructions before PHI nodes. 4244572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman while (isa<PHINode>(IP)) ++IP; 4245d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4246a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling // Ignore landingpad instructions. 4247a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling while (isa<LandingPadInst>(IP)) ++IP; 4248a4c86ab073d4b7a36477fc7c54c9d52499f04586Bill Wendling 4249d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Ignore debug intrinsics. 4250449f31cb9dbf4762935b63946e8120dbe98808ffDan Gohman while (isa<DbgInfoIntrinsic>(IP)) ++IP; 4251572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4252b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick // Set IP below instructions recently inserted by SCEVExpander. This keeps the 4253b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick // IP consistent across expansions and allows the previously inserted 4254b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick // instructions to be reused by subsequent expansion. 4255b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick while (Rewriter.isInsertedInstruction(IP) && IP != LowestIP) ++IP; 4256b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick 4257d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman return IP; 4258d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman} 4259d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 426076c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// Expand - Emit instructions for the leading candidate expression for this 426176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// LSRUse (this is called "expanding"). 4262d96eae80107a0881e21d1dda97e5e848ed055ec2Dan GohmanValue *LSRInstance::Expand(const LSRFixup &LF, 4263d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const Formula &F, 4264d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman BasicBlock::iterator IP, 4265d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SCEVExpander &Rewriter, 4266d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts) const { 4267d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman const LSRUse &LU = Uses[LF.LUIdx]; 4268d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4269d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // Determine an input position which will be dominated by the operands and 4270d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman // which will dominate the result. 4271b5c26ef9da16052597d59a412eaae32098aa1be0Andrew Trick IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter); 4272d96eae80107a0881e21d1dda97e5e848ed055ec2Dan Gohman 4273572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Inform the Rewriter if we have a post-increment use, so that it can 4274572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // perform an advantageous expansion. 4275448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.setPostInc(LF.PostIncLoops); 4276572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4277572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type that the user actually needs. 4278db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 4279572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This will be the type that we'll initially expand to. 4280db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *Ty = F.getType(); 4281572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!Ty) 4282572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // No type known; just expand directly to the ultimate type. 4283572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 4284572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy)) 4285572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand directly to the ultimate type if it's the right size. 4286572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ty = OpTy; 4287572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // This is the type to do integer arithmetic in. 4288db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *IntTy = SE.getEffectiveSCEVType(Ty); 4289572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4290572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Build up a list of operands to add together to form the full base. 4291572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const SCEV *, 8> Ops; 4292572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4293572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the BaseRegs portion. 4294572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), 4295572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = F.BaseRegs.end(); I != E; ++I) { 4296572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *Reg = *I; 4297572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!Reg->isZero() && "Zero allocated in a base register!"); 4298572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4299448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 4300448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 4301448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Reg = TransformForPostIncUse(Denormalize, Reg, 4302448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 4303448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 4304572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4305572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, 0, IP))); 4306572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4307572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4308572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the ScaledReg portion. 4309572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *ICmpScaledV = 0; 4310572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale != 0) { 4311572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *ScaledS = F.ScaledReg; 4312572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4313448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman // If we're expanding for a post-inc user, make the post-inc adjustment. 4314448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); 4315448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman ScaledS = TransformForPostIncUse(Denormalize, ScaledS, 4316448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman LF.UserInst, LF.OperandValToReplace, 4317448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Loops, SE, DT); 4318572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4319572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 4320572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An interesting way of "folding" with an icmp is to use a negated 4321572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // scale, which we'll implement by inserting it into the other operand 4322572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // of the icmp. 4323572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == -1 && 4324572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "The only scale supported by ICmpZero uses is -1!"); 4325572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Rewriter.expandCodeFor(ScaledS, 0, IP); 4326572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4327572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Otherwise just expand the scaled register and an explicit scale, 4328572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which is expected to be matched as part of the address. 4329b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick 4330b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick // Flush the operand list to suppress SCEVExpander hoisting address modes. 4331b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick if (!Ops.empty() && LU.Kind == LSRUse::Address) { 4332b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 4333b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick Ops.clear(); 4334b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick Ops.push_back(SE.getUnknown(FullV)); 4335b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick } 4336572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, 0, IP)); 4337572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ScaledS = SE.getMulExpr(ScaledS, 4338deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(ScaledS->getType(), F.AM.Scale)); 4339572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(ScaledS); 4340572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4341572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4342572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4343087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the GV portion. 4344087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman if (F.AM.BaseGV) { 4345b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick // Flush the operand list to suppress SCEVExpander hoisting. 4346b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick if (!Ops.empty()) { 4347b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 4348b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick Ops.clear(); 4349b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick Ops.push_back(SE.getUnknown(FullV)); 4350b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick } 4351087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(F.AM.BaseGV)); 4352b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick } 4353087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 4354b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick // Flush the operand list to suppress SCEVExpander hoisting of both folded and 4355b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick // unfolded offsets. LSR assumes they both live next to their uses. 4356b6b5b7b69113c5c3e49caf78adb1c2c4cf216db3Andrew Trick if (!Ops.empty()) { 4357087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP); 4358087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.clear(); 4359087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(FullV)); 4360087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman } 4361087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman 4362087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman // Expand the immediate portion. 4363572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman int64_t Offset = (uint64_t)F.AM.BaseOffs + LF.Offset; 4364572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Offset != 0) { 4365572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 4366572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // The other interesting way of "folding" with an ICmpZero is to use a 4367572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // negated immediate. 4368572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!ICmpScaledV) 4369dae36ba802f12966e4fc44d99097a55ff0b7d87bEli Friedman ICmpScaledV = ConstantInt::get(IntTy, -(uint64_t)Offset); 4370572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else { 4371572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Ops.push_back(SE.getUnknown(ICmpScaledV)); 4372572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = ConstantInt::get(IntTy, Offset); 4373572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4374572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4375572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Just add the immediate values. These again are expected to be matched 4376572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // as part of the address. 4377087bd1e3a12893873761736bf0f905a350e9e708Dan Gohman Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset))); 4378572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4379572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4380572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4381cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Expand the unfolded offset portion. 4382cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman int64_t UnfoldedOffset = F.UnfoldedOffset; 4383cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman if (UnfoldedOffset != 0) { 4384cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman // Just add the immediate values. 4385cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, 4386cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman UnfoldedOffset))); 4387cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman } 4388cca82149adef8306a295abdc963213ae3b11bbb6Dan Gohman 4389572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Emit instructions summing all the operands. 4390572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const SCEV *FullS = Ops.empty() ? 4391deff621abdd48bd70434bd4d7ef30f08ddba1cd8Dan Gohman SE.getConstant(IntTy, 0) : 4392572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE.getAddExpr(Ops); 4393572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Value *FullV = Rewriter.expandCodeFor(FullS, Ty, IP); 4394572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4395572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We're done expanding now, so reset the rewriter. 4396448db1cdef5872713ef77beffacf502ae3450cd7Dan Gohman Rewriter.clearPostInc(); 4397572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4398572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // An ICmpZero Formula represents an ICmp which we're handling as a 4399572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // comparison against zero. Now that we've expanded an expression for that 4400572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // form, update the ICmp's other operand. 4401572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (LU.Kind == LSRUse::ICmpZero) { 4402572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpInst *CI = cast<ICmpInst>(LF.UserInst); 4403572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(CI->getOperand(1)); 4404572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(!F.AM.BaseGV && "ICmp does not support folding a global value and " 4405572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 4406572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (F.AM.Scale == -1) { 4407572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (ICmpScaledV->getType() != OpTy) { 4408572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 4409572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false, 4410572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 4411572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV, OpTy, "tmp", CI); 4412572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ICmpScaledV = Cast; 4413572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4414572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, ICmpScaledV); 4415572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4416572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(F.AM.Scale == 0 && 4417572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "ICmp does not support folding a global value and " 4418572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "a scale at the same time!"); 4419572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy), 4420572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman -(uint64_t)Offset); 4421572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (C->getType() != OpTy) 4422572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 4423572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OpTy, false), 4424572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman C, OpTy); 4425572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4426572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CI->setOperand(1, C); 4427572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4428572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4429572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4430572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return FullV; 4431572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4432572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 44333a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// RewriteForPHI - Helper for Rewrite. PHI nodes are special because the use 44343a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// of their operands effectively happens in their predecessor blocks, so the 44353a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman/// expression may need to be expanded in multiple places. 44363a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohmanvoid LSRInstance::RewriteForPHI(PHINode *PN, 44373a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const LSRFixup &LF, 44383a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman const Formula &F, 44393a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SCEVExpander &Rewriter, 44403a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 44413a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pass *P) const { 44423a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman DenseMap<BasicBlock *, Value *> Inserted; 44433a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 44443a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (PN->getIncomingValue(i) == LF.OperandValToReplace) { 44453a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman BasicBlock *BB = PN->getIncomingBlock(i); 44463a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 44473a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is a critical edge, split the edge so that we do not insert 44483a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // the code on all predecessor/successor paths. We do this unless this 44493a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // is the canonical backedge for this loop, which complicates post-inc 44503a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // users. 44513a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 && 44523ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman !isa<IndirectBrInst>(BB->getTerminator())) { 445389d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling BasicBlock *Parent = PN->getParent(); 445489d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling Loop *PNLoop = LI.getLoopFor(Parent); 445589d4411cef736898047aa7e3bc159da39cacf8e6Bill Wendling if (!PNLoop || Parent != PNLoop->getHeader()) { 44563ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Split the critical edge. 44578b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling BasicBlock *NewBB = 0; 44588b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling if (!Parent->isLandingPad()) { 4459f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick NewBB = SplitCriticalEdge(BB, Parent, P, 4460f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick /*MergeIdenticalEdges=*/true, 4461f143b79b78d1d244809fa59320f2af2edf4e1a86Andrew Trick /*DontDeleteUselessPhis=*/true); 44628b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling } else { 44638b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling SmallVector<BasicBlock*, 2> NewBBs; 44648b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling SplitLandingPadPredecessors(Parent, BB, "", "", P, NewBBs); 44658b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling NewBB = NewBBs[0]; 44668b6af8a2a9a36bc9324c60d80cea021abf4d42d8Bill Wendling } 44673ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 44683ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // If PN is outside of the loop and BB is in the loop, we want to 44693ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // move the block to be immediately before the PHI block, not 44703ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // immediately after BB. 44713ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman if (L->contains(BB) && !L->contains(PN)) 44723ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman NewBB->moveBefore(PN->getParent()); 44733ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman 44743ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman // Splitting the edge can reduce the number of PHI entries we have. 44753ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman e = PN->getNumIncomingValues(); 44763ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman BB = NewBB; 44773ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman i = PN->getBasicBlockIndex(BB); 44783ef9838f89617fc471b6b84a64c7af824a070e50Dan Gohman } 44793a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 44803a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 44813a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair = 44823a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Inserted.insert(std::make_pair(BB, static_cast<Value *>(0))); 44833a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (!Pair.second) 44843a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, Pair.first->second); 44853a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman else { 4486454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, BB->getTerminator(), Rewriter, DeadInsts); 44873a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 44883a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 4489db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 44903a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman if (FullV->getType() != OpTy) 44913a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV = 44923a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, 44933a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman OpTy, false), 44943a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman FullV, LF.OperandValToReplace->getType(), 44953a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman "tmp", BB->getTerminator()); 44963a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 44973a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman PN->setIncomingValue(i, FullV); 44983a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman Pair.first->second = FullV; 44993a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 45003a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman } 45013a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman} 45023a02cbcd0310e6c63a4ac9b8d25fffc258c8b3e4Dan Gohman 4503572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// Rewrite - Emit instructions for the leading candidate expression for this 4504572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// LSRUse (this is called "expanding"), and update the UserInst to reference 4505572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman/// the newly expanded value. 4506572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::Rewrite(const LSRFixup &LF, 4507572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const Formula &F, 4508572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SCEVExpander &Rewriter, 4509572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVectorImpl<WeakVH> &DeadInsts, 4510572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) const { 4511572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // First, find an insertion point that dominates UserInst. For PHI nodes, 4512572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // find the nearest block which dominates all the relevant uses. 4513572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) { 4514454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman RewriteForPHI(PN, LF, F, Rewriter, DeadInsts, P); 4515572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } else { 4516454d26dc43207ec537d843229db6f5e6a302e23dDan Gohman Value *FullV = Expand(LF, F, LF.UserInst, Rewriter, DeadInsts); 4517572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4518572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If this is reuse-by-noop-cast, insert the noop cast. 4519db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner Type *OpTy = LF.OperandValToReplace->getType(); 4520572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (FullV->getType() != OpTy) { 4521572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Instruction *Cast = 4522572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false), 4523572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV, OpTy, "tmp", LF.UserInst); 4524572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FullV = Cast; 4525572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4526572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4527572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Update the user. ICmpZero is handled specially here (for now) because 4528572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand may have updated one of the operands of the icmp already, and 4529572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // its new value may happen to be equal to LF.OperandValToReplace, in 4530572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // which case doing replaceUsesOfWith leads to replacing both operands 4531572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // with the same value. TODO: Reorganize this. 4532572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Uses[LF.LUIdx].Kind == LSRUse::ICmpZero) 4533572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->setOperand(0, FullV); 4534572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman else 4535572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV); 4536572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4537572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4538572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DeadInsts.push_back(LF.OperandValToReplace); 4539572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4540572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 454176c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// ImplementSolution - Rewrite all the fixup locations with new values, 454276c315a26c9c6eb51fca2d88ee756094bed3b846Dan Gohman/// following the chosen solution. 4543572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid 4544572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::ImplementSolution(const SmallVectorImpl<const Formula *> &Solution, 4545572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Pass *P) { 4546572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Keep track of instructions we may have made dead, so that 4547572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // we can remove them after we are done working. 4548572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<WeakVH, 16> DeadInsts; 4549572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 45505e7645be4c9dd2193add44d30b5fef8036d7a3ceAndrew Trick SCEVExpander Rewriter(SE, "lsr"); 45518bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick#ifndef NDEBUG 45528bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick Rewriter.setDebugType(DEBUG_TYPE); 45538bf295b1bf345dda7f3f5cd12b5c0dafea283e81Andrew Trick#endif 4554572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.disableCanonicalMode(); 4555c5701910604cdf65811fabd31d41e38f1d1d4eb1Andrew Trick Rewriter.enableLSRMode(); 4556572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.setIVIncInsertPos(L, IVIncInsertPos); 4557572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 455864925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick // Mark phi nodes that terminate chains so the expander tries to reuse them. 455964925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(), 456064925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) { 456170a1860a463ce5278486f70d3808efdfc4c2e191Jakob Stoklund Olesen if (PHINode *PN = dyn_cast<PHINode>(ChainI->tailUserInst())) 456264925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick Rewriter.setChainedPhi(PN); 456364925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick } 456464925c55c65f9345a69fb67db07aa62cfb723577Andrew Trick 4565572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Expand the new value definitions and update the users. 4566402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 4567402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman E = Fixups.end(); I != E; ++I) { 4568402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman const LSRFixup &Fixup = *I; 4569572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4570402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman Rewrite(Fixup, *Solution[Fixup.LUIdx], Rewriter, DeadInsts, P); 4571572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4572572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed = true; 4573572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4574572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 457522d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(), 457622d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) { 457722d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick GenerateIVChain(*ChainI, Rewriter, DeadInsts); 457822d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick Changed = true; 457922d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick } 4580572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Clean up after ourselves. This must be done before deleting any 4581572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // instructions. 4582572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Rewriter.clear(); 4583f7912df4cbdb44aeac9ac9907c192dfc1e22646dDan Gohman 4584572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= DeleteTriviallyDeadInstructions(DeadInsts); 4585572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4586010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 4587572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P) 4588572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman : IU(P->getAnalysis<IVUsers>()), 4589572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SE(P->getAnalysis<ScalarEvolution>()), 4590572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DT(P->getAnalysis<DominatorTree>()), 4591e5f76877aee6f33964de105893f0ef338661ecadDan Gohman LI(P->getAnalysis<LoopInfo>()), 4592572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman TLI(tli), L(l), Changed(false), IVIncInsertPos(0) { 45935792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 4594572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // If LoopSimplify form is not available, stay out of trouble. 4595acdb4aaf9b1f2edd96163c27bcc4e0557014f51eAndrew Trick if (!L->isLoopSimplifyForm()) 4596acdb4aaf9b1f2edd96163c27bcc4e0557014f51eAndrew Trick return; 4597572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 459875ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // If there's no interesting work to be done, bail early. 459975ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick if (IU.empty()) return; 460075ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick 4601b5122635966a980a850c028895e275a43e0f946dAndrew Trick // If there's too much analysis to be done, bail early. We won't be able to 4602b5122635966a980a850c028895e275a43e0f946dAndrew Trick // model the problem anyway. 4603b5122635966a980a850c028895e275a43e0f946dAndrew Trick unsigned NumUsers = 0; 4604b5122635966a980a850c028895e275a43e0f946dAndrew Trick for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) { 4605b5122635966a980a850c028895e275a43e0f946dAndrew Trick if (++NumUsers > MaxIVUsers) { 4606b5122635966a980a850c028895e275a43e0f946dAndrew Trick DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << *L 4607b5122635966a980a850c028895e275a43e0f946dAndrew Trick << "\n"); 4608b5122635966a980a850c028895e275a43e0f946dAndrew Trick return; 4609b5122635966a980a850c028895e275a43e0f946dAndrew Trick } 4610b5122635966a980a850c028895e275a43e0f946dAndrew Trick } 4611b5122635966a980a850c028895e275a43e0f946dAndrew Trick 461275ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick#ifndef NDEBUG 46130f080913d1ff80bb61476724304359e14822b193Andrew Trick // All dominating loops must have preheaders, or SCEVExpander may not be able 46140f080913d1ff80bb61476724304359e14822b193Andrew Trick // to materialize an AddRecExpr whose Start is an outer AddRecExpr. 46150f080913d1ff80bb61476724304359e14822b193Andrew Trick // 461675ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // IVUsers analysis should only create users that are dominated by simple loop 461775ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // headers. Since this loop should dominate all of its users, its user list 461875ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick // should be empty if this loop itself is not within a simple loop nest. 46190f080913d1ff80bb61476724304359e14822b193Andrew Trick for (DomTreeNode *Rung = DT.getNode(L->getLoopPreheader()); 46200f080913d1ff80bb61476724304359e14822b193Andrew Trick Rung; Rung = Rung->getIDom()) { 46210f080913d1ff80bb61476724304359e14822b193Andrew Trick BasicBlock *BB = Rung->getBlock(); 46220f080913d1ff80bb61476724304359e14822b193Andrew Trick const Loop *DomLoop = LI.getLoopFor(BB); 46230f080913d1ff80bb61476724304359e14822b193Andrew Trick if (DomLoop && DomLoop->getHeader() == BB) { 462475ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick assert(DomLoop->getLoopPreheader() && "LSR needs a simplified loop nest"); 46250f080913d1ff80bb61476724304359e14822b193Andrew Trick } 4626acdb4aaf9b1f2edd96163c27bcc4e0557014f51eAndrew Trick } 462775ae20366fd1b480f4cc38400bb075c43c9f4f7fAndrew Trick#endif // DEBUG 4628572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4629572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "\nLSR on loop "; 4630572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman WriteAsOperand(dbgs(), L->getHeader(), /*PrintType=*/false); 4631572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << ":\n"); 4632572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4633402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // First, perform some low-level loop optimizations. 4634572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OptimizeShadowIV(); 4635c6519f916b5922de81c53547fd21364994195a70Dan Gohman OptimizeLoopTermCond(); 4636572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 463737eb38d3f8531115f17f4e829013ccb513952badAndrew Trick // If loop preparation eliminates all interesting IV users, bail. 463837eb38d3f8531115f17f4e829013ccb513952badAndrew Trick if (IU.empty()) return; 463937eb38d3f8531115f17f4e829013ccb513952badAndrew Trick 46405219f86a0bab11dd6895a31653e371e9871a6734Andrew Trick // Skip nested loops until we can model them better with formulae. 4641bd618f1b7ff73bde1e421eb6daf978e74984e180Andrew Trick if (!L->empty()) { 46420c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n"); 46435219f86a0bab11dd6895a31653e371e9871a6734Andrew Trick return; 46440c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick } 46450c01bc385a4c01bee012bda504c8ce0c3d402f2cAndrew Trick 4646402d43529c18d662d1964d70dbf55bc135a8b473Dan Gohman // Start collecting data and preparing for the solver. 46476c7d0ae8dc8beb37efd6c0ff586035253856e07cAndrew Trick CollectChains(); 4648572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectInterestingTypesAndFactors(); 4649572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectFixupsAndInitialFormulae(); 4650572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman CollectLoopInvariantFixupsAndFormulae(); 4651572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 465222d20c218aeb14af388bff2346d6d4cc131e8449Andrew Trick assert(!Uses.empty() && "IVUsers reported at least one use"); 4653572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n"; 4654572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(dbgs())); 4655572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4656572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now use the reuse data to generate a bunch of interesting ways 4657572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // to formulate the values needed for the uses. 4658572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman GenerateAllReuseFormulae(); 4659572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4660572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman FilterOutUndesirableDedicatedRegisters(); 4661572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman NarrowSearchSpaceUsingHeuristics(); 4662572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4663572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman SmallVector<const Formula *, 8> Solution; 4664572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Solve(Solution); 4665572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4666572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Release memory that is no longer needed. 4667572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Factors.clear(); 4668572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Types.clear(); 4669572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman RegUses.clear(); 4670572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 467180ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick if (Solution.empty()) 467280ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick return; 467380ef1b287fa1b62ad3de8a7c3658ff37b5acca8eAndrew Trick 4674572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#ifndef NDEBUG 4675572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Formulae should be legal. 4676572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 4677572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 4678572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 4679572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 4680572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) 4681572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman assert(isLegalUse(J->AM, LU.MinOffset, LU.MaxOffset, 4682572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.Kind, LU.AccessTy, TLI) && 4683572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman "Illegal formula generated!"); 4684572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman }; 4685572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman#endif 4686010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 4687572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Now that we've decided what we want, make it so. 4688572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman ImplementSolution(Solution, P); 4689572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4690169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 4691572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_factors_and_types(raw_ostream &OS) const { 4692572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (Factors.empty() && Types.empty()) return; 46931ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 4694572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR has identified the following interesting factors and types: "; 4695572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool First = true; 4696eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 4697572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallSetVector<int64_t, 8>::const_iterator 4698572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Factors.begin(), E = Factors.end(); I != E; ++I) { 4699572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 4700572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 4701572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '*' << *I; 4702572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 47038b0ade3eb8281f9b332d5764cfe5c4ed3b2b32d8Dan Gohman 4704db125cfaf57cc83e7dd7453de2d509bc8efd0e5eChris Lattner for (SmallSetVector<Type *, 4>::const_iterator 4705572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman I = Types.begin(), E = Types.end(); I != E; ++I) { 4706572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman if (!First) OS << ", "; 4707572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman First = false; 4708572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '(' << **I << ')'; 4709572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4710572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4711572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4712c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 4713572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_fixups(raw_ostream &OS) const { 4714572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following fixup sites:\n"; 4715572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(), 4716572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Fixups.end(); I != E; ++I) { 4717572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 47189f383eb9503580a1425073beee99fdf74ed31a17Dan Gohman I->print(OS); 4719572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4720572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 4721572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4722010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman 4723572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print_uses(raw_ostream &OS) const { 4724572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << "LSR is examining the following uses:\n"; 4725572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), 4726572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman E = Uses.end(); I != E; ++I) { 4727572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const LSRUse &LU = *I; 4728572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman dbgs() << " "; 4729572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman LU.print(OS); 4730572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4731572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(), 4732572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman JE = LU.Formulae.end(); J != JE; ++J) { 4733572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << " "; 4734572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman J->print(OS); 4735572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman OS << '\n'; 4736572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman } 47376bec5bb344fc0374431aed1cb63418de607a1aecDan Gohman } 4738572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4739572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4740572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::print(raw_ostream &OS) const { 4741572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_factors_and_types(OS); 4742572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_fixups(OS); 4743572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print_uses(OS); 4744572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4745572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4746cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#ifndef NDEBUG 4747572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LSRInstance::dump() const { 4748572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman print(errs()); errs() << '\n'; 4749572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4750cc77eece74c8db09acc2af425e7e6c88a5bb30d1Manman Ren#endif 4751572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4752572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmannamespace { 4753572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4754572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanclass LoopStrengthReduce : public LoopPass { 4755572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// TLI - Keep a pointer of a TargetLowering to consult for determining 4756572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman /// transformation profitability. 4757572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman const TargetLowering *const TLI; 4758572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4759572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanpublic: 4760572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman static char ID; // Pass ID, replacement for typeid 4761572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman explicit LoopStrengthReduce(const TargetLowering *tli = 0); 4762572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4763572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanprivate: 4764572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool runOnLoop(Loop *L, LPPassManager &LPM); 4765572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman void getAnalysisUsage(AnalysisUsage &AU) const; 4766572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman}; 4767572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4768572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4769572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4770572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanchar LoopStrengthReduce::ID = 0; 47712ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce", 4772ce665bd2e2b581ab0858d1afe359192bac96b868Owen Anderson "Loop Strength Reduction", false, false) 47732ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(DominatorTree) 47742ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(ScalarEvolution) 47752ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_DEPENDENCY(IVUsers) 4776205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopInfo) 4777205942a4a55d568e93480fc22d25cc7dac525fb7Owen AndersonINITIALIZE_PASS_DEPENDENCY(LoopSimplify) 47782ab36d350293c77fc8941ce1023e4899df7e3a82Owen AndersonINITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce", 47792ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson "Loop Strength Reduction", false, false) 47802ab36d350293c77fc8941ce1023e4899df7e3a82Owen Anderson 4781572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4782572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanPass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) { 4783572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman return new LoopStrengthReduce(TLI); 4784572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4785572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4786572645cf84060c0fc25cb91d38cb9079918b3a88Dan GohmanLoopStrengthReduce::LoopStrengthReduce(const TargetLowering *tli) 4787081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson : LoopPass(ID), TLI(tli) { 4788081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry()); 4789081c34b725980f995be9080eaec24cd3dfaaf065Owen Anderson } 4790572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4791572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanvoid LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const { 4792572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // We split critical edges, so we change the CFG. However, we do update 4793572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // many analyses if they are around. 47946793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreservedID(LoopSimplifyID); 4795572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 47966793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequired<LoopInfo>(); 47976793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addPreserved<LoopInfo>(); 47986793c49bb4b7d20532e530404740422036d84788Eric Christopher AU.addRequiredID(LoopSimplifyID); 4799572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<DominatorTree>(); 4800572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<DominatorTree>(); 4801572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<ScalarEvolution>(); 4802572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<ScalarEvolution>(); 48032c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // Requiring LoopSimplify a second time here prevents IVUsers from running 48042c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich // twice, since LoopSimplify was invalidated by running ScalarEvolution. 48052c2b933037ecd5a0ebcfa3077606892802c04a29Cameron Zwarich AU.addRequiredID(LoopSimplifyID); 4806572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addRequired<IVUsers>(); 4807572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman AU.addPreserved<IVUsers>(); 4808572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman} 4809572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4810572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohmanbool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) { 4811572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman bool Changed = false; 4812572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman 4813572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman // Run the main LSR transformation. 4814572645cf84060c0fc25cb91d38cb9079918b3a88Dan Gohman Changed |= LSRInstance(TLI, L, this).getChanged(); 4815eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 4816f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick // Remove any extra phis created by processing inner loops. 48179fff2187a21f765ed87a25c48552a6942450f3e2Dan Gohman Changed |= DeleteDeadPHIs(L->getHeader()); 4818f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick if (EnablePhiElim) { 4819f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick SmallVector<WeakVH, 16> DeadInsts; 4820f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick SCEVExpander Rewriter(getAnalysis<ScalarEvolution>(), "lsr"); 4821f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick#ifndef NDEBUG 4822f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick Rewriter.setDebugType(DEBUG_TYPE); 4823f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick#endif 4824f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick unsigned numFolded = Rewriter. 4825f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick replaceCongruentIVs(L, &getAnalysis<DominatorTree>(), DeadInsts, TLI); 4826f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick if (numFolded) { 4827f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick Changed = true; 4828f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick DeleteTriviallyDeadInstructions(DeadInsts); 4829f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick DeleteDeadPHIs(L->getHeader()); 4830f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick } 4831f231a6dc7f251859af61677991b9c70ade6e1bfaAndrew Trick } 48321ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng return Changed; 4833eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 4834