LoopStrengthReduce.cpp revision c17e0cf6c03a36f424fafe88497b5fdf351cd50a
1eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===- LoopStrengthReduce.cpp - Strength Reduce GEPs in Loops -------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// The LLVM Compiler Infrastructure 4eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 9eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 10eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// This pass performs a strength reduction on array references inside loops that 11eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// have as one or more of their components the loop induction variable. This is 12eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// accomplished by creating a new Value to hold the initial value of the array 13eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// access for the first iteration, and then creating a new GEP instruction in 14eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// the loop to increment the value by the appropriate amount. 15eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 16eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 17eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 18be3e5212e23edc9210f774fac18d801de252e906Chris Lattner#define DEBUG_TYPE "loop-reduce" 19eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Scalar.h" 20eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Constants.h" 21eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Instructions.h" 22e5b01bea7b9b7dce7c24484d2d915b0c118d4d07Dan Gohman#include "llvm/IntrinsicInst.h" 23eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Type.h" 242f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/DerivedTypes.h" 25eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Analysis/Dominators.h" 26eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Analysis/LoopInfo.h" 270f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel#include "llvm/Analysis/LoopPass.h" 28169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Analysis/ScalarEvolutionExpander.h" 29eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Support/CFG.h" 30169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/GetElementPtrTypeIterator.h" 31e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner#include "llvm/Transforms/Utils/BasicBlockUtils.h" 32eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Utils/Local.h" 332f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/Target/TargetData.h" 34168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng#include "llvm/ADT/SmallPtrSet.h" 35eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/ADT/Statistic.h" 36169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/Debug.h" 37a4f0b3a084d120cfc5b5bb06f64b222f5cb72740Chris Lattner#include "llvm/Support/Compiler.h" 38c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman#include "llvm/Support/CommandLine.h" 39d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng#include "llvm/Target/TargetLowering.h" 40cfb1d4235fe3291028341e6acf4139723b4749e3Jeff Cohen#include <algorithm> 41eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include <set> 42eaa13851a7fe604363577350c5cf65c257c4d41aNate Begemanusing namespace llvm; 43eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 44cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumReduced , "Number of GEPs strength reduced"); 45cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumInserted, "Number of PHIs inserted"); 46cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumVariable, "Number of PHIs with variable strides"); 47541532724e29203e28c2fe0136cf6eabd49d4532Devang PatelSTATISTIC(NumEliminated, "Number of strides eliminated"); 48541532724e29203e28c2fe0136cf6eabd49d4532Devang PatelSTATISTIC(NumShadow, "Number of Shadow IVs optimized"); 49eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 50c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic cl::opt<bool> EnableFullLSRMode("enable-full-lsr", 51c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cl::init(false), 52c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cl::Hidden); 53c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 540e5f499638c8d277b9dc4a4385712177c53b5681Chris Lattnernamespace { 55dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 56c01a53007a4f4f9a601f1cc83ff4e2935405b905Jeff Cohen struct BasedUser; 57dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 58ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// IVStrideUse - Keep track of one use of a strided induction variable, where 59ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// the stride is stored externally. The Offset member keeps track of the 609330c3af45453d0a3dd9caef40910a9a053e4de5Dan Gohman /// offset from the IV, User is the actual user of the operand, and 619330c3af45453d0a3dd9caef40910a9a053e4de5Dan Gohman /// 'OperandValToReplace' is the operand of the User that is the use. 629133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVStrideUse { 63ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner SCEVHandle Offset; 64ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Instruction *User; 65ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Value *OperandValToReplace; 66010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 67010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // isUseOfPostIncrementedValue - True if this should use the 68010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // post-incremented version of this IV, not the preincremented version. 69010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // This can only be set in special cases, such as the terminating setcc 70c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // instruction for a loop or uses dominated by the loop. 71010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner bool isUseOfPostIncrementedValue; 72ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner 73ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner IVStrideUse(const SCEVHandle &Offs, Instruction *U, Value *O) 74010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner : Offset(Offs), User(U), OperandValToReplace(O), 75010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner isUseOfPostIncrementedValue(false) {} 76ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner }; 77ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner 78ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// IVUsersOfOneStride - This structure keeps track of all instructions that 79ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// have an operand that is based on the trip count multiplied by some stride. 80ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// The stride for all of these users is common and kept external to this 81ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// structure. 829133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVUsersOfOneStride { 83169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Users - Keep track of all of the users of this stride as well as the 84ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// initial value and the operand that uses the IV. 85ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner std::vector<IVStrideUse> Users; 86ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner 87ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner void addUser(const SCEVHandle &Offset,Instruction *User, Value *Operand) { 88ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Users.push_back(IVStrideUse(Offset, User, Operand)); 89169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 90169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman }; 91169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 92d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVInfo - This structure keeps track of one IV expression inserted during 9321495775e710d37003e100862cdc647cbdc3b999Evan Cheng /// StrengthReduceStridedIVUsers. It contains the stride, the common base, as 9421495775e710d37003e100862cdc647cbdc3b999Evan Cheng /// well as the PHI node and increment value created for rewrite. 959133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVExpr { 9621495775e710d37003e100862cdc647cbdc3b999Evan Cheng SCEVHandle Stride; 97d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng SCEVHandle Base; 98d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng PHINode *PHI; 99d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng Value *IncV; 100d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 10121495775e710d37003e100862cdc647cbdc3b999Evan Cheng IVExpr(const SCEVHandle &stride, const SCEVHandle &base, PHINode *phi, 10221495775e710d37003e100862cdc647cbdc3b999Evan Cheng Value *incv) 10321495775e710d37003e100862cdc647cbdc3b999Evan Cheng : Stride(stride), Base(base), PHI(phi), IncV(incv) {} 104d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng }; 105d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 106d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVsOfOneStride - This structure keeps track of all IV expression inserted 107d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// during StrengthReduceStridedIVUsers for a particular stride of the IV. 1089133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVsOfOneStride { 109d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng std::vector<IVExpr> IVs; 110d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 11121495775e710d37003e100862cdc647cbdc3b999Evan Cheng void addIV(const SCEVHandle &Stride, const SCEVHandle &Base, PHINode *PHI, 11221495775e710d37003e100862cdc647cbdc3b999Evan Cheng Value *IncV) { 11321495775e710d37003e100862cdc647cbdc3b999Evan Cheng IVs.push_back(IVExpr(Stride, Base, PHI, IncV)); 114d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng } 115d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng }; 116169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 1170f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel class VISIBILITY_HIDDEN LoopStrengthReduce : public LoopPass { 118eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman LoopInfo *LI; 119b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel DominatorTree *DT; 120169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman ScalarEvolution *SE; 121169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman const TargetData *TD; 122169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman const Type *UIntPtrTy; 123eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman bool Changed; 1247e608bbb5dfe4f827e64e91b0bb68a1d95d737aeChris Lattner 125169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// IVUsesByStride - Keep track of all uses of induction variables that we 126169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// are interested in. The key of the map is the stride of the access. 12750fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner std::map<SCEVHandle, IVUsersOfOneStride> IVUsesByStride; 128169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 129d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVsByStride - Keep track of all IVs that have been inserted for a 130d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// particular stride. 131d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng std::map<SCEVHandle, IVsOfOneStride> IVsByStride; 132d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1337305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner /// StrideOrder - An ordering of the keys in IVUsesByStride that is stable: 1347305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner /// We use this to iterate over the IVUsesByStride collection without being 1357305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner /// dependent on random ordering of pointers in the process. 1368392772727ed9105c92fe4514d53dab74c333edcEvan Cheng SmallVector<SCEVHandle, 16> StrideOrder; 1377305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner 1382f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// GEPlist - A list of the GEP's that have been remembered in the SCEV 1392f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// data structures. SCEV does not know to update these when the operands 1402f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// of the GEP are changed, which means we cannot leave them live across 1412f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// loops. 1422f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SmallVector<GetElementPtrInst *, 16> GEPlist; 1432f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 14449f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// CastedValues - As we need to cast values to uintptr_t, this keeps track 14549f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// of the casted version of each value. This is accessed by 14649f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// getCastedVersionOf. 1478392772727ed9105c92fe4514d53dab74c333edcEvan Cheng DenseMap<Value*, Value*> CastedPointers; 148169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 149169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// DeadInsts - Keep track of instructions we may have made dead, so that 150169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// we can remove them after we are done working. 15109fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVector<Instruction*, 16> DeadInsts; 152d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng 153d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng /// TLI - Keep a pointer of a TargetLowering to consult for determining 154d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng /// transformation profitability. 155d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng const TargetLowering *TLI; 156d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng 157eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman public: 1581997473cf72957d0e70322e2fe6fe2ab141c58a6Devang Patel static char ID; // Pass ID, replacement for typeid 159c2bbfc18e9adbbdcf5b3375d8d25e2452f7df7f1Dan Gohman explicit LoopStrengthReduce(const TargetLowering *tli = NULL) : 160ae73dc1448d25b02cabc7c64c86c64371453dda8Dan Gohman LoopPass(&ID), TLI(tli) { 1612f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen } 1622f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen 1630f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel bool runOnLoop(Loop *L, LPPassManager &LPM); 164eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 165eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman virtual void getAnalysisUsage(AnalysisUsage &AU) const { 166aa96ae780afa5475e62df284855a971216289212Chris Lattner // We split critical edges, so we change the CFG. However, we do update 167aa96ae780afa5475e62df284855a971216289212Chris Lattner // many analyses if they are around. 168aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreservedID(LoopSimplifyID); 169aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<LoopInfo>(); 170aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<DominanceFrontier>(); 171aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<DominatorTree>(); 172aa96ae780afa5475e62df284855a971216289212Chris Lattner 173f465db6c6a5a877aa791abfc3837d62c491dacd5Jeff Cohen AU.addRequiredID(LoopSimplifyID); 174eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman AU.addRequired<LoopInfo>(); 175b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel AU.addRequired<DominatorTree>(); 1762f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen AU.addRequired<TargetData>(); 177169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman AU.addRequired<ScalarEvolution>(); 178a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel AU.addPreserved<ScalarEvolution>(); 179eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 18049f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 18149f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// getCastedVersionOf - Return the specified value casted to uintptr_t. 18249f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// 1833ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Value *getCastedVersionOf(Instruction::CastOps opcode, Value *V); 18449f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattnerprivate: 1853416e5f645186a7e3321f927eab662d0ecef404bChris Lattner bool AddUsersIfInteresting(Instruction *I, Loop *L, 186168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SmallPtrSet<Instruction*,16> &Processed); 1878480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman SCEVHandle GetExpressionSCEV(Instruction *E); 188cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst *ChangeCompareStride(Loop *L, ICmpInst *Cond, 189cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVStrideUse* &CondUse, 190cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng const SCEVHandle* &CondStride); 191010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner void OptimizeIndvars(Loop *L); 192a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 193a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /// OptimizeShadowIV - If IV is used in a int-to-float cast 194a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /// inside the loop then try to eliminate the cast opeation. 195a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel void OptimizeShadowIV(Loop *L); 196a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 197ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman /// OptimizeSMax - Rewrite the loop's terminating condition 198ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman /// if it uses an smax computation. 199ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman ICmpInst *OptimizeSMax(Loop *L, ICmpInst *Cond, 200ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman IVStrideUse* &CondUse); 201ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 202c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse, 203a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const SCEVHandle *&CondStride); 2045f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool RequiresTypeConversion(const Type *Ty, const Type *NewTy); 2052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SCEVHandle CheckForIVReuse(bool, bool, bool, const SCEVHandle&, 20602e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman IVExpr&, const Type*, 207dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess); 20802e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman bool ValidStride(bool, int64_t, 20902e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman const std::vector<BasedUser>& UsersToProcess); 2105f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng SCEVHandle CollectIVUsers(const SCEVHandle &Stride, 2115f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 2125f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 2135f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool &AllUsesAreAddresses, 214b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool &AllUsesAreOutsideLoop, 2155f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> &UsersToProcess); 216c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool ShouldUseFullStrengthReductionMode( 217c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const std::vector<BasedUser> &UsersToProcess, 218c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 219c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool AllUsesAreAddresses, 220c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride); 221c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceFully( 222c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 223c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 224c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 225c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 226c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter); 227c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceFromSmallerStride( 228c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 229c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 230c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const IVExpr &ReuseIV, 231c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Instruction *PreInsertPt); 232c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceWithNewPhi( 233c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 234c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 235c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 236c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 237c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 238c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter); 23950fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner void StrengthReduceStridedIVUsers(const SCEVHandle &Stride, 24050fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner IVUsersOfOneStride &Uses, 241ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Loop *L, bool isOnlyStride); 242a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner void DeleteTriviallyDeadInstructions(); 243eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman }; 244eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 245eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 246844731a7f1909f55935e3514c9e713a62d67662eDan Gohmanchar LoopStrengthReduce::ID = 0; 247844731a7f1909f55935e3514c9e713a62d67662eDan Gohmanstatic RegisterPass<LoopStrengthReduce> 248844731a7f1909f55935e3514c9e713a62d67662eDan GohmanX("loop-reduce", "Loop Strength Reduction"); 249844731a7f1909f55935e3514c9e713a62d67662eDan Gohman 250394f0441e06dafca29f0752cf400990a5b8fe4b1Daniel DunbarPass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) { 251d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng return new LoopStrengthReduce(TLI); 252eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 253eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2544da49122f3f3c8da68a52723d846b88c72166a68Reid Spencer/// getCastedVersionOf - Return the specified value casted to uintptr_t. This 2554da49122f3f3c8da68a52723d846b88c72166a68Reid Spencer/// assumes that the Value* V is of integer or pointer type only. 25649f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner/// 2573ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid SpencerValue *LoopStrengthReduce::getCastedVersionOf(Instruction::CastOps opcode, 2583ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Value *V) { 25949f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner if (V->getType() == UIntPtrTy) return V; 26049f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner if (Constant *CB = dyn_cast<Constant>(V)) 2613ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer return ConstantExpr::getCast(opcode, CB, UIntPtrTy); 26249f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 26349f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner Value *&New = CastedPointers[V]; 26449f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner if (New) return New; 26549f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 2663ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer New = SCEVExpander::InsertCastOfTo(opcode, V, UIntPtrTy); 26709fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(cast<Instruction>(New)); 2687db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return New; 26949f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner} 27049f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 27149f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 272eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// DeleteTriviallyDeadInstructions - If any of the instructions is the 273eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// specified set are trivially dead, delete them and see if this makes any of 274eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// their operands subsequently dead. 275a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattnervoid LoopStrengthReduce::DeleteTriviallyDeadInstructions() { 27609fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (DeadInsts.empty()) return; 27709fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 27809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // Sort the deadinsts list so that we can trivially eliminate duplicates as we 27909fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // go. The code below never adds a non-dead instruction to the worklist, but 28009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // callers may not be so careful. 28199d0015735f8e2aee1a4b99e39ffdaadc8a1dba8Chris Lattner array_pod_sort(DeadInsts.begin(), DeadInsts.end()); 28209fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 28309fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // Drop duplicate instructions and those with uses. 28409fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner for (unsigned i = 0, e = DeadInsts.size()-1; i < e; ++i) { 28509fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner Instruction *I = DeadInsts[i]; 28609fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (!I->use_empty()) DeadInsts[i] = 0; 28746a879ebd384fb39cb680e4c3c182fefcdef5778Chris Lattner while (i != e && DeadInsts[i+1] == I) 28809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts[++i] = 0; 28909fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner } 29009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 291a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner while (!DeadInsts.empty()) { 292a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner Instruction *I = DeadInsts.back(); 293a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeadInsts.pop_back(); 29409fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 29509fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (I == 0 || !isInstructionTriviallyDead(I)) 296bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner continue; 297bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner 298bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner SE->deleteValueFromRecords(I); 299411052bb96fb4a30035de6cafa492fa4f598e824Bill Wendling 30009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) { 30109fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (Instruction *U = dyn_cast<Instruction>(*OI)) { 30209fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner *OI = 0; 303bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner if (U->use_empty()) 30409fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(U); 305bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner } 306eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 307bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner 308bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner I->eraseFromParent(); 309bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner Changed = true; 310eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 311eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 312eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 313169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 3143416e5f645186a7e3321f927eab662d0ecef404bChris Lattner/// GetExpressionSCEV - Compute and return the SCEV for the specified 3153416e5f645186a7e3321f927eab662d0ecef404bChris Lattner/// instruction. 3168480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan GohmanSCEVHandle LoopStrengthReduce::GetExpressionSCEV(Instruction *Exp) { 317da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen // Pointer to pointer bitcast instructions return the same value as their 318da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen // operand. 319da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen if (BitCastInst *BCI = dyn_cast<BitCastInst>(Exp)) { 320da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen if (SE->hasSCEV(BCI) || !isa<Instruction>(BCI->getOperand(0))) 321da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen return SE->getSCEV(BCI); 3228480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman SCEVHandle R = GetExpressionSCEV(cast<Instruction>(BCI->getOperand(0))); 323da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen SE->setSCEV(BCI, R); 324da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen return R; 325da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen } 326da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen 32787265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // Scalar Evolutions doesn't know how to compute SCEV's for GEP instructions. 32887265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // If this is a GEP that SE doesn't know about, compute it now and insert it. 32987265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // If this is not a GEP, or if we have already done this computation, just let 33087265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // SE figure it out. 3313416e5f645186a7e3321f927eab662d0ecef404bChris Lattner GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Exp); 33287265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner if (!GEP || SE->hasSCEV(GEP)) 3333416e5f645186a7e3321f927eab662d0ecef404bChris Lattner return SE->getSCEV(Exp); 3343416e5f645186a7e3321f927eab662d0ecef404bChris Lattner 335169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Analyze all of the subscripts of this getelementptr instruction, looking 3368480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman // for uses that are determined by the trip count of the loop. First, skip 3378480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman // all operands the are not dependent on the IV. 338169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 339169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Build up the base expression. Insert an LLVM cast of the pointer to 340169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // uintptr_t first. 341246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle GEPVal = SE->getUnknown( 3423ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer getCastedVersionOf(Instruction::PtrToInt, GEP->getOperand(0))); 343169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 344169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman gep_type_iterator GTI = gep_type_begin(GEP); 3453416e5f645186a7e3321f927eab662d0ecef404bChris Lattner 3466725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end(); 3476725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif i != e; ++i, ++GTI) { 348169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // If this is a use of a recurrence that we can analyze, and it comes before 349169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Op does in the GEP operand list, we will handle this when we process this 350169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // operand. 351169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman if (const StructType *STy = dyn_cast<StructType>(*GTI)) { 352169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman const StructLayout *SL = TD->getStructLayout(STy); 3536725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif unsigned Idx = cast<ConstantInt>(*i)->getZExtValue(); 354b1919e2f08ecb37140af676fd2916f8d5ed7df3dChris Lattner uint64_t Offset = SL->getElementOffset(Idx); 355246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman GEPVal = SE->getAddExpr(GEPVal, 356246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SE->getIntegerSCEV(Offset, UIntPtrTy)); 3572f62fdc9a71d790d381d7f17d16e7098e30217e3Chris Lattner } else { 3583ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer unsigned GEPOpiBits = 3596725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif (*i)->getType()->getPrimitiveSizeInBits(); 3603ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer unsigned IntPtrBits = UIntPtrTy->getPrimitiveSizeInBits(); 3613ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::CastOps opcode = (GEPOpiBits < IntPtrBits ? 3623ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::SExt : (GEPOpiBits > IntPtrBits ? Instruction::Trunc : 3633ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::BitCast)); 3646725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif Value *OpVal = getCastedVersionOf(opcode, *i); 3657db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner SCEVHandle Idx = SE->getSCEV(OpVal); 3667db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 367ceb4d1aecb9deffe59b3dcdc9a783ffde8477be9Duncan Sands uint64_t TypeSize = TD->getTypePaddedSize(GTI.getIndexedType()); 3683416e5f645186a7e3321f927eab662d0ecef404bChris Lattner if (TypeSize != 1) 369246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Idx = SE->getMulExpr(Idx, 370246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SE->getConstant(ConstantInt::get(UIntPtrTy, 371246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman TypeSize))); 372246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman GEPVal = SE->getAddExpr(GEPVal, Idx); 3732f62fdc9a71d790d381d7f17d16e7098e30217e3Chris Lattner } 374eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 375169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 37687265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner SE->setSCEV(GEP, GEPVal); 3772f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen GEPlist.push_back(GEP); 3783416e5f645186a7e3321f927eab662d0ecef404bChris Lattner return GEPVal; 379169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 380169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 3812f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// containsAddRecFromDifferentLoop - Determine whether expression S involves a 3822f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// subexpression that is an AddRec from a loop other than L. An outer loop 3832f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// of L is OK, but not an inner loop nor a disjoint loop. 3842f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesenstatic bool containsAddRecFromDifferentLoop(SCEVHandle S, Loop *L) { 3852f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // This is very common, put it first. 3862f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (isa<SCEVConstant>(S)) 3872f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 3882f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVCommutativeExpr *AE = dyn_cast<SCEVCommutativeExpr>(S)) { 3892f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned int i=0; i< AE->getNumOperands(); i++) 3902f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (containsAddRecFromDifferentLoop(AE->getOperand(i), L)) 3912f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return true; 3922f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 3932f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 3942f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVAddRecExpr *AE = dyn_cast<SCEVAddRecExpr>(S)) { 3952f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (const Loop *newLoop = AE->getLoop()) { 3962f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (newLoop == L) 3972f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 3982f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // if newLoop is an outer loop of L, this is OK. 3992f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!LoopInfoBase<BasicBlock>::isNotAlreadyContainedIn(L, newLoop)) 4002f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 4012f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 4022f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return true; 4032f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 4042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVUDivExpr *DE = dyn_cast<SCEVUDivExpr>(S)) 4052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(DE->getLHS(), L) || 4062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen containsAddRecFromDifferentLoop(DE->getRHS(), L); 4072f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen#if 0 4082f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // SCEVSDivExpr has been backed out temporarily, but will be back; we'll 4092f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // need this when it is. 4102f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVSDivExpr *DE = dyn_cast<SCEVSDivExpr>(S)) 4112f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(DE->getLHS(), L) || 4122f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen containsAddRecFromDifferentLoop(DE->getRHS(), L); 4132f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen#endif 4142f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVTruncateExpr *TE = dyn_cast<SCEVTruncateExpr>(S)) 4152f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(TE->getOperand(), L); 4162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVZeroExtendExpr *ZE = dyn_cast<SCEVZeroExtendExpr>(S)) 4172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(ZE->getOperand(), L); 4182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVSignExtendExpr *SE = dyn_cast<SCEVSignExtendExpr>(S)) 4192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(SE->getOperand(), L); 4202f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 4212f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen} 4222f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 4237db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner/// getSCEVStartAndStride - Compute the start and stride of this expression, 4247db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner/// returning false if the expression is not a start/stride pair, or true if it 4257db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner/// is. The stride must be a loop invariant expression, but the start may be 4262f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// a mix of loop invariant and loop variant expressions. The start cannot, 4272f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// however, contain an AddRec from a different loop, unless that loop is an 4282f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// outer loop of the current loop. 4297db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattnerstatic bool getSCEVStartAndStride(const SCEVHandle &SH, Loop *L, 430246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle &Start, SCEVHandle &Stride, 4318f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng ScalarEvolution *SE, DominatorTree *DT) { 4327db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner SCEVHandle TheAddRec = Start; // Initialize to zero. 4337db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4347db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // If the outer level is an AddExpr, the operands are all start values except 4357db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // for a nested AddRecExpr. 4367db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(SH)) { 4377db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner for (unsigned i = 0, e = AE->getNumOperands(); i != e; ++i) 4387db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (SCEVAddRecExpr *AddRec = 4397db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner dyn_cast<SCEVAddRecExpr>(AE->getOperand(i))) { 4407db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (AddRec->getLoop() == L) 441246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman TheAddRec = SE->getAddExpr(AddRec, TheAddRec); 4427db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner else 4437db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return false; // Nested IV of some sort? 4447db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } else { 445246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Start = SE->getAddExpr(Start, AE->getOperand(i)); 4467db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } 4477db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4483ed469ccd7b028a030b550d84b7336d146f5d8faReid Spencer } else if (isa<SCEVAddRecExpr>(SH)) { 4497db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner TheAddRec = SH; 4507db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } else { 4517db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return false; // not analyzable. 4527db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } 4537db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4547db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(TheAddRec); 4557db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (!AddRec || AddRec->getLoop() != L) return false; 4567db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4577db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // FIXME: Generalize to non-affine IV's. 4587db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (!AddRec->isAffine()) return false; 4597db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4602f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If Start contains an SCEVAddRecExpr from a different loop, other than an 4611de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // outer loop of the current loop, reject it. SCEV has no concept of 4621de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // operating on one loop at a time so don't confuse it with such expressions. 4639194e8b0c827c97f396b209c7f4eeb75759e7068Dan Gohman if (containsAddRecFromDifferentLoop(AddRec->getOperand(0), L)) 4642f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 4652f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 466246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Start = SE->getAddExpr(Start, AddRec->getOperand(0)); 4677db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4688f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng if (!isa<SCEVConstant>(AddRec->getOperand(1))) { 4695a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng // If stride is an instruction, make sure it dominates the loop preheader. 4708f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng // Otherwise we could end up with a use before def situation. 4715a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng BasicBlock *Preheader = L->getLoopPreheader(); 4725a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng if (!AddRec->getOperand(1)->dominates(Preheader, DT)) 4735a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng return false; 4748f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng 475b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling DOUT << "[" << L->getHeader()->getName() 476b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling << "] Variable stride: " << *AddRec << "\n"; 4778f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng } 47850fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner 47950fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner Stride = AddRec->getOperand(1); 4807db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return true; 4817db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner} 4827db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4830ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// IVUseShouldUsePostIncValue - We have discovered a "User" of an IV expression 4840ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// and now we need to decide whether the user should use the preinc or post-inc 4850ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// value. If this user should use the post-inc version of the IV, return true. 4860ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// 4870ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// Choosing wrong here can break dominance properties (if we choose to use the 4880ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// post-inc value when we cannot) or it can end up adding extra live-ranges to 4890ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// the loop, resulting in reg-reg copies (if we use the pre-inc value when we 4900ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// should use the post-inc value). 4910ae33eb243417982fbdca792460bdd986108ac09Chris Lattnerstatic bool IVUseShouldUsePostIncValue(Instruction *User, Instruction *IV, 4920e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng Loop *L, DominatorTree *DT, Pass *P, 49309fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVectorImpl<Instruction*> &DeadInsts){ 4940ae33eb243417982fbdca792460bdd986108ac09Chris Lattner // If the user is in the loop, use the preinc value. 4950ae33eb243417982fbdca792460bdd986108ac09Chris Lattner if (L->contains(User->getParent())) return false; 4960ae33eb243417982fbdca792460bdd986108ac09Chris Lattner 4975e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner BasicBlock *LatchBlock = L->getLoopLatch(); 4985e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 4995e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // Ok, the user is outside of the loop. If it is dominated by the latch 5005e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // block, use the post-inc value. 501b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel if (DT->dominates(LatchBlock, User->getParent())) 5025e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner return true; 5035e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5045e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // There is one case we have to be careful of: PHI nodes. These little guys 5055e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // can live in blocks that do not dominate the latch block, but (since their 5065e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // uses occur in the predecessor block, not the block the PHI lives in) should 5075e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // still use the post-inc value. Check for this case now. 5085e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner PHINode *PN = dyn_cast<PHINode>(User); 5095e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (!PN) return false; // not a phi, not dominated by latch block. 5105e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5115e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // Look at all of the uses of IV by the PHI node. If any use corresponds to 5125e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // a block that is not dominated by the latch block, give up and use the 5135e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // preincremented value. 5145e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner unsigned NumUses = 0; 5155e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 5165e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (PN->getIncomingValue(i) == IV) { 5175e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner ++NumUses; 518b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel if (!DT->dominates(LatchBlock, PN->getIncomingBlock(i))) 5195e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner return false; 5205e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner } 5215e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5225e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // Okay, all uses of IV by PN are in predecessor blocks that really are 5235e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // dominated by the latch block. Split the critical edges and use the 5245e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // post-incremented value. 5255e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 5265e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (PN->getIncomingValue(i) == IV) { 5278392772727ed9105c92fe4514d53dab74c333edcEvan Cheng SplitCriticalEdge(PN->getIncomingBlock(i), PN->getParent(), P, false); 5281b9c8e73b5d620cf2d9a8e150b179fe95ddb8c4eChris Lattner // Splitting the critical edge can reduce the number of entries in this 5291b9c8e73b5d620cf2d9a8e150b179fe95ddb8c4eChris Lattner // PHI. 5301b9c8e73b5d620cf2d9a8e150b179fe95ddb8c4eChris Lattner e = PN->getNumIncomingValues(); 5315e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (--NumUses == 0) break; 5325e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner } 5330e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 5340e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng // PHI node might have become a constant value after SplitCriticalEdge. 53509fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(User); 5365e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5375e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner return true; 5380ae33eb243417982fbdca792460bdd986108ac09Chris Lattner} 5390ae33eb243417982fbdca792460bdd986108ac09Chris Lattner 540f284ce203b656f1d6f933b67ad096a93d3cd7290Dan Gohman/// isAddressUse - Returns true if the specified instruction is using the 541203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// specified value as an address. 542203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstatic bool isAddressUse(Instruction *Inst, Value *OperandVal) { 543203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen bool isAddress = isa<LoadInst>(Inst); 544203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 545203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (SI->getOperand(1) == OperandVal) 546203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen isAddress = true; 547203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 548203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Addressing modes can also be folded into prefetches and a variety 549203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // of intrinsics. 550203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen switch (II->getIntrinsicID()) { 551203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen default: break; 552203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::prefetch: 553203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_loadu_dq: 554203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_loadu_pd: 555203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse_loadu_ps: 556203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse_storeu_ps: 557203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storeu_pd: 558203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storeu_dq: 559203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storel_dq: 560203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (II->getOperand(1) == OperandVal) 561203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen isAddress = true; 562203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen break; 563203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 564203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 565203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen return isAddress; 566203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen} 5670ae33eb243417982fbdca792460bdd986108ac09Chris Lattner 568169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// AddUsersIfInteresting - Inspect the specified instruction. If it is a 569169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// reducible SCEV, recursively add its users to the IVUsesByStride set and 570169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// return true. Otherwise, return false. 5713416e5f645186a7e3321f927eab662d0ecef404bChris Lattnerbool LoopStrengthReduce::AddUsersIfInteresting(Instruction *I, Loop *L, 572168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SmallPtrSet<Instruction*,16> &Processed) { 57342a75517250017a52afb03a0ade03cbd49559fe5Chris Lattner if (!I->getType()->isInteger() && !isa<PointerType>(I->getType())) 5744a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman return false; // Void and FP expressions cannot be reduced. 575168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng if (!Processed.insert(I)) 5763416e5f645186a7e3321f927eab662d0ecef404bChris Lattner return true; // Instruction already handled. 5773416e5f645186a7e3321f927eab662d0ecef404bChris Lattner 5787db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // Get the symbolic expression for this instruction. 5798480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman SCEVHandle ISE = GetExpressionSCEV(I); 5807db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (isa<SCEVCouldNotCompute>(ISE)) return false; 5817db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 5827db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // Get the start and stride for this expression. 583246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Start = SE->getIntegerSCEV(0, ISE->getType()); 58450fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner SCEVHandle Stride = Start; 5858f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng if (!getSCEVStartAndStride(ISE, L, Start, Stride, SE, DT)) 5867db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return false; // Non-reducible symbolic expression, bail out. 5874fe26582c09ec19873753cb4e9bb2ac3cdace88aDevang Patel 5882a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel std::vector<Instruction *> IUsers; 5892a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel // Collect all I uses now because IVUseShouldUsePostIncValue may 5902a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel // invalidate use_iterator. 5912a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); UI != E; ++UI) 5922a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel IUsers.push_back(cast<Instruction>(*UI)); 593169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 5942a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel for (unsigned iused_index = 0, iused_size = IUsers.size(); 5952a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel iused_index != iused_size; ++iused_index) { 5962a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel 5972a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel Instruction *User = IUsers[iused_index]; 5984fe26582c09ec19873753cb4e9bb2ac3cdace88aDevang Patel 599169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Do not infinitely recurse on PHI nodes. 600396b2baf3ca0ee46e696713d34943035206d7a46Chris Lattner if (isa<PHINode>(User) && Processed.count(User)) 601169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman continue; 602169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 6032f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Descend recursively, but not into PHI nodes outside the current loop. 6042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // It's important to see the entire expression outside the loop to get 6052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // choices that depend on addressing mode use right, although we won't 6062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // consider references ouside the loop in all cases. 6072f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If User is already in Processed, we don't want to recurse into it again, 6082f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // but do want to record a second reference in the same instruction. 6097db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner bool AddUserToIVUsers = false; 610f9186596f06f2c1c5357420d44e2fe2f38f98068Chris Lattner if (LI->getLoopFor(User->getParent()) != L) { 6112f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (isa<PHINode>(User) || Processed.count(User) || 6122f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !AddUsersIfInteresting(User, L, Processed)) { 6132f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen DOUT << "FOUND USER in other loop: " << *User 6142f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen << " OF SCEV: " << *ISE << "\n"; 6152f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen AddUserToIVUsers = true; 6162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 6172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } else if (Processed.count(User) || 6182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !AddUsersIfInteresting(User, L, Processed)) { 619b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling DOUT << "FOUND USER: " << *User 620b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling << " OF SCEV: " << *ISE << "\n"; 6217db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner AddUserToIVUsers = true; 6227db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } 623fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 6247db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (AddUserToIVUsers) { 6257305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner IVUsersOfOneStride &StrideUses = IVUsesByStride[Stride]; 626b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen if (StrideUses.Users.empty()) // First occurrence of this stride? 6277305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideOrder.push_back(Stride); 6287305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner 629a4479ad25f7f184fc4600beb1d39fd1e71849c4dChris Lattner // Okay, we found a user that we cannot reduce. Analyze the instruction 630c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // and decide what to do with it. If we are a use inside of the loop, use 631c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // the value before incrementation, otherwise use it after incrementation. 6320e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng if (IVUseShouldUsePostIncValue(User, I, L, DT, this, DeadInsts)) { 633c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // The value used will be incremented by the stride more than we are 634c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // expecting, so subtract this off. 635246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle NewStart = SE->getMinusSCEV(Start, Stride); 6367305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideUses.addUser(NewStart, User, I); 6377305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideUses.Users.back().isUseOfPostIncrementedValue = true; 638b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling DOUT << " USING POSTINC SCEV, START=" << *NewStart<< "\n"; 6390ae33eb243417982fbdca792460bdd986108ac09Chris Lattner } else { 6407305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideUses.addUser(Start, User, I); 641c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner } 642169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 643169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 644169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman return true; 645169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 646169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 647169974856781a1ce27af9ce6220c390b20c9e6ddNate Begemannamespace { 648169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// BasedUser - For a particular base value, keep information about how we've 649169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// partitioned the expression so far. 650169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman struct BasedUser { 651246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman /// SE - The current ScalarEvolution object. 652246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE; 653246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman 654a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// Base - The Base value for the PHI node that needs to be inserted for 655a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// this use. As the use is processed, information gets moved from this 656a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// field to the Imm field (below). BasedUser values are sorted by this 657a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// field. 658a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner SCEVHandle Base; 659a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 660169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Inst - The instruction using the induction variable. 661169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman Instruction *Inst; 662169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 663ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// OperandValToReplace - The operand value of Inst to replace with the 664ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// EmittedBase. 665ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Value *OperandValToReplace; 666169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 667169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Imm - The immediate value that should be added to the base immediately 668169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// before Inst, because it will be folded into the imm field of the 669169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// instruction. 670169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman SCEVHandle Imm; 671169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 672c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// Phi - The induction variable that performs the striding that 673c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// should be used for this user. 674c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *Phi; 675c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 676c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// IncV - The post-incremented value of Phi. 677c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *IncV; 678c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 679010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // isUseOfPostIncrementedValue - True if this should use the 680010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // post-incremented version of this IV, not the preincremented version. 681010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // This can only be set in special cases, such as the terminating setcc 682c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // instruction for a loop and uses outside the loop that are dominated by 683c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // the loop. 684010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner bool isUseOfPostIncrementedValue; 685a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 686246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman BasedUser(IVStrideUse &IVSU, ScalarEvolution *se) 687246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman : SE(se), Base(IVSU.Offset), Inst(IVSU.User), 688a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner OperandValToReplace(IVSU.OperandValToReplace), 689308f24d4525a6365f8d65ba821786189c080c0ceDale Johannesen Imm(SE->getIntegerSCEV(0, Base->getType())), 690a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner isUseOfPostIncrementedValue(IVSU.isUseOfPostIncrementedValue) {} 691169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 6922114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Once we rewrite the code to insert the new IVs we want, update the 6932114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // operands of Inst to use the new expression 'NewBase', with 'Imm' added 6942114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // to it. 6951bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner void RewriteInstructionToUseNewBase(const SCEVHandle &NewBase, 696f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *InsertPt, 6970e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng SCEVExpander &Rewriter, Loop *L, Pass *P, 69809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVectorImpl<Instruction*> &DeadInsts); 699221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 700221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Value *InsertCodeForBaseAtPosition(const SCEVHandle &NewBase, 701221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVExpander &Rewriter, 702221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Instruction *IP, Loop *L); 703169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman void dump() const; 704169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman }; 705169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 706169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 707169974856781a1ce27af9ce6220c390b20c9e6ddNate Begemanvoid BasedUser::dump() const { 708e81561909d128c6e2d8033cb5465a49b2596b26aBill Wendling cerr << " Base=" << *Base; 709e81561909d128c6e2d8033cb5465a49b2596b26aBill Wendling cerr << " Imm=" << *Imm; 710e81561909d128c6e2d8033cb5465a49b2596b26aBill Wendling cerr << " Inst: " << *Inst; 711169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 712169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 713221fc3c6d69bd3854e9121f51e3283492c222ab7Chris LattnerValue *BasedUser::InsertCodeForBaseAtPosition(const SCEVHandle &NewBase, 714221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVExpander &Rewriter, 715221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Instruction *IP, Loop *L) { 716221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Figure out where we *really* want to insert this code. In particular, if 717221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // the user is inside of a loop that is nested inside of L, we really don't 718221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // want to insert this expression before the user, we'd rather pull it out as 719221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // many loops as possible. 720221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner LoopInfo &LI = Rewriter.getLoopInfo(); 721221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Instruction *BaseInsertPt = IP; 722221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 723221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Figure out the most-nested loop that IP is in. 724221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Loop *InsertLoop = LI.getLoopFor(IP->getParent()); 725221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 726221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If InsertLoop is not L, and InsertLoop is nested inside of L, figure out 727221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // the preheader of the outer-most loop where NewBase is not loop invariant. 728eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen if (L->contains(IP->getParent())) 729eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen while (InsertLoop && NewBase->isLoopInvariant(InsertLoop)) { 730eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen BaseInsertPt = InsertLoop->getLoopPreheader()->getTerminator(); 731eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen InsertLoop = InsertLoop->getParentLoop(); 732eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen } 733221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 7342f09f519542202b8af227c2a524f8fe82378a934Dan Gohman Value *Base = Rewriter.expandCodeFor(NewBase, BaseInsertPt); 7352f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 736221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If there is no immediate value, skip the next part. 737cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman if (Imm->isZero()) 7382f09f519542202b8af227c2a524f8fe82378a934Dan Gohman return Base; 739b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner 740b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner // If we are inserting the base and imm values in the same block, make sure to 741b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner // adjust the IP position if insertion reused a result. 742b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner if (IP == BaseInsertPt) 743b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner IP = Rewriter.getInsertionPoint(); 744221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 745221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Always emit the immediate (if non-zero) into the same block as the user. 746246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle NewValSCEV = SE->getAddExpr(SE->getUnknown(Base), Imm); 747d19534add90a2a894af61523b830887097bb780bDan Gohman return Rewriter.expandCodeFor(NewValSCEV, IP); 748b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner 749221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner} 750221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 751221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 7522114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner// Once we rewrite the code to insert the new IVs we want, update the 7532114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner// operands of Inst to use the new expression 'NewBase', with 'Imm' added 754f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// to it. NewBasePt is the last instruction which contributes to the 755f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// value of NewBase in the case that it's a diffferent instruction from 756f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// the PHI that NewBase is computed from, or null otherwise. 757f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// 7581bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattnervoid BasedUser::RewriteInstructionToUseNewBase(const SCEVHandle &NewBase, 759f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *NewBasePt, 7600e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng SCEVExpander &Rewriter, Loop *L, Pass *P, 76109fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVectorImpl<Instruction*> &DeadInsts){ 7622114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner if (!isa<PHINode>(Inst)) { 763c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // By default, insert code at the user instruction. 764c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner BasicBlock::iterator InsertPt = Inst; 765c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner 766c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // However, if the Operand is itself an instruction, the (potentially 767c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // complex) inserted code may be shared by many users. Because of this, we 768c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // want to emit code for the computation of the operand right before its old 769c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // computation. This is usually safe, because we obviously used to use the 770c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // computation when it was computed in its current block. However, in some 771c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // cases (e.g. use of a post-incremented induction variable) the NewBase 772c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // value will be pinned to live somewhere after the original computation. 773c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // In this case, we have to back off. 774f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // 775f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If this is a use outside the loop (which means after, since it is based 776f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // on a loop indvar) we use the post-incremented value, so that we don't 777f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // artificially make the preinc value live out the bottom of the loop. 778589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (!isUseOfPostIncrementedValue && L->contains(Inst->getParent())) { 779ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman if (NewBasePt && isa<PHINode>(OperandValToReplace)) { 780f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman InsertPt = NewBasePt; 781f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman ++InsertPt; 7826725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif } else if (Instruction *OpInst 7836725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif = dyn_cast<Instruction>(OperandValToReplace)) { 784c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner InsertPt = OpInst; 785c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner while (isa<PHINode>(InsertPt)) ++InsertPt; 786c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner } 787c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner } 788c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner Value *NewVal = InsertCodeForBaseAtPosition(NewBase, Rewriter, InsertPt, L); 789a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman // Adjust the type back to match the Inst. Note that we can't use InsertPt 790a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman // here because the SCEVExpander may have inserted the instructions after 791a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman // that point, in its efforts to avoid inserting redundant expressions. 792d19534add90a2a894af61523b830887097bb780bDan Gohman if (isa<PointerType>(OperandValToReplace->getType())) { 793a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman NewVal = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr, 794a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman NewVal, 795a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman OperandValToReplace->getType()); 796d19534add90a2a894af61523b830887097bb780bDan Gohman } 7972114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Replace the use of the operand Value with the new Phi we just created. 7982114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner Inst->replaceUsesOfWith(OperandValToReplace, NewVal); 7992f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8002f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Replacing with "; 8014a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, NewVal, /*PrintType=*/false)); 8022f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << ", which has value " << *NewBase << " plus IMM " << *Imm << "\n"; 8032114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner return; 8042114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 8052f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8062114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // PHI nodes are more complex. We have to insert one copy of the NewBase+Imm 807c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // expression into each operand block that uses it. Note that PHI nodes can 808c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // have multiple entries for the same predecessor. We use a map to make sure 809c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // that a PHI node only has a single Value* for each predecessor (which also 810c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // prevents us from inserting duplicate code in some blocks). 8118392772727ed9105c92fe4514d53dab74c333edcEvan Cheng DenseMap<BasicBlock*, Value*> InsertedCode; 8122114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner PHINode *PN = cast<PHINode>(Inst); 8132114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 8142114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner if (PN->getIncomingValue(i) == OperandValToReplace) { 8152f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If the original expression is outside the loop, put the replacement 8162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // code in the same place as the original expression, 8172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // which need not be an immediate predecessor of this PHI. This way we 8182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // need only one copy of it even if it is referenced multiple times in 8192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // the PHI. We don't do this when the original expression is inside the 8201de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // loop because multiple copies sometimes do useful sinking of code in 8211de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // that case(?). 8222f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen Instruction *OldLoc = dyn_cast<Instruction>(OperandValToReplace); 8232f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(OldLoc->getParent())) { 8241de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // If this is a critical edge, split the edge so that we do not insert 8251de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // the code on all predecessor/successor paths. We do this unless this 8261de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // is the canonical backedge for this loop, as this can make some 8271de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // inserted code be in an illegal position. 8282f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen BasicBlock *PHIPred = PN->getIncomingBlock(i); 8292f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (e != 1 && PHIPred->getTerminator()->getNumSuccessors() > 1 && 8302f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen (PN->getParent() != L->getHeader() || !L->contains(PHIPred))) { 8312f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 8322f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // First step, split the critical edge. 8332f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SplitCriticalEdge(PHIPred, PN->getParent(), P, false); 8342f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 8352f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Next step: move the basic block. In particular, if the PHI node 8362f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // is outside of the loop, and PredTI is in the loop, we want to 8372f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // move the block to be immediately before the PHI block, not 8382f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // immediately after PredTI. 8392f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(PHIPred) && !L->contains(PN->getParent())) { 8402f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen BasicBlock *NewBB = PN->getIncomingBlock(i); 8412f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen NewBB->moveBefore(PN->getParent()); 8422f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 8432f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 8442f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Splitting the edge can reduce the number of PHI entries we have. 8452f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen e = PN->getNumIncomingValues(); 846e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner } 847e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner } 848c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner Value *&Code = InsertedCode[PN->getIncomingBlock(i)]; 849c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner if (!Code) { 850c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // Insert the code into the end of the predecessor block. 8512f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen Instruction *InsertPt = (L->contains(OldLoc->getParent())) ? 8522f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen PN->getIncomingBlock(i)->getTerminator() : 8532f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen OldLoc->getParent()->getTerminator(); 854221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Code = InsertCodeForBaseAtPosition(NewBase, Rewriter, InsertPt, L); 855d19534add90a2a894af61523b830887097bb780bDan Gohman 856684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // Adjust the type back to match the PHI. Note that we can't use 857684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // InsertPt here because the SCEVExpander may have inserted its 858684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // instructions after that point, in its efforts to avoid inserting 859684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // redundant expressions. 860d19534add90a2a894af61523b830887097bb780bDan Gohman if (isa<PointerType>(PN->getType())) { 861a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman Code = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr, 862a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman Code, 863a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman PN->getType()); 864d19534add90a2a894af61523b830887097bb780bDan Gohman } 8652f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8662f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Changing PHI use to "; 8674a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, Code, /*PrintType=*/false)); 8682f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << ", which has value " << *NewBase << " plus IMM " << *Imm << "\n"; 869c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner } 8702f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8712114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Replace the use of the operand Value with the new Phi we just created. 872c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner PN->setIncomingValue(i, Code); 8732114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner Rewriter.clear(); 8742114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 8752114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 8760e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 8770e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng // PHI node might have become a constant value after SplitCriticalEdge. 87809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(Inst); 8792114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner} 8802114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner 8812114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner 882203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// fitsInAddressMode - Return true if V can be subsumed within an addressing 883203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// mode, and does not need to be put in a register first. 884203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstatic bool fitsInAddressMode(const SCEVHandle &V, const Type *UseTy, 885203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const TargetLowering *TLI, bool HasBaseReg) { 8863821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner if (SCEVConstant *SC = dyn_cast<SCEVConstant>(V)) { 8875eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t VC = SC->getValue()->getSExtValue(); 888579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (TLI) { 889579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 890579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseOffs = VC; 891203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen AM.HasBaseReg = HasBaseReg; 892579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner return TLI->isLegalAddressingMode(AM, UseTy); 893579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } else { 894d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng // Defaults to PPC. PPC allows a sign-extended 16-bit immediate field. 8955eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng return (VC > -(1 << 16) && VC < (1 << 16)-1); 896579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } 8973821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner } 898d29b6aa608d69f19b57ebd2ae630b040b1c4951dJeff Cohen 899169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman if (SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) 900169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman if (ConstantExpr *CE = dyn_cast<ConstantExpr>(SU->getValue())) 901579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (TLI && CE->getOpcode() == Instruction::PtrToInt) { 902d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng Constant *Op0 = CE->getOperand(0); 903579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (GlobalValue *GV = dyn_cast<GlobalValue>(Op0)) { 904579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 905579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseGV = GV; 906203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen AM.HasBaseReg = HasBaseReg; 907579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner return TLI->isLegalAddressingMode(AM, UseTy); 908579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } 909d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng } 910169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman return false; 911169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 912169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 913544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen/// MoveLoopVariantsToImmediateField - Move any subexpressions from Val that are 91444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner/// loop varying to the Imm operand. 915544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesenstatic void MoveLoopVariantsToImmediateField(SCEVHandle &Val, SCEVHandle &Imm, 916246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Loop *L, ScalarEvolution *SE) { 91744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (Val->isLoopInvariant(L)) return; // Nothing to do. 91844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 91944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (SCEVAddExpr *SAE = dyn_cast<SCEVAddExpr>(Val)) { 92044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner std::vector<SCEVHandle> NewOps; 92144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner NewOps.reserve(SAE->getNumOperands()); 92244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 92344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner for (unsigned i = 0; i != SAE->getNumOperands(); ++i) 92444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (!SAE->getOperand(i)->isLoopInvariant(L)) { 92544b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // If this is a loop-variant expression, it must stay in the immediate 92644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // field of the expression. 927246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, SAE->getOperand(i)); 92844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else { 92944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner NewOps.push_back(SAE->getOperand(i)); 93044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 93144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 93244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (NewOps.empty()) 933246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 93444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner else 935246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddExpr(NewOps); 93644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else if (SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Val)) { 93744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Try to pull immediates out of the start value of nested addrec's. 93844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner SCEVHandle Start = SARE->getStart(); 939544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen MoveLoopVariantsToImmediateField(Start, Imm, L, SE); 94044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 94144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner std::vector<SCEVHandle> Ops(SARE->op_begin(), SARE->op_end()); 94244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner Ops[0] = Start; 943246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddRecExpr(Ops, SARE->getLoop()); 94444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else { 94544b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Otherwise, all of Val is variant, move the whole thing over. 946246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, Val); 947246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 94844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 94944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner} 95044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 95144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 95226d91f16464db56283087176a73981048331dd2dChris Lattner/// MoveImmediateValues - Look at Val, and pull out any additions of constants 953169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// that can fit into the immediate field of instructions in the target. 95426d91f16464db56283087176a73981048331dd2dChris Lattner/// Accumulate these immediate values into the Imm value. 955d277f2c66914aecb619c12855f6afae4c7ef883bEvan Chengstatic void MoveImmediateValues(const TargetLowering *TLI, 9561d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng Instruction *User, 957d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng SCEVHandle &Val, SCEVHandle &Imm, 958246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman bool isAddress, Loop *L, 959246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE) { 9601d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng const Type *UseTy = User->getType(); 9611d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng if (StoreInst *SI = dyn_cast<StoreInst>(User)) 9621d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng UseTy = SI->getOperand(0)->getType(); 9631d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng 9647a65839f4118c99fb1636c3cbb41b0bf7ef27665Chris Lattner if (SCEVAddExpr *SAE = dyn_cast<SCEVAddExpr>(Val)) { 96526d91f16464db56283087176a73981048331dd2dChris Lattner std::vector<SCEVHandle> NewOps; 96626d91f16464db56283087176a73981048331dd2dChris Lattner NewOps.reserve(SAE->getNumOperands()); 96726d91f16464db56283087176a73981048331dd2dChris Lattner 968221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner for (unsigned i = 0; i != SAE->getNumOperands(); ++i) { 969221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVHandle NewOp = SAE->getOperand(i); 970246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman MoveImmediateValues(TLI, User, NewOp, Imm, isAddress, L, SE); 971221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 972221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner if (!NewOp->isLoopInvariant(L)) { 9737db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // If this is a loop-variant expression, it must stay in the immediate 9747db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // field of the expression. 975246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, NewOp); 97626d91f16464db56283087176a73981048331dd2dChris Lattner } else { 977221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner NewOps.push_back(NewOp); 978169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 979221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 98026d91f16464db56283087176a73981048331dd2dChris Lattner 98126d91f16464db56283087176a73981048331dd2dChris Lattner if (NewOps.empty()) 982246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 98326d91f16464db56283087176a73981048331dd2dChris Lattner else 984246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddExpr(NewOps); 98526d91f16464db56283087176a73981048331dd2dChris Lattner return; 9867a65839f4118c99fb1636c3cbb41b0bf7ef27665Chris Lattner } else if (SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Val)) { 9877a65839f4118c99fb1636c3cbb41b0bf7ef27665Chris Lattner // Try to pull immediates out of the start value of nested addrec's. 98826d91f16464db56283087176a73981048331dd2dChris Lattner SCEVHandle Start = SARE->getStart(); 989246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman MoveImmediateValues(TLI, User, Start, Imm, isAddress, L, SE); 99026d91f16464db56283087176a73981048331dd2dChris Lattner 99126d91f16464db56283087176a73981048331dd2dChris Lattner if (Start != SARE->getStart()) { 99226d91f16464db56283087176a73981048331dd2dChris Lattner std::vector<SCEVHandle> Ops(SARE->op_begin(), SARE->op_end()); 99326d91f16464db56283087176a73981048331dd2dChris Lattner Ops[0] = Start; 994246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddRecExpr(Ops, SARE->getLoop()); 99526d91f16464db56283087176a73981048331dd2dChris Lattner } 99626d91f16464db56283087176a73981048331dd2dChris Lattner return; 997221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } else if (SCEVMulExpr *SME = dyn_cast<SCEVMulExpr>(Val)) { 998221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Transform "8 * (4 + v)" -> "32 + 8*V" if "32" fits in the immed field. 999203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (isAddress && fitsInAddressMode(SME->getOperand(0), UseTy, TLI, false) && 1000221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SME->getNumOperands() == 2 && SME->isLoopInvariant(L)) { 1001221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 1002246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle SubImm = SE->getIntegerSCEV(0, Val->getType()); 1003221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVHandle NewOp = SME->getOperand(1); 1004246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman MoveImmediateValues(TLI, User, NewOp, SubImm, isAddress, L, SE); 1005221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 1006221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If we extracted something out of the subexpressions, see if we can 1007221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // simplify this! 1008221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner if (NewOp != SME->getOperand(1)) { 1009221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Scale SubImm up by "8". If the result is a target constant, we are 1010221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // good. 1011246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SubImm = SE->getMulExpr(SubImm, SME->getOperand(0)); 1012203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (fitsInAddressMode(SubImm, UseTy, TLI, false)) { 1013221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Accumulate the immediate. 1014246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, SubImm); 1015221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 1016221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Update what is left of 'Val'. 1017246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getMulExpr(SME->getOperand(0), NewOp); 1018221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner return; 1019221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 1020221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 1021221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 1022169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 1023169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 102426d91f16464db56283087176a73981048331dd2dChris Lattner // Loop-variant expressions must stay in the immediate field of the 102526d91f16464db56283087176a73981048331dd2dChris Lattner // expression. 1026203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if ((isAddress && fitsInAddressMode(Val, UseTy, TLI, false)) || 102726d91f16464db56283087176a73981048331dd2dChris Lattner !Val->isLoopInvariant(L)) { 1028246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, Val); 1029246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 103026d91f16464db56283087176a73981048331dd2dChris Lattner return; 10317a2ca56ef3bdda6874bcd4df2910fb5a33999f85Chris Lattner } 103226d91f16464db56283087176a73981048331dd2dChris Lattner 103326d91f16464db56283087176a73981048331dd2dChris Lattner // Otherwise, no immediates to move. 1034169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 1035169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 1036934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 10377e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// SeparateSubExprs - Decompose Expr into all of the subexpressions that are 10387e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// added together. This is used to reassociate common addition subexprs 10397e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// together for maximal sharing when rewriting bases. 1040934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattnerstatic void SeparateSubExprs(std::vector<SCEVHandle> &SubExprs, 1041246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Expr, 1042246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE) { 1043934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(Expr)) { 1044934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned j = 0, e = AE->getNumOperands(); j != e; ++j) 1045246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, AE->getOperand(j), SE); 1046934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } else if (SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Expr)) { 1047246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Zero = SE->getIntegerSCEV(0, Expr->getType()); 1048934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SARE->getOperand(0) == Zero) { 1049934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.push_back(Expr); 1050934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } else { 1051934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Compute the addrec with zero as its base. 1052934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner std::vector<SCEVHandle> Ops(SARE->op_begin(), SARE->op_end()); 1053934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner Ops[0] = Zero; // Start with zero base. 1054246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SubExprs.push_back(SE->getAddRecExpr(Ops, SARE->getLoop())); 1055934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1056934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1057246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, SARE->getOperand(0), SE); 1058934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1059cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman } else if (!Expr->isZero()) { 1060934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Do not add zero. 1061934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.push_back(Expr); 1062934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1063934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner} 1064934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1065203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen// This is logically local to the following function, but C++ says we have 1066203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen// to make it file scope. 1067203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstruct SubExprUseData { unsigned Count; bool notAllUsesAreFree; }; 1068934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1069203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// RemoveCommonExpressionsFromUseBases - Look through all of the Bases of all 1070203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// the Uses, removing any common subexpressions, except that if all such 1071203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// subexpressions can be folded into an addressing mode for all uses inside 1072203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// the loop (this case is referred to as "free" in comments herein) we do 1073203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// not remove anything. This looks for things like (a+b+c) and 1074f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner/// (a+c+d) and computes the common (a+c) subexpression. The common expression 1075f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner/// is *removed* from the Bases and returned. 10761bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattnerstatic SCEVHandle 1077246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan GohmanRemoveCommonExpressionsFromUseBases(std::vector<BasedUser> &Uses, 1078203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen ScalarEvolution *SE, Loop *L, 1079203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const TargetLowering *TLI) { 10801bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner unsigned NumUses = Uses.size(); 10811bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1082f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Only one use? This is a very common case, so we handle it specially and 1083f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // cheaply. 1084246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Zero = SE->getIntegerSCEV(0, Uses[0].Base->getType()); 10851bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner SCEVHandle Result = Zero; 1086203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SCEVHandle FreeResult = Zero; 10871bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner if (NumUses == 1) { 1088f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If the use is inside the loop, use its base, regardless of what it is: 1089f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // it is clearly shared across all the IV's. If the use is outside the loop 1090f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // (which means after it) we don't want to factor anything *into* the loop, 1091f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // so just use 0 as the base. 1092589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (L->contains(Uses[0].Inst->getParent())) 1093589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen std::swap(Result, Uses[0].Base); 10941bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner return Result; 10951bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner } 10961bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 10971bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // To find common subexpressions, count how many of Uses use each expression. 10981bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If any subexpressions are used Uses.size() times, they are common. 1099203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Also track whether all uses of each expression can be moved into an 1100203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // an addressing mode "for free"; such expressions are left within the loop. 1101203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // struct SubExprUseData { unsigned Count; bool notAllUsesAreFree; }; 1102203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen std::map<SCEVHandle, SubExprUseData> SubExpressionUseData; 11031bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1104d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // UniqueSubExprs - Keep track of all of the subexpressions we see in the 1105d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // order we see them. 1106d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner std::vector<SCEVHandle> UniqueSubExprs; 1107d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner 1108934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner std::vector<SCEVHandle> SubExprs; 1109f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner unsigned NumUsesInsideLoop = 0; 1110934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned i = 0; i != NumUses; ++i) { 1111f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If the user is outside the loop, just ignore it for base computation. 1112f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Since the user is outside the loop, it must be *after* the loop (if it 1113f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // were before, it could not be based on the loop IV). We don't want users 1114f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // after the loop to affect base computation of values *inside* the loop, 1115f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // because we can always add their offsets to the result IV after the loop 1116f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // is done, ensuring we get good code inside the loop. 1117589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (!L->contains(Uses[i].Inst->getParent())) 1118589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen continue; 1119589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen NumUsesInsideLoop++; 1120589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 1121934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // If the base is zero (which is common), return zero now, there are no 1122934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // CSEs we can find. 1123934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (Uses[i].Base == Zero) return Zero; 1124934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1125203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // If this use is as an address we may be able to put CSEs in the addressing 1126203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // mode rather than hoisting them. 1127203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen bool isAddrUse = isAddressUse(Uses[i].Inst, Uses[i].OperandValToReplace); 1128203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We may need the UseTy below, but only when isAddrUse, so compute it 1129203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // only in that case. 1130203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const Type *UseTy = 0; 1131203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (isAddrUse) { 1132203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen UseTy = Uses[i].Inst->getType(); 1133203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (StoreInst *SI = dyn_cast<StoreInst>(Uses[i].Inst)) 1134203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen UseTy = SI->getOperand(0)->getType(); 1135203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1136203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 1137934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Split the expression into subexprs. 1138246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, Uses[i].Base, SE); 1139203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Add one to SubExpressionUseData.Count for each subexpr present, and 1140203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // if the subexpr is not a valid immediate within an addressing mode use, 1141203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // set SubExpressionUseData.notAllUsesAreFree. We definitely want to 1142203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // hoist these out of the loop (if they are common to all uses). 1143203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) { 1144203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (++SubExpressionUseData[SubExprs[j]].Count == 1) 1145d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner UniqueSubExprs.push_back(SubExprs[j]); 1146203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!isAddrUse || !fitsInAddressMode(SubExprs[j], UseTy, TLI, false)) 1147203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData[SubExprs[j]].notAllUsesAreFree = true; 1148203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1149934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.clear(); 1150934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1151934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1152d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // Now that we know how many times each is used, build Result. Iterate over 1153d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // UniqueSubexprs so that we have a stable ordering. 1154d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner for (unsigned i = 0, e = UniqueSubExprs.size(); i != e; ++i) { 1155203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen std::map<SCEVHandle, SubExprUseData>::iterator I = 1156203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.find(UniqueSubExprs[i]); 1157203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen assert(I != SubExpressionUseData.end() && "Entry not found?"); 1158203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (I->second.Count == NumUsesInsideLoop) { // Found CSE! 1159203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (I->second.notAllUsesAreFree) 1160203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen Result = SE->getAddExpr(Result, I->first); 1161203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen else 1162203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen FreeResult = SE->getAddExpr(FreeResult, I->first); 1163203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } else 1164203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Remove non-cse's from SubExpressionUseData. 1165203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.erase(I); 1166d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner } 1167203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 1168203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (FreeResult != Zero) { 1169203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We have some subexpressions that can be subsumed into addressing 1170203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // modes in every use inside the loop. However, it's possible that 1171203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // there are so many of them that the combined FreeResult cannot 1172203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // be subsumed, or that the target cannot handle both a FreeResult 1173203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // and a Result in the same instruction (for example because it would 1174203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // require too many registers). Check this. 1175203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned i=0; i<NumUses; ++i) { 1176203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!L->contains(Uses[i].Inst->getParent())) 1177203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen continue; 1178203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We know this is an addressing mode use; if there are any uses that 1179203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // are not, FreeResult would be Zero. 1180203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const Type *UseTy = Uses[i].Inst->getType(); 1181203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (StoreInst *SI = dyn_cast<StoreInst>(Uses[i].Inst)) 1182203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen UseTy = SI->getOperand(0)->getType(); 1183203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!fitsInAddressMode(FreeResult, UseTy, TLI, Result!=Zero)) { 1184203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // FIXME: could split up FreeResult into pieces here, some hoisted 1185b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // and some not. There is no obvious advantage to this. 1186203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen Result = SE->getAddExpr(Result, FreeResult); 1187203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen FreeResult = Zero; 1188203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen break; 1189203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1190203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1191203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1192203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 11931bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If we found no CSE's, return now. 11941bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner if (Result == Zero) return Result; 11951bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1196203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // If we still have a FreeResult, remove its subexpressions from 1197203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // SubExpressionUseData. This means they will remain in the use Bases. 1198203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (FreeResult != Zero) { 1199203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SeparateSubExprs(SubExprs, FreeResult, SE); 1200203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) { 1201203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen std::map<SCEVHandle, SubExprUseData>::iterator I = 1202203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.find(SubExprs[j]); 1203203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.erase(I); 1204203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1205203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExprs.clear(); 1206203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1207203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 12081bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Otherwise, remove all of the CSE's we found from each of the base values. 1209934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned i = 0; i != NumUses; ++i) { 1210fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // Uses outside the loop don't necessarily include the common base, but 1211fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // the final IV value coming into those uses does. Instead of trying to 1212fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // remove the pieces of the common base, which might not be there, 1213fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // subtract off the base to compensate for this. 1214fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen if (!L->contains(Uses[i].Inst->getParent())) { 1215fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen Uses[i].Base = SE->getMinusSCEV(Uses[i].Base, Result); 1216589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen continue; 1217fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen } 1218589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 1219934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Split the expression into subexprs. 1220246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, Uses[i].Base, SE); 1221934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1222934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Remove any common subexpressions. 1223934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) 1224203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (SubExpressionUseData.count(SubExprs[j])) { 1225934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.erase(SubExprs.begin()+j); 1226934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner --j; --e; 1227934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1228934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1229f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Finally, add the non-shared expressions together. 1230934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SubExprs.empty()) 12311bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Uses[i].Base = Zero; 1232934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner else 1233246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Uses[i].Base = SE->getAddExpr(SubExprs); 123427e5142309946ca12c633b265673af0c24684427Chris Lattner SubExprs.clear(); 1235934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 12361bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 12371bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner return Result; 12381bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner} 12391bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1240dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen/// ValidStride - Check whether the given Scale is valid for all loads and 1241579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner/// stores in UsersToProcess. 1242dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen/// 124302e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohmanbool LoopStrengthReduce::ValidStride(bool HasBaseReg, 124402e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman int64_t Scale, 1245dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess) { 1246d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (!TLI) 1247d6b62a572210aff965a55626cf36a68821838844Evan Cheng return true; 1248d6b62a572210aff965a55626cf36a68821838844Evan Cheng 12498e59e163db8cd3e7b4c96e438fbedf78bff06707Dale Johannesen for (unsigned i=0, e = UsersToProcess.size(); i!=e; ++i) { 12501ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner // If this is a load or other access, pass the type of the access in. 12511ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner const Type *AccessTy = Type::VoidTy; 12521ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner if (StoreInst *SI = dyn_cast<StoreInst>(UsersToProcess[i].Inst)) 12531ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner AccessTy = SI->getOperand(0)->getType(); 12541ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner else if (LoadInst *LI = dyn_cast<LoadInst>(UsersToProcess[i].Inst)) 12551ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner AccessTy = LI->getType(); 125655e641b766a18878b51551d626d5a566102e487eEvan Cheng else if (isa<PHINode>(UsersToProcess[i].Inst)) 125755e641b766a18878b51551d626d5a566102e487eEvan Cheng continue; 12581ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner 1259579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 1260579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (SCEVConstant *SC = dyn_cast<SCEVConstant>(UsersToProcess[i].Imm)) 1261579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseOffs = SC->getValue()->getSExtValue(); 1262cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman AM.HasBaseReg = HasBaseReg || !UsersToProcess[i].Base->isZero(); 1263579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.Scale = Scale; 1264579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner 1265579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner // If load[imm+r*scale] is illegal, bail out. 1266d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (!TLI->isLegalAddressingMode(AM, AccessTy)) 1267dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen return false; 12688e59e163db8cd3e7b4c96e438fbedf78bff06707Dale Johannesen } 1269dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen return true; 1270dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen} 12711bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 12721de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen/// RequiresTypeConversion - Returns true if converting Ty1 to Ty2 is not 12735f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// a nop. 12742bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Chengbool LoopStrengthReduce::RequiresTypeConversion(const Type *Ty1, 12752bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng const Type *Ty2) { 12762bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (Ty1 == Ty2) 12775f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return false; 12781de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (Ty1->canLosslesslyBitCastTo(Ty2)) 12791de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return false; 12802bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (TLI && TLI->isTruncateFree(Ty1, Ty2)) 12812bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng return false; 12821de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (isa<PointerType>(Ty2) && Ty1->canLosslesslyBitCastTo(UIntPtrTy)) 12831de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return false; 12841de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (isa<PointerType>(Ty1) && Ty2->canLosslesslyBitCastTo(UIntPtrTy)) 12851de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return false; 12861de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return true; 12875f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 12885f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1289eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng/// CheckForIVReuse - Returns the multiple if the stride is the multiple 1290eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng/// of a previous stride and it is a legal value for the target addressing 129102e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman/// mode scale component and optional base reg. This allows the users of 129202e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman/// this stride to be rewritten as prev iv * factor. It returns 0 if no 1293b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen/// reuse is possible. Factors can be negative on same targets, e.g. ARM. 12942f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// 12952f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// If all uses are outside the loop, we don't require that all multiplies 12962f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// be folded into the addressing mode, nor even that the factor be constant; 12972f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// a multiply (executed once) outside the loop is better than another IV 12982f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// within. Well, usually. 12992f46bb8178e30e3b845859a44b57c048db06ef84Dale JohannesenSCEVHandle LoopStrengthReduce::CheckForIVReuse(bool HasBaseReg, 13002bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng bool AllUsesAreAddresses, 1301b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool AllUsesAreOutsideLoop, 130202e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman const SCEVHandle &Stride, 1303dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen IVExpr &IV, const Type *Ty, 1304dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess) { 1305eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng if (SCEVConstant *SC = dyn_cast<SCEVConstant>(Stride)) { 1306502db93a8ab376730164db43ca3ce8032b72bd59Reid Spencer int64_t SInt = SC->getValue()->getSExtValue(); 1307b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen for (unsigned NewStride = 0, e = StrideOrder.size(); NewStride != e; 1308b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen ++NewStride) { 1309b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen std::map<SCEVHandle, IVsOfOneStride>::iterator SI = 1310b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen IVsByStride.find(StrideOrder[NewStride]); 13112f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end() || !isa<SCEVConstant>(SI->first)) 1312b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen continue; 13135eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 13142bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (SI->first != Stride && 13151d31290634eccc3b360c427282d59780d76b9169Chris Lattner (unsigned(abs(SInt)) < SSInt || (SInt % SSInt) != 0)) 1316eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng continue; 13175eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t Scale = SInt / SSInt; 1318dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // Check that this stride is valid for all the types used for loads and 1319dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // stores; if it can be used for some and not others, we might as well use 1320dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // the original stride everywhere, since we have to create the IV for it 1321aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman // anyway. If the scale is 1, then we don't need to worry about folding 1322aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman // multiplications. 1323aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman if (Scale == 1 || 1324aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman (AllUsesAreAddresses && 1325aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman ValidStride(HasBaseReg, Scale, UsersToProcess))) 13265eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 13275eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng IE = SI->second.IVs.end(); II != IE; ++II) 13285eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng // FIXME: Only handle base == 0 for now. 13295eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng // Only reuse previous IV if it would not require a type conversion. 1330cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman if (II->Base->isZero() && 13312bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng !RequiresTypeConversion(II->Base->getType(), Ty)) { 13325eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng IV = *II; 13332f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(Scale, Stride->getType()); 13345eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng } 1335eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 13362f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } else if (AllUsesAreOutsideLoop) { 13372f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonconstant strides here; it is really really right to substitute 13382f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // an existing IV if we can. 13392f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned NewStride = 0, e = StrideOrder.size(); NewStride != e; 13402f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen ++NewStride) { 13412f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen std::map<SCEVHandle, IVsOfOneStride>::iterator SI = 13422f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IVsByStride.find(StrideOrder[NewStride]); 13432f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end() || !isa<SCEVConstant>(SI->first)) 13442f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 13452f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 13462f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI->first != Stride && SSInt != 1) 13472f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 13482f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 13492f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IE = SI->second.IVs.end(); II != IE; ++II) 13502f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonzero base here. 13512f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Only reuse previous IV if it would not require a type conversion. 13522f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!RequiresTypeConversion(II->Base->getType(), Ty)) { 13532f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IV = *II; 13542f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return Stride; 13552f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 13562f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 13572f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Special case, old IV is -1*x and this one is x. Can treat this one as 13582f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // -1*old. 13592f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned NewStride = 0, e = StrideOrder.size(); NewStride != e; 13602f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen ++NewStride) { 13612f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen std::map<SCEVHandle, IVsOfOneStride>::iterator SI = 13622f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IVsByStride.find(StrideOrder[NewStride]); 13632f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end()) 13642f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 13652f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(SI->first)) 13662f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVConstant *SC = dyn_cast<SCEVConstant>(ME->getOperand(0))) 13672f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (Stride == ME->getOperand(1) && 13682f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SC->getValue()->getSExtValue() == -1LL) 13692f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 13702f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IE = SI->second.IVs.end(); II != IE; ++II) 13712f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonzero base here. 13722f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Only reuse previous IV if it would not require type conversion. 13732f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!RequiresTypeConversion(II->Base->getType(), Ty)) { 13742f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IV = *II; 13752f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(-1LL, Stride->getType()); 13762f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 13772f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 1378eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 13792f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(0, Stride->getType()); 1380eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng} 1381eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 13827e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// PartitionByIsUseOfPostIncrementedValue - Simple boolean predicate that 13837e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// returns true if Val's isUseOfPostIncrementedValue is true. 13847e79b3898ddd919170d367a516f51296017146c2Chris Lattnerstatic bool PartitionByIsUseOfPostIncrementedValue(const BasedUser &Val) { 13857e79b3898ddd919170d367a516f51296017146c2Chris Lattner return Val.isUseOfPostIncrementedValue; 13867e79b3898ddd919170d367a516f51296017146c2Chris Lattner} 1387eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 13884a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman/// isNonConstantNegative - Return true if the specified scev is negated, but 1389fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner/// not a constant. 1390fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattnerstatic bool isNonConstantNegative(const SCEVHandle &Expr) { 1391fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Expr); 1392fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner if (!Mul) return false; 1393fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 1394fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner // If there is a constant factor, it will be first. 1395fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 1396fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner if (!SC) return false; 1397fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 1398fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner // Return true if the value is negative, this matches things like (-42 * V). 1399fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner return SC->getValue()->getValue().isNegative(); 1400fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner} 1401fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 14025f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng// CollectIVUsers - Transform our list of users and offsets to a bit more 140373b43b9b549a75fb0015c825df68abd95705a67cDan Gohman// complex table. In this new vector, each 'BasedUser' contains 'Base', the base 140473b43b9b549a75fb0015c825df68abd95705a67cDan Gohman// of the strided accesses, as well as the old information from Uses. We 14055f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng// progressively move information from the Base field to the Imm field, until 14065f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng// we eventually have the full access expression to rewrite the use. 14075f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan ChengSCEVHandle LoopStrengthReduce::CollectIVUsers(const SCEVHandle &Stride, 14085f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 14095f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 14105f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool &AllUsesAreAddresses, 1411b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool &AllUsesAreOutsideLoop, 14125f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> &UsersToProcess) { 1413169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman UsersToProcess.reserve(Uses.Users.size()); 1414a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner for (unsigned i = 0, e = Uses.Users.size(); i != e; ++i) { 1415246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess.push_back(BasedUser(Uses.Users[i], SE)); 1416a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 141767c79892949332568b082f124d9598971fa3277fDale Johannesen // Move any loop variant operands from the offset field to the immediate 1418a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner // field of the use, so that we don't try to use something before it is 1419a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner // computed. 1420544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen MoveLoopVariantsToImmediateField(UsersToProcess.back().Base, 1421246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess.back().Imm, L, SE); 1422a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner assert(UsersToProcess.back().Base->isLoopInvariant(L) && 142326d91f16464db56283087176a73981048331dd2dChris Lattner "Base value is not loop invariant!"); 14242461dff0700d0e34b9854d96a5cc03921b375525Chris Lattner } 1425eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 142631e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // We now have a whole bunch of uses of like-strided induction variables, but 142731e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // they might all have different bases. We want to emit one PHI node for this 142831e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // stride which we fold as many common expressions (between the IVs) into as 142931e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // possible. Start by identifying the common expressions in the base values 143031e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // for the strides (e.g. if we have "A+C+B" and "A+B+D" as our bases, find 143131e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // "A+B"), emit it to the preheader, then remove the expression from the 143231e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // UsersToProcess base values. 143331e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng SCEVHandle CommonExprs = 1434203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen RemoveCommonExpressionsFromUseBases(UsersToProcess, SE, L, TLI); 143502e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman 143644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Next, figure out what we can represent in the immediate fields of 143744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // instructions. If we can represent anything there, move it to the imm 14381bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // fields of the BasedUsers. We do this so that it increases the commonality 14391bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // of the remaining uses. 144032e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng unsigned NumPHI = 0; 144144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 144280b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // If the user is not in the current loop, this means it is using the exit 144380b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // value of the IV. Do not put anything in the base, make sure it's all in 144480b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // the immediate field to allow as much factoring as possible. 144580b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner if (!L->contains(UsersToProcess[i].Inst->getParent())) { 1446246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Imm = SE->getAddExpr(UsersToProcess[i].Imm, 1447246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Base); 14488385e51e210fb5c1e7e48eae150b31679b3e137dChris Lattner UsersToProcess[i].Base = 1449246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SE->getIntegerSCEV(0, UsersToProcess[i].Base->getType()); 145080b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner } else { 1451b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen 145280b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // Addressing modes can be folded into loads and stores. Be careful that 145380b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // the store is through the expression, not of the expression though. 145432e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng bool isPHI = false; 1455d6b62a572210aff965a55626cf36a68821838844Evan Cheng bool isAddress = isAddressUse(UsersToProcess[i].Inst, 1456d6b62a572210aff965a55626cf36a68821838844Evan Cheng UsersToProcess[i].OperandValToReplace); 1457d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (isa<PHINode>(UsersToProcess[i].Inst)) { 145832e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng isPHI = true; 145932e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng ++NumPHI; 14602acc7601650654d03cd53faeece8d7685a203105Dan Gohman } 146102e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman 1462b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // Not all uses are outside the loop. 1463b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen AllUsesAreOutsideLoop = false; 1464b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen 146502e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman // If this use isn't an address, then not all uses are addresses. 146655e641b766a18878b51551d626d5a566102e487eEvan Cheng if (!isAddress && !isPHI) 146702e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman AllUsesAreAddresses = false; 146880b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner 14691d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng MoveImmediateValues(TLI, UsersToProcess[i].Inst, UsersToProcess[i].Base, 1470246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Imm, isAddress, L, SE); 147180b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner } 147244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 1473d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 147432e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng // If one of the use if a PHI node and all other uses are addresses, still 147532e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng // allow iv reuse. Essentially we are trading one constant multiplication 147632e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng // for one fewer iv. 147732e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng if (NumPHI > 1) 147832e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng AllUsesAreAddresses = false; 147932e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng 14805f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return CommonExprs; 14815f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 14825f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1483c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// ShouldUseFullStrengthReductionMode - Test whether full strength-reduction 1484c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// is valid and profitable for the given set of users of a stride. In 1485c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// full strength-reduction mode, all addresses at the current stride are 1486c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// strength-reduced all the way down to pointer arithmetic. 1487c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1488c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanbool LoopStrengthReduce::ShouldUseFullStrengthReductionMode( 1489c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const std::vector<BasedUser> &UsersToProcess, 1490c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1491c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool AllUsesAreAddresses, 1492c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride) { 1493c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!EnableFullLSRMode) 1494c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1495c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1496c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // The heuristics below aim to avoid increasing register pressure, but 1497c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // fully strength-reducing all the addresses increases the number of 1498c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // add instructions, so don't do this when optimizing for size. 1499c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: If the loop is large, the savings due to simpler addresses 1500c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // may oughtweight the costs of the extra increment instructions. 1501c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (L->getHeader()->getParent()->hasFnAttr(Attribute::OptimizeForSize)) 1502c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1503c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1504c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: For now, don't do full strength reduction if there could 1505c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // potentially be greater-stride multiples of the current stride 1506c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // which could reuse the current stride IV. 1507c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (StrideOrder.back() != Stride) 1508c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1509c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1510c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Iterate through the uses to find conditions that automatically rule out 1511c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // full-lsr mode. 1512c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ) { 1513c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEV *Base = UsersToProcess[i].Base; 1514c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEV *Imm = UsersToProcess[i].Imm; 1515c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If any users have a loop-variant component, they can't be fully 1516c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength-reduced. 1517c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (Imm && !Imm->isLoopInvariant(L)) 1518c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1519c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If there are to users with the same base and the difference between 1520c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // the two Imm values can't be folded into the address, full 1521c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength reduction would increase register pressure. 1522c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman do { 1523c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEV *CurImm = UsersToProcess[i].Imm; 1524c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (CurImm || Imm && CurImm != Imm) { 1525c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!CurImm) CurImm = SE->getIntegerSCEV(0, Stride->getType()); 1526c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!Imm) Imm = SE->getIntegerSCEV(0, Stride->getType()); 1527c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Instruction *Inst = UsersToProcess[i].Inst; 1528c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Type *UseTy = Inst->getType(); 1529c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) 1530c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UseTy = SI->getOperand(0)->getType(); 1531c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Diff = SE->getMinusSCEV(UsersToProcess[i].Imm, Imm); 1532c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!Diff->isZero() && 1533c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (!AllUsesAreAddresses || 1534c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman !fitsInAddressMode(Diff, UseTy, TLI, /*HasBaseReg=*/true))) 1535c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1536c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1537c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } while (++i != e && Base == UsersToProcess[i].Base); 1538c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1539c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1540c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If there's exactly one user in this stride, fully strength-reducing it 1541c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // won't increase register pressure. If it's starting from a non-zero base, 1542c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // it'll be simpler this way. 1543c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (UsersToProcess.size() == 1 && !UsersToProcess[0].Base->isZero()) 1544c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return true; 1545c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1546c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Otherwise, if there are any users in this stride that don't require 1547c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // a register for their base, full strength-reduction will increase 1548c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // register pressure. 1549c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) 1550c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!UsersToProcess[i].Base || 1551c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Base->isZero()) 1552c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1553c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1554c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Otherwise, go for it. 1555c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return true; 1556c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1557c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1558c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// InsertAffinePhi Create and insert a PHI node for an induction variable 1559c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// with the specified start and step values in the specified loop. 1560c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1561c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// If NegateStride is true, the stride should be negated by using a 1562c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// subtract instead of an add. 1563c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1564c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// Return the created phi node, and return the step instruction by 1565c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// reference in IncV. 1566c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1567c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic PHINode *InsertAffinePhi(SCEVHandle Start, SCEVHandle Step, 1568c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1569c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &Rewriter, 1570c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *&IncV) { 1571c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(Start->isLoopInvariant(L) && "New PHI start is not loop invariant!"); 1572c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(Step->isLoopInvariant(L) && "New PHI stride is not loop invariant!"); 1573c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1574c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman BasicBlock *Header = L->getHeader(); 1575c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman BasicBlock *Preheader = L->getLoopPreheader(); 1576c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1577c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PHINode *PN = PHINode::Create(Start->getType(), "lsr.iv", Header->begin()); 1578c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PN->addIncoming(Rewriter.expandCodeFor(Start, Preheader->getTerminator()), 1579c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Preheader); 1580c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1581c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman pred_iterator HPI = pred_begin(Header); 1582c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(HPI != pred_end(Header) && "Loop with zero preds???"); 1583c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!L->contains(*HPI)) ++HPI; 1584c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(HPI != pred_end(Header) && L->contains(*HPI) && 1585c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "No backedge in loop?"); 1586c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1587c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If the stride is negative, insert a sub instead of an add for the 1588c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // increment. 1589c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool isNegative = isNonConstantNegative(Step); 1590c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle IncAmount = Step; 1591c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isNegative) 1592c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncAmount = Rewriter.SE.getNegativeSCEV(Step); 1593c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1594c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Insert an add instruction right before the terminator corresponding 1595c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // to the back-edge. 1596c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *StepV = Rewriter.expandCodeFor(IncAmount, Preheader->getTerminator()); 1597c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isNegative) { 1598c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV = BinaryOperator::CreateSub(PN, StepV, "lsr.iv.next", 1599c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (*HPI)->getTerminator()); 1600c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } else { 1601c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV = BinaryOperator::CreateAdd(PN, StepV, "lsr.iv.next", 1602c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (*HPI)->getTerminator()); 1603c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1604c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!isa<ConstantInt>(StepV)) ++NumVariable; 1605c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1606c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman pred_iterator PI = pred_begin(Header); 1607c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (*PI == L->getLoopPreheader()) 1608c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++PI; 1609c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PN->addIncoming(IncV, *PI); 1610c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1611c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++NumInserted; 1612c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return PN; 1613c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1614c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1615c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic void SortUsersToProcess(std::vector<BasedUser> &UsersToProcess) { 1616c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We want to emit code for users inside the loop first. To do this, we 1617c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // rearrange BasedUser so that the entries at the end have 1618c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // isUseOfPostIncrementedValue = false, because we pop off the end of the 1619c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // vector (so we handle them first). 1620c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::partition(UsersToProcess.begin(), UsersToProcess.end(), 1621c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PartitionByIsUseOfPostIncrementedValue); 1622c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1623c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Sort this by base, so that things with the same base are handled 1624c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // together. By partitioning first and stable-sorting later, we are 1625c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // guaranteed that within each base we will pop off users from within the 1626c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // loop before users outside of the loop with a particular base. 1627c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // 1628c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We would like to use stable_sort here, but we can't. The problem is that 1629c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // SCEVHandle's don't have a deterministic ordering w.r.t to each other, so 1630c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // we don't have anything to do a '<' comparison on. Because we think the 1631c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // number of uses is small, do a horrible bubble sort which just relies on 1632c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // ==. 1633c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1634c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Get a base value. 1635c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Base = UsersToProcess[i].Base; 1636c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1637c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Compact everything with this base to be consecutive with this one. 1638c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned j = i+1; j != e; ++j) { 1639c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (UsersToProcess[j].Base == Base) { 1640c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::swap(UsersToProcess[i+1], UsersToProcess[j]); 1641c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++i; 1642c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1643c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1644c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1645c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1646c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1647c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// PrepareToStrengthReduceFully - Prepare to fully strength-reduce UsersToProcess, 1648c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// meaning lowering addresses all the way down to direct pointer arithmetic. 1649c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1650c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1651c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceFully( 1652c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1653c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 1654c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 1655c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1656c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter) { 1657c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " Fully reducing all users\n"; 1658c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1659c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Rewrite the UsersToProcess records, creating a separate PHI for each 1660c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // unique Base value. 1661c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ) { 1662c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: The uses are grouped by base, but not sorted. We arbitrarily 1663c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // pick the first Imm value here to start with, and adjust it for the 1664c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // other uses. 1665c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Imm = UsersToProcess[i].Imm; 1666c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Base = UsersToProcess[i].Base; 1667c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Start = SE->getAddExpr(CommonExprs, Base, Imm); 1668c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *IncV; 1669c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PHINode *Phi = InsertAffinePhi(Start, Stride, L, 1670c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PreheaderRewriter, 1671c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV); 1672c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Loop over all the users with the same base. 1673c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman do { 1674c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Base = SE->getIntegerSCEV(0, Stride->getType()); 1675c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Imm = SE->getMinusSCEV(UsersToProcess[i].Imm, Imm); 1676c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = Phi; 1677c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].IncV = IncV; 1678c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(UsersToProcess[i].Imm->isLoopInvariant(L) && 1679c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "ShouldUseFullStrengthReductionMode should reject this!"); 1680c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } while (++i != e && Base == UsersToProcess[i].Base); 1681c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1682c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1683c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1684c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// PrepareToStrengthReduceWithNewPhi - Insert a new induction variable for the 1685c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// given users to share. 1686c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1687c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1688c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceWithNewPhi( 1689c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1690c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 1691c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 1692c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 1693c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1694c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter) { 1695c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " Inserting new PHI:\n"; 1696c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1697c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *IncV; 1698c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PHINode *Phi = InsertAffinePhi(SE->getUnknown(CommonBaseV), 1699c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Stride, L, 1700c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PreheaderRewriter, 1701c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV); 1702c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1703c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Remember this in case a later stride is multiple of this. 1704c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IVsByStride[Stride].addIV(Stride, CommonExprs, Phi, IncV); 1705c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1706c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // All the users will share this new IV. 1707c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1708c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = Phi; 1709c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].IncV = IncV; 1710c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1711c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1712c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " IV="; 1713c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DEBUG(WriteAsOperand(*DOUT, Phi, /*PrintType=*/false)); 1714c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << ", INC="; 1715c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DEBUG(WriteAsOperand(*DOUT, IncV, /*PrintType=*/false)); 1716c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << "\n"; 1717c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1718c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1719c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// PrepareToStrengthReduceWithNewPhi - Prepare for the given users to reuse 1720c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// an induction variable with a stride that is a factor of the current 1721c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// induction variable. 1722c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1723c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1724c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceFromSmallerStride( 1725c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1726c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 1727c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const IVExpr &ReuseIV, 1728c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Instruction *PreInsertPt) { 1729c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " Rewriting in terms of existing IV of STRIDE " << *ReuseIV.Stride 1730c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman << " and BASE " << *ReuseIV.Base << "\n"; 1731c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1732c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // All the users will share the reused IV. 1733c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1734c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = ReuseIV.PHI; 1735c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].IncV = ReuseIV.IncV; 1736c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1737c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1738c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Constant *C = dyn_cast<Constant>(CommonBaseV); 1739c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (C && 1740c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (!C->isNullValue() && 1741c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman !fitsInAddressMode(SE->getUnknown(CommonBaseV), CommonBaseV->getType(), 1742c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman TLI, false))) 1743c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We want the common base emitted into the preheader! This is just 1744c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // using cast as a copy so BitCast (no-op cast) is appropriate 1745c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman CommonBaseV = new BitCastInst(CommonBaseV, CommonBaseV->getType(), 1746c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "commonbase", PreInsertPt); 1747c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1748c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 17495f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// StrengthReduceStridedIVUsers - Strength reduce all of the users of a single 17505f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// stride of IV. All of the users may have different starting values, and this 17515f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// may not be the only stride (we know it is if isOnlyStride is true). 17525f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Chengvoid LoopStrengthReduce::StrengthReduceStridedIVUsers(const SCEVHandle &Stride, 17535f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 17545f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 17555f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool isOnlyStride) { 17565f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // If all the users are moved to another stride, then there is nothing to do. 1757303595942502f17c087fa28874c2b89117148c45Dan Gohman if (Uses.Users.empty()) 17585f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return; 17595f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 17605f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // Keep track if every use in UsersToProcess is an address. If they all are, 17615f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // we may be able to rewrite the entire collection of them in terms of a 17625f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // smaller-stride IV. 17635f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool AllUsesAreAddresses = true; 17645f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1765b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // Keep track if every use of a single stride is outside the loop. If so, 1766b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // we want to be more aggressive about reusing a smaller-stride IV; a 1767b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // multiply outside the loop is better than another IV inside. Well, usually. 1768b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool AllUsesAreOutsideLoop = true; 1769b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen 17705f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // Transform our list of users and offsets to a bit more complex table. In 17715f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // this new vector, each 'BasedUser' contains 'Base' the base of the 17725f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // strided accessas well as the old information from Uses. We progressively 17735f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // move information from the Base field to the Imm field, until we eventually 17745f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // have the full access expression to rewrite the use. 17755f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> UsersToProcess; 17765f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng SCEVHandle CommonExprs = CollectIVUsers(Stride, Uses, L, AllUsesAreAddresses, 1777b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen AllUsesAreOutsideLoop, 17785f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng UsersToProcess); 17795f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1780c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Sort the UsersToProcess array so that users with common bases are 1781c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // next to each other. 1782c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SortUsersToProcess(UsersToProcess); 1783c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 17845f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // If we managed to find some expressions in common, we'll need to carry 17855f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // their value in a register and add it in for each use. This will take up 17865f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // a register operand, which potentially restricts what stride values are 17875f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // valid. 1788cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman bool HaveCommonExprs = !CommonExprs->isZero(); 1789c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1790fe35555d09ecdad4b42da1138a1d40ea4b83898eChris Lattner const Type *ReplacedTy = CommonExprs->getType(); 1791c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 17921bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Now that we know what we need to do, insert the PHI node itself. 17931bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // 17942f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << "LSR: Examining IVs of TYPE " << *ReplacedTy << " of STRIDE " 17952f09f519542202b8af227c2a524f8fe82378a934Dan Gohman << *Stride << ":\n" 17962f09f519542202b8af227c2a524f8fe82378a934Dan Gohman << " Common base: " << *CommonExprs << "\n"; 1797d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 17981bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner SCEVExpander Rewriter(*SE, *LI); 17991bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner SCEVExpander PreheaderRewriter(*SE, *LI); 1800c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 18011bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner BasicBlock *Preheader = L->getLoopPreheader(); 18021bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Instruction *PreInsertPt = Preheader->getTerminator(); 180312b50410cd0a8cd81a7f528f862005e80921cf58Chris Lattner BasicBlock *LatchBlock = L->getLoopLatch(); 1804d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1805c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV = ConstantInt::get(ReplacedTy, 0); 1806eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 1807c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle RewriteFactor = SE->getIntegerSCEV(0, ReplacedTy); 1808c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IVExpr ReuseIV(SE->getIntegerSCEV(0, Type::Int32Ty), 1809c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SE->getIntegerSCEV(0, Type::Int32Ty), 1810c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 0, 0); 18111bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1812c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// Choose a strength-reduction strategy and prepare for it by creating 1813c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// the necessary PHIs and adjusting the bookkeeping. 1814c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (ShouldUseFullStrengthReductionMode(UsersToProcess, L, 1815c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman AllUsesAreAddresses, Stride)) { 1816c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceFully(UsersToProcess, Stride, CommonExprs, L, 1817c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PreheaderRewriter); 1818c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } else { 1819c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Emit the initial base value into the loop preheader. 1820c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman CommonBaseV = PreheaderRewriter.expandCodeFor(CommonExprs, PreInsertPt); 1821c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1822c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If all uses are addresses, check if it is possible to reuse an IV with a 1823c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // stride that is a factor of this stride. And that the multiple is a number 1824c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // that can be encoded in the scale field of the target addressing mode. And 1825c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // that we will have a valid instruction after this substition, including the 1826c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // immediate field, if any. 1827c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman RewriteFactor = CheckForIVReuse(HaveCommonExprs, AllUsesAreAddresses, 1828c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman AllUsesAreOutsideLoop, 1829c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Stride, ReuseIV, CommonExprs->getType(), 1830c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess); 1831c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isa<SCEVConstant>(RewriteFactor) && 1832c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cast<SCEVConstant>(RewriteFactor)->isZero()) 1833c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceWithNewPhi(UsersToProcess, Stride, CommonExprs, 1834c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman CommonBaseV, L, PreheaderRewriter); 1835c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman else 1836c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceFromSmallerStride(UsersToProcess, CommonBaseV, 1837c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ReuseIV, PreInsertPt); 18387e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 18397e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1840c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Process all the users now, replacing their strided uses with 1841c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength-reduced forms. This outer loop handles all bases, the inner 18427e79b3898ddd919170d367a516f51296017146c2Chris Lattner // loop handles all users of a particular base. 1843169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman while (!UsersToProcess.empty()) { 18447b445c521bc191d0d25799b289e17b45f202a1afChris Lattner SCEVHandle Base = UsersToProcess.back().Base; 18452f09f519542202b8af227c2a524f8fe82378a934Dan Gohman Instruction *Inst = UsersToProcess.back().Inst; 1846be3e5212e23edc9210f774fac18d801de252e906Chris Lattner 18471bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Emit the code for Base into the preheader. 1848d19534add90a2a894af61523b830887097bb780bDan Gohman Value *BaseV = PreheaderRewriter.expandCodeFor(Base, PreInsertPt); 18497d8ed8a94118ace05962e88689a4089c2f49366dChris Lattner 18502f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Examining uses with BASE "; 18514a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, BaseV, /*PrintType=*/false)); 18522f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << ":\n"; 18537d8ed8a94118ace05962e88689a4089c2f49366dChris Lattner 18541bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If BaseV is a constant other than 0, make sure that it gets inserted into 18551bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // the preheader, instead of being forward substituted into the uses. We do 18563da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // this by forcing a BitCast (noop cast) to be inserted into the preheader 18573da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // in this case. 18587e79b3898ddd919170d367a516f51296017146c2Chris Lattner if (Constant *C = dyn_cast<Constant>(BaseV)) { 1859203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!C->isNullValue() && !fitsInAddressMode(Base, ReplacedTy, 1860203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen TLI, false)) { 18613da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // We want this constant emitted into the preheader! This is just 18623da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // using cast as a copy so BitCast (no-op cast) is appropriate 18633da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer BaseV = new BitCastInst(BaseV, BaseV->getType(), "preheaderinsert", 18644a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman PreInsertPt); 18651bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner } 18667e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 18677e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1868169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Emit the code to add the immediate offset to the Phi value, just before 18692351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner // the instructions that we identified as using this stride and base. 18707b445c521bc191d0d25799b289e17b45f202a1afChris Lattner do { 18717e79b3898ddd919170d367a516f51296017146c2Chris Lattner // FIXME: Use emitted users to emit other users. 18727b445c521bc191d0d25799b289e17b45f202a1afChris Lattner BasedUser &User = UsersToProcess.back(); 18732351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 18742f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Examining use "; 18754a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, UsersToProcess.back().OperandValToReplace, 18764a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman /*PrintType=*/false)); 18772f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " in Inst: " << *Inst; 18782f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 1879010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // If this instruction wants to use the post-incremented value, move it 1880010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // after the post-inc and use its value instead of the PHI. 1881c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *RewriteOp = User.Phi; 1882010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (User.isUseOfPostIncrementedValue) { 1883c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman RewriteOp = User.IncV; 1884c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner 1885c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // If this user is in the loop, make sure it is the last thing in the 1886c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // loop to ensure it is dominated by the increment. 1887c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner if (L->contains(User.Inst->getParent())) 1888c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner User.Inst->moveBefore(LatchBlock->getTerminator()); 1889010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 18903ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer if (RewriteOp->getType() != ReplacedTy) { 18913ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::CastOps opcode = Instruction::Trunc; 18923ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer if (ReplacedTy->getPrimitiveSizeInBits() == 18933ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer RewriteOp->getType()->getPrimitiveSizeInBits()) 18943ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer opcode = Instruction::BitCast; 18953ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer RewriteOp = SCEVExpander::InsertCastOfTo(opcode, RewriteOp, ReplacedTy); 18963ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer } 189786c75d31133319a88216c1b1cd26a789e4023000Evan Cheng 1898246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle RewriteExpr = SE->getUnknown(RewriteOp); 18991bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1900b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // If we had to insert new instructions for RewriteOp, we have to 1901f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // consider that they may not have been able to end up immediately 1902f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // next to RewriteOp, because non-PHI instructions may never precede 1903f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // PHI instructions in a block. In this case, remember where the last 1904ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // instruction was inserted so that if we're replacing a different 1905ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // PHI node, we can use the later point to expand the final 1906ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // RewriteExpr. 1907f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *NewBasePt = dyn_cast<Instruction>(RewriteOp); 1908c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (RewriteOp == User.Phi) NewBasePt = 0; 1909f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman 19101bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Clear the SCEVExpander's expression map so that we are guaranteed 19111bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // to have the code emitted where we expect it. 19121bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Rewriter.clear(); 1913d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1914d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng // If we are reusing the iv, then it must be multiplied by a constant 19151de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // factor to take advantage of the addressing mode scale component. 19162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!isa<SCEVConstant>(RewriteFactor) || 19172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !cast<SCEVConstant>(RewriteFactor)->isZero()) { 19182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If we're reusing an IV with a nonzero base (currently this happens 19192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // only when all reuses are outside the loop) subtract that base here. 19202f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // The base has been used to initialize the PHI node but we don't want 19212f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // it here. 19221de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (!ReuseIV.Base->isZero()) { 19231de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen SCEVHandle typedBase = ReuseIV.Base; 19241de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (RewriteExpr->getType()->getPrimitiveSizeInBits() != 19251de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen ReuseIV.Base->getType()->getPrimitiveSizeInBits()) { 19261de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // It's possible the original IV is a larger type than the new IV, 19271de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // in which case we have to truncate the Base. We checked in 19281de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // RequiresTypeConversion that this is valid. 19291de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen assert (RewriteExpr->getType()->getPrimitiveSizeInBits() < 19301de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen ReuseIV.Base->getType()->getPrimitiveSizeInBits() && 19311de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen "Unexpected lengthening conversion!"); 19321de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen typedBase = SE->getTruncateExpr(ReuseIV.Base, 19331de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen RewriteExpr->getType()); 19341de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen } 19351de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen RewriteExpr = SE->getMinusSCEV(RewriteExpr, typedBase); 19361de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen } 19372f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 19382f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Multiply old variable, with base removed, by new scale factor. 19392f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getMulExpr(RewriteFactor, 19408392772727ed9105c92fe4514d53dab74c333edcEvan Cheng RewriteExpr); 1941eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 1942eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // The common base is emitted in the loop preheader. But since we 1943eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // are reusing an IV, it has not been used to initialize the PHI node. 1944eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // Add it to the expression used to rewrite the uses. 19452f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // When this use is outside the loop, we earlier subtracted the 19462f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // common base, and are adding it back here. Use the same expression 19472f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // as before, rather than CommonBaseV, so DAGCombiner will zap it. 1948eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng if (!isa<ConstantInt>(CommonBaseV) || 19492f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !cast<ConstantInt>(CommonBaseV)->isZero()) { 19502f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(User.Inst->getParent())) 19512f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getAddExpr(RewriteExpr, 1952b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen SE->getUnknown(CommonBaseV)); 19532f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen else 19542f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getAddExpr(RewriteExpr, CommonExprs); 19552f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 1956eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 1957d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 19581bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Now that we know what we need to do, insert code before User for the 19591bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // immediate and any loop-variant expressions. 1960bee0f663d842f5aa41286c22815446e2d22de95aReid Spencer if (!isa<ConstantInt>(BaseV) || !cast<ConstantInt>(BaseV)->isZero()) 19611bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Add BaseV to the PHI value if needed. 1962246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman RewriteExpr = SE->getAddExpr(RewriteExpr, SE->getUnknown(BaseV)); 1963d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1964f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman User.RewriteInstructionToUseNewBase(RewriteExpr, NewBasePt, 1965f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Rewriter, L, this, 19660e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng DeadInsts); 19672351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 1968a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner // Mark old value we replaced as possibly dead, so that it is eliminated 19692351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner // if we just replaced the last use of that value. 197009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(cast<Instruction>(User.OperandValToReplace)); 19712351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 19727b445c521bc191d0d25799b289e17b45f202a1afChris Lattner UsersToProcess.pop_back(); 19732351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner ++NumReduced; 19747b445c521bc191d0d25799b289e17b45f202a1afChris Lattner 19757e79b3898ddd919170d367a516f51296017146c2Chris Lattner // If there are any more users to process with the same base, process them 19767e79b3898ddd919170d367a516f51296017146c2Chris Lattner // now. We sorted by base above, so we just have to check the last elt. 19777b445c521bc191d0d25799b289e17b45f202a1afChris Lattner } while (!UsersToProcess.empty() && UsersToProcess.back().Base == Base); 1978169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // TODO: Next, find out which base index is the most common, pull it out. 1979169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 1980169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 1981169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // IMPORTANT TODO: Figure out how to partition the IV's with this stride, but 1982169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // different starting values, into different PHIs. 1983eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 1984eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 1985c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel/// FindIVUserForCond - If Cond has an operand that is an expression of an IV, 1986aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner/// set the IV user and stride information and return true, otherwise return 1987aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner/// false. 1988c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patelbool LoopStrengthReduce::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse, 1989aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner const SCEVHandle *&CondStride) { 1990aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner for (unsigned Stride = 0, e = StrideOrder.size(); Stride != e && !CondUse; 1991aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner ++Stride) { 1992aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 1993aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner IVUsesByStride.find(StrideOrder[Stride]); 1994aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner assert(SI != IVUsesByStride.end() && "Stride doesn't exist!"); 1995aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 1996aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner for (std::vector<IVStrideUse>::iterator UI = SI->second.Users.begin(), 1997aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner E = SI->second.Users.end(); UI != E; ++UI) 1998aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner if (UI->User == Cond) { 1999aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // NOTE: we could handle setcc instructions with multiple uses here, but 2000aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // InstCombine does it as well for simple uses, it's not clear that it 2001aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // occurs enough in real life to handle. 2002aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner CondUse = &*UI; 2003aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner CondStride = &SI->first; 2004aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return true; 2005aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner } 2006aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner } 2007aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return false; 2008aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner} 2009aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 2010cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Chengnamespace { 2011cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Constant strides come first which in turns are sorted by their absolute 2012cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // values. If absolute values are the same, then positive strides comes first. 2013cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // e.g. 2014cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // 4, -1, X, 1, 2 ==> 1, -1, 2, 4, X 2015cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng struct StrideCompare { 2016cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng bool operator()(const SCEVHandle &LHS, const SCEVHandle &RHS) { 2017cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS); 2018cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS); 2019cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (LHSC && RHSC) { 2020cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t LV = LHSC->getValue()->getSExtValue(); 2021cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t RV = RHSC->getValue()->getSExtValue(); 2022cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng uint64_t ALV = (LV < 0) ? -LV : LV; 2023cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng uint64_t ARV = (RV < 0) ? -RV : RV; 2024bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman if (ALV == ARV) { 2025bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman if (LV != RV) 2026bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return LV > RV; 2027bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman } else { 2028cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return ALV < ARV; 2029bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman } 2030bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman 2031bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // If it's the same value but different type, sort by bit width so 2032bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // that we emit larger induction variables before smaller 2033bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // ones, letting the smaller be re-written in terms of larger ones. 2034bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return RHS->getBitWidth() < LHS->getBitWidth(); 2035cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2036bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return LHSC && !RHSC; 2037cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2038cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng }; 2039cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 2040cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2041cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ChangeCompareStride - If a loop termination compare instruction is the 2042cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// only use of its stride, and the compaison is against a constant value, 2043cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// try eliminate the stride by moving the compare instruction to another 2044cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// stride and change its constant operand accordingly. e.g. 2045cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// 2046cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// loop: 2047cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ... 2048cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v1 = v1 + 3 2049cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v2 = v2 + 1 2050cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// if (v2 < 10) goto loop 2051cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// => 2052cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// loop: 2053cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ... 2054cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v1 = v1 + 3 2055cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// if (v1 < 30) goto loop 2056cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengICmpInst *LoopStrengthReduce::ChangeCompareStride(Loop *L, ICmpInst *Cond, 20570e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng IVStrideUse* &CondUse, 2058cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng const SCEVHandle* &CondStride) { 2059cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (StrideOrder.size() < 2 || 2060cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVUsesByStride[*CondStride].Users.size() != 1) 2061cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return Cond; 2062cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng const SCEVConstant *SC = dyn_cast<SCEVConstant>(*CondStride); 2063cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (!SC) return Cond; 2064cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ConstantInt *C = dyn_cast<ConstantInt>(Cond->getOperand(1)); 2065cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (!C) return Cond; 2066cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2067cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst::Predicate Predicate = Cond->getPredicate(); 2068cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t CmpSSInt = SC->getValue()->getSExtValue(); 2069cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t CmpVal = C->getValue().getSExtValue(); 2070168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng unsigned BitWidth = C->getValue().getBitWidth(); 2071168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng uint64_t SignBit = 1ULL << (BitWidth-1); 2072168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng const Type *CmpTy = C->getType(); 2073168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng const Type *NewCmpTy = NULL; 2074af62c09437675369afcc481e88ed9cd5806491feEvan Cheng unsigned TyBits = CmpTy->getPrimitiveSizeInBits(); 2075af62c09437675369afcc481e88ed9cd5806491feEvan Cheng unsigned NewTyBits = 0; 2076cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t NewCmpVal = CmpVal; 2077cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng SCEVHandle *NewStride = NULL; 2078cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng Value *NewIncV = NULL; 2079cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t Scale = 1; 2080cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2081d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel // Check stride constant and the comparision constant signs to detect 2082d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel // overflow. 20834b3f08bac7def05de67f5c0e9f15c53d0870c7c1Devang Patel if ((CmpVal & SignBit) != (CmpSSInt & SignBit)) 2084d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel return Cond; 2085d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel 2086cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Look for a suitable stride / iv as replacement. 2087cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng std::stable_sort(StrideOrder.begin(), StrideOrder.end(), StrideCompare()); 2088cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng for (unsigned i = 0, e = StrideOrder.size(); i != e; ++i) { 2089cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 2090cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVUsesByStride.find(StrideOrder[i]); 2091cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (!isa<SCEVConstant>(SI->first)) 2092cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng continue; 2093cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 2094168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng if (abs(SSInt) <= abs(CmpSSInt) || (SSInt % CmpSSInt) != 0) 2095cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng continue; 2096cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2097168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng Scale = SSInt / CmpSSInt; 2098168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng NewCmpVal = CmpVal * Scale; 2099168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng APInt Mul = APInt(BitWidth, NewCmpVal); 2100168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng // Check for overflow. 2101168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng if (Mul.getSExtValue() != NewCmpVal) { 2102168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng NewCmpVal = CmpVal; 2103168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng continue; 2104168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng } 2105168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2106cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Watch out for overflow. 2107168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng if (ICmpInst::isSignedPredicate(Predicate) && 2108168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng (CmpVal & SignBit) != (NewCmpVal & SignBit)) 2109cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng NewCmpVal = CmpVal; 2110168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2111cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (NewCmpVal != CmpVal) { 2112cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Pick the best iv to use trying to avoid a cast. 2113cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng NewIncV = NULL; 2114cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng for (std::vector<IVStrideUse>::iterator UI = SI->second.Users.begin(), 2115cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng E = SI->second.Users.end(); UI != E; ++UI) { 2116cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng NewIncV = UI->OperandValToReplace; 2117cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (NewIncV->getType() == CmpTy) 2118cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng break; 2119cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2120cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (!NewIncV) { 2121cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng NewCmpVal = CmpVal; 2122cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng continue; 2123cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2124cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2125cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng NewCmpTy = NewIncV->getType(); 2126af62c09437675369afcc481e88ed9cd5806491feEvan Cheng NewTyBits = isa<PointerType>(NewCmpTy) 2127af62c09437675369afcc481e88ed9cd5806491feEvan Cheng ? UIntPtrTy->getPrimitiveSizeInBits() 2128af62c09437675369afcc481e88ed9cd5806491feEvan Cheng : NewCmpTy->getPrimitiveSizeInBits(); 2129af62c09437675369afcc481e88ed9cd5806491feEvan Cheng if (RequiresTypeConversion(NewCmpTy, CmpTy)) { 21306725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif // Check if it is possible to rewrite it using 21316725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif // an iv / stride of a smaller integer type. 2132af62c09437675369afcc481e88ed9cd5806491feEvan Cheng bool TruncOk = false; 2133af62c09437675369afcc481e88ed9cd5806491feEvan Cheng if (NewCmpTy->isInteger()) { 2134af62c09437675369afcc481e88ed9cd5806491feEvan Cheng unsigned Bits = NewTyBits; 2135af62c09437675369afcc481e88ed9cd5806491feEvan Cheng if (ICmpInst::isSignedPredicate(Predicate)) 2136af62c09437675369afcc481e88ed9cd5806491feEvan Cheng --Bits; 2137af62c09437675369afcc481e88ed9cd5806491feEvan Cheng uint64_t Mask = (1ULL << Bits) - 1; 2138af62c09437675369afcc481e88ed9cd5806491feEvan Cheng if (((uint64_t)NewCmpVal & Mask) == (uint64_t)NewCmpVal) 2139af62c09437675369afcc481e88ed9cd5806491feEvan Cheng TruncOk = true; 2140af62c09437675369afcc481e88ed9cd5806491feEvan Cheng } 2141af62c09437675369afcc481e88ed9cd5806491feEvan Cheng if (!TruncOk) { 2142af62c09437675369afcc481e88ed9cd5806491feEvan Cheng NewCmpVal = CmpVal; 2143af62c09437675369afcc481e88ed9cd5806491feEvan Cheng continue; 2144af62c09437675369afcc481e88ed9cd5806491feEvan Cheng } 2145af62c09437675369afcc481e88ed9cd5806491feEvan Cheng } 2146af62c09437675369afcc481e88ed9cd5806491feEvan Cheng 2147af62c09437675369afcc481e88ed9cd5806491feEvan Cheng // Don't rewrite if use offset is non-constant and the new type is 2148af62c09437675369afcc481e88ed9cd5806491feEvan Cheng // of a different type. 2149af62c09437675369afcc481e88ed9cd5806491feEvan Cheng // FIXME: too conservative? 2150af62c09437675369afcc481e88ed9cd5806491feEvan Cheng if (NewTyBits != TyBits && !isa<SCEVConstant>(CondUse->Offset)) { 21515f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng NewCmpVal = CmpVal; 21525f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng continue; 21535f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng } 21545f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 21555f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool AllUsesAreAddresses = true; 2156b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool AllUsesAreOutsideLoop = true; 21575f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> UsersToProcess; 21585f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng SCEVHandle CommonExprs = CollectIVUsers(SI->first, SI->second, L, 21595f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng AllUsesAreAddresses, 2160b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen AllUsesAreOutsideLoop, 21615f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng UsersToProcess); 21625f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // Avoid rewriting the compare instruction with an iv of new stride 21635f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // if it's likely the new stride uses will be rewritten using the 2164e6986965691543d5415b98571801141b6a974984Dan Gohman // stride of the compare instruction. 21655f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng if (AllUsesAreAddresses && 2166cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman ValidStride(!CommonExprs->isZero(), Scale, UsersToProcess)) { 2167cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng NewCmpVal = CmpVal; 2168cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng continue; 2169cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2170cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2171f5e25f32c7605569390f5e81ebb582c96d74a1d2Evan Cheng // If scale is negative, use swapped predicate unless it's testing 2172cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // for equality. 2173cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (Scale < 0 && !Cond->isEquality()) 2174f5e25f32c7605569390f5e81ebb582c96d74a1d2Evan Cheng Predicate = ICmpInst::getSwappedPredicate(Predicate); 2175cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2176cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng NewStride = &StrideOrder[i]; 2177cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng break; 2178cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2179cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2180cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 21819b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // Forgo this transformation if it the increment happens to be 21829b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // unfortunately positioned after the condition, and the condition 21839b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // has multiple uses which prevent it from being moved immediately 21849b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // before the branch. See 21859b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // test/Transforms/LoopStrengthReduce/change-compare-stride-trickiness-*.ll 21869b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // for an example of this situation. 2187d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel if (!Cond->hasOneUse()) { 21889b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman for (BasicBlock::iterator I = Cond, E = Cond->getParent()->end(); 21899b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman I != E; ++I) 21909b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman if (I == NewIncV) 21919b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman return Cond; 2192d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel } 21939b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman 2194cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (NewCmpVal != CmpVal) { 2195cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Create a new compare instruction using new stride / iv. 2196cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst *OldCond = Cond; 2197af62c09437675369afcc481e88ed9cd5806491feEvan Cheng Value *RHS; 2198af62c09437675369afcc481e88ed9cd5806491feEvan Cheng if (!isa<PointerType>(NewCmpTy)) 2199af62c09437675369afcc481e88ed9cd5806491feEvan Cheng RHS = ConstantInt::get(NewCmpTy, NewCmpVal); 2200af62c09437675369afcc481e88ed9cd5806491feEvan Cheng else { 2201af62c09437675369afcc481e88ed9cd5806491feEvan Cheng RHS = ConstantInt::get(UIntPtrTy, NewCmpVal); 2202af62c09437675369afcc481e88ed9cd5806491feEvan Cheng RHS = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr, RHS, NewCmpTy); 2203cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2204168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng // Insert new compare instruction. 2205e562b1725ee068ff525082d1e9ba885c8928c72eDan Gohman Cond = new ICmpInst(Predicate, NewIncV, RHS, 2206e562b1725ee068ff525082d1e9ba885c8928c72eDan Gohman L->getHeader()->getName() + ".termcond", 2207e562b1725ee068ff525082d1e9ba885c8928c72eDan Gohman OldCond); 2208168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2209168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng // Remove the old compare instruction. The old indvar is probably dead too. 221009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(cast<Instruction>(CondUse->OperandValToReplace)); 2211168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SE->deleteValueFromRecords(OldCond); 2212010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman OldCond->replaceAllUsesWith(Cond); 2213cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng OldCond->eraseFromParent(); 2214168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2215cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVUsesByStride[*CondStride].Users.pop_back(); 2216af62c09437675369afcc481e88ed9cd5806491feEvan Cheng SCEVHandle NewOffset = TyBits == NewTyBits 2217af62c09437675369afcc481e88ed9cd5806491feEvan Cheng ? SE->getMulExpr(CondUse->Offset, 2218af62c09437675369afcc481e88ed9cd5806491feEvan Cheng SE->getConstant(ConstantInt::get(CmpTy, Scale))) 2219af62c09437675369afcc481e88ed9cd5806491feEvan Cheng : SE->getConstant(ConstantInt::get(NewCmpTy, 2220af62c09437675369afcc481e88ed9cd5806491feEvan Cheng cast<SCEVConstant>(CondUse->Offset)->getValue()->getSExtValue()*Scale)); 2221cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVUsesByStride[*NewStride].addUser(NewOffset, Cond, NewIncV); 2222cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng CondUse = &IVUsesByStride[*NewStride].Users.back(); 2223cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng CondStride = NewStride; 2224cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ++NumEliminated; 2225cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2226cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2227cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return Cond; 2228cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 2229cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2230ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// OptimizeSMax - Rewrite the loop's terminating condition if it uses 2231ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// an smax computation. 2232ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2233ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// This is a narrow solution to a specific, but acute, problem. For loops 2234ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// like this: 2235ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2236ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2237ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2238ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2239ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i < n); 2240ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2241ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// where the comparison is signed, the trip count isn't just 'n', because 2242ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 'n' could be negative. And unfortunately this can come up even for loops 2243ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// where the user didn't use a C do-while loop. For example, seemingly 2244ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// well-behaved top-test loops will commonly be lowered like this: 2245ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman// 2246ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// if (n > 0) { 2247ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2248ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2249ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2250ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i < n); 2251ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } 2252ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2253ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// and then it's possible for subsequent optimization to obscure the if 2254ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// test in such a way that indvars can't find it. 2255ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2256ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// When indvars can't find the if test in loops like this, it creates a 2257ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// signed-max expression, which allows it to give the loop a canonical 2258ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// induction variable: 2259ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2260ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2261ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// smax = n < 1 ? 1 : n; 2262ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2263ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2264ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i != smax); 2265ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2266ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// Canonical induction variables are necessary because the loop passes 2267ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// are designed around them. The most obvious example of this is the 2268ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// LoopInfo analysis, which doesn't remember trip count values. It 2269ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// expects to be able to rediscover the trip count each time it is 2270ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// needed, and it does this using a simple analyis that only succeeds if 2271ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// the loop has a canonical induction variable. 2272ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2273ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// However, when it comes time to generate code, the maximum operation 2274ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// can be quite costly, especially if it's inside of an outer loop. 2275ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2276ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// This function solves this problem by detecting this type of loop and 2277ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting 2278ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// the instructions for the maximum computation. 2279ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2280ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan GohmanICmpInst *LoopStrengthReduce::OptimizeSMax(Loop *L, ICmpInst *Cond, 2281ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman IVStrideUse* &CondUse) { 2282ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check that the loop matches the pattern we're looking for. 2283ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (Cond->getPredicate() != CmpInst::ICMP_EQ && 2284ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->getPredicate() != CmpInst::ICMP_NE) 2285ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 2286ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2287ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); 2288ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!Sel || !Sel->hasOneUse()) return Cond; 2289ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2290ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle IterationCount = SE->getIterationCount(L); 2291ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (isa<SCEVCouldNotCompute>(IterationCount)) 2292ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 2293ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle One = SE->getIntegerSCEV(1, IterationCount->getType()); 2294ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2295ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Adjust for an annoying getIterationCount quirk. 2296ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman IterationCount = SE->getAddExpr(IterationCount, One); 2297ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2298ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check for a max calculation that matches the pattern. 2299ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(IterationCount); 2300ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!SMax || SMax != SE->getSCEV(Sel)) return Cond; 2301ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2302ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle SMaxLHS = SMax->getOperand(0); 2303ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle SMaxRHS = SMax->getOperand(1); 2304ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!SMaxLHS || SMaxLHS != One) return Cond; 2305ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2306ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check the relevant induction variable for conformance to 2307ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // the pattern. 2308ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle IV = SE->getSCEV(Cond->getOperand(0)); 2309ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 2310ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!AR || !AR->isAffine() || 2311ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman AR->getStart() != One || 2312ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman AR->getStepRecurrence(*SE) != One) 2313ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 2314ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2315ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check the right operand of the select, and remember it, as it will 2316ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // be used in the new comparison instruction. 2317ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Value *NewRHS = 0; 2318ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (SE->getSCEV(Sel->getOperand(1)) == SMaxRHS) 2319ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman NewRHS = Sel->getOperand(1); 2320ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman else if (SE->getSCEV(Sel->getOperand(2)) == SMaxRHS) 2321ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman NewRHS = Sel->getOperand(2); 2322ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!NewRHS) return Cond; 2323ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2324ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Ok, everything looks ok to change the condition into an SLT or SGE and 2325ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // delete the max calculation. 2326ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman ICmpInst *NewCond = 2327ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman new ICmpInst(Cond->getPredicate() == CmpInst::ICMP_NE ? 2328ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman CmpInst::ICMP_SLT : 2329ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman CmpInst::ICMP_SGE, 2330ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->getOperand(0), NewRHS, "scmp", Cond); 2331ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2332ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Delete the max calculation instructions. 2333586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman SE->deleteValueFromRecords(Cond); 2334ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->replaceAllUsesWith(NewCond); 2335ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->eraseFromParent(); 2336ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); 2337ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SE->deleteValueFromRecords(Sel); 2338586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman Sel->eraseFromParent(); 2339ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (Cmp->use_empty()) { 2340ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SE->deleteValueFromRecords(Cmp); 2341586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman Cmp->eraseFromParent(); 2342ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman } 2343ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman CondUse->User = NewCond; 2344ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return NewCond; 2345ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman} 2346ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2347a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel/// OptimizeShadowIV - If IV is used in a int-to-float cast 2348a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel/// inside the loop then try to eliminate the cast opeation. 2349a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patelvoid LoopStrengthReduce::OptimizeShadowIV(Loop *L) { 2350a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2351a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel SCEVHandle IterationCount = SE->getIterationCount(L); 2352a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (isa<SCEVCouldNotCompute>(IterationCount)) 2353a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel return; 2354a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2355a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned Stride = 0, e = StrideOrder.size(); Stride != e; 2356a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ++Stride) { 2357a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 2358a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel IVUsesByStride.find(StrideOrder[Stride]); 2359a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel assert(SI != IVUsesByStride.end() && "Stride doesn't exist!"); 2360a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!isa<SCEVConstant>(SI->first)) 2361a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2362a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2363a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (std::vector<IVStrideUse>::iterator UI = SI->second.Users.begin(), 2364a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel E = SI->second.Users.end(); UI != E; /* empty */) { 2365a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel std::vector<IVStrideUse>::iterator CandidateUI = UI; 2366541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel ++UI; 2367a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Instruction *ShadowUse = CandidateUI->User; 2368a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const Type *DestTy = NULL; 2369a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2370a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* If shadow use is a int->float cast then insert a second IV 2371541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel to eliminate this cast. 2372a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2373a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned i = 0; i < n; ++i) 2374a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel foo((double)i); 2375a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2376541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel is transformed into 2377a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2378a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel double d = 0.0; 2379a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned i = 0; i < n; ++i, ++d) 2380a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel foo(d); 2381a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel */ 2382541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->User)) 2383a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel DestTy = UCast->getDestTy(); 2384541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->User)) 2385a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel DestTy = SCast->getDestTy(); 238618bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (!DestTy) continue; 238718bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel 238818bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (TLI) { 238918bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel /* If target does not support DestTy natively then do not apply 239018bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel this transformation. */ 239118bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel MVT DVT = TLI->getValueType(DestTy); 239218bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (!TLI->isTypeLegal(DVT)) continue; 239318bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel } 239418bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel 2395a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); 2396a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!PH) continue; 2397a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (PH->getNumIncomingValues() != 2) continue; 2398a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2399a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const Type *SrcTy = PH->getType(); 2400a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel int Mantissa = DestTy->getFPMantissaWidth(); 2401a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Mantissa == -1) continue; 2402a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if ((int)TD->getTypeSizeInBits(SrcTy) > Mantissa) 2403a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2404a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2405a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel unsigned Entry, Latch; 2406a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { 2407a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Entry = 0; 2408a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Latch = 1; 2409a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } else { 2410a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Entry = 1; 2411a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Latch = 0; 2412a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2413a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2414a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); 2415a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!Init) continue; 2416a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantFP *NewInit = ConstantFP::get(DestTy, Init->getZExtValue()); 2417a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2418a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator *Incr = 2419a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); 2420a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!Incr) continue; 2421a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Incr->getOpcode() != Instruction::Add 2422a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel && Incr->getOpcode() != Instruction::Sub) 2423a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2424a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2425a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Initialize new IV, double d = 0.0 in above example. */ 2426a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantInt *C = NULL; 2427a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Incr->getOperand(0) == PH) 2428a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel C = dyn_cast<ConstantInt>(Incr->getOperand(1)); 2429a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel else if (Incr->getOperand(1) == PH) 2430a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel C = dyn_cast<ConstantInt>(Incr->getOperand(0)); 2431a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel else 2432a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2433a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2434a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!C) continue; 2435a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2436a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Add new PHINode. */ 2437a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel PHINode *NewPH = PHINode::Create(DestTy, "IV.S.", PH); 2438a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2439541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel /* create new increment. '++d' in above example. */ 2440a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantFP *CFP = ConstantFP::get(DestTy, C->getZExtValue()); 2441a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator *NewIncr = 2442a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator::Create(Incr->getOpcode(), 2443a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH, CFP, "IV.S.next.", Incr); 2444a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2445a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); 2446a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); 2447a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2448a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Remove cast operation */ 2449a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel SE->deleteValueFromRecords(ShadowUse); 2450a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ShadowUse->replaceAllUsesWith(NewPH); 2451a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ShadowUse->eraseFromParent(); 2452a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel SI->second.Users.erase(CandidateUI); 2453a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NumShadow++; 2454a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel break; 2455a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2456a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2457a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel} 2458a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2459010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner// OptimizeIndvars - Now that IVUsesByStride is set up with all of the indvar 2460010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner// uses in the loop, look to see if we can eliminate some, in favor of using 2461010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner// common indvars for the different uses. 2462010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattnervoid LoopStrengthReduce::OptimizeIndvars(Loop *L) { 2463010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // TODO: implement optzns here. 2464010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2465a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel OptimizeShadowIV(L); 2466a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2467010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Finally, get the terminating condition for the loop if possible. If we 2468010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // can, we want to change it to use a post-incremented version of its 246998d9811db263ee040d9a6a69ef9e1c4fdc8c219dChris Lattner // induction variable, to allow coalescing the live ranges for the IV into 2470010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // one register value. 2471010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner PHINode *SomePHI = cast<PHINode>(L->getHeader()->begin()); 2472010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner BasicBlock *Preheader = L->getLoopPreheader(); 2473010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner BasicBlock *LatchBlock = 2474010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner SomePHI->getIncomingBlock(SomePHI->getIncomingBlock(0) == Preheader); 2475010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner BranchInst *TermBr = dyn_cast<BranchInst>(LatchBlock->getTerminator()); 2476e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer if (!TermBr || TermBr->isUnconditional() || 2477e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer !isa<ICmpInst>(TermBr->getCondition())) 2478010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner return; 2479e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); 2480010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2481010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Search IVUsesByStride to find Cond's IVUse if there is one. 2482010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner IVStrideUse *CondUse = 0; 248350fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner const SCEVHandle *CondStride = 0; 2484010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2485c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel if (!FindIVUserForCond(Cond, CondUse, CondStride)) 2486aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return; // setcc doesn't use the IV. 2487cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2488ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // If the trip count is computed in terms of an smax (due to ScalarEvolution 2489ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // being unable to find a sufficient guard, for example), change the loop 2490ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // comparison to use SLT instead of NE. 2491ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond = OptimizeSMax(L, Cond, CondUse); 2492ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2493cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // If possible, change stride and operands of the compare instruction to 2494cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // eliminate one stride. 2495cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng Cond = ChangeCompareStride(L, Cond, CondUse, CondStride); 2496010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2497010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // It's possible for the setcc instruction to be anywhere in the loop, and 2498010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // possible for it to have multiple users. If it is not immediately before 2499010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // the latch block branch, move it. 2500010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (&*++BasicBlock::iterator(Cond) != (Instruction*)TermBr) { 2501010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (Cond->hasOneUse()) { // Condition has a single use, just move it. 2502010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner Cond->moveBefore(TermBr); 2503010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } else { 2504010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Otherwise, clone the terminating condition and insert into the loopend. 2505e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer Cond = cast<ICmpInst>(Cond->clone()); 2506010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner Cond->setName(L->getHeader()->getName() + ".termcond"); 2507010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner LatchBlock->getInstList().insert(TermBr, Cond); 2508010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2509010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Clone the IVUse, as the old use still exists! 251050fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner IVUsesByStride[*CondStride].addUser(CondUse->Offset, Cond, 2511010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner CondUse->OperandValToReplace); 251250fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner CondUse = &IVUsesByStride[*CondStride].Users.back(); 2513010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2514010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2515010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2516010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // If we get to here, we know that we can transform the setcc instruction to 251798d9811db263ee040d9a6a69ef9e1c4fdc8c219dChris Lattner // use the post-incremented version of the IV, allowing us to coalesce the 2518010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // live ranges for the IV correctly. 2519246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman CondUse->Offset = SE->getMinusSCEV(CondUse->Offset, *CondStride); 2520010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner CondUse->isUseOfPostIncrementedValue = true; 25211ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng Changed = true; 2522010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner} 2523169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 25240f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patelbool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager &LPM) { 2525eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 25260f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel LI = &getAnalysis<LoopInfo>(); 2527b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel DT = &getAnalysis<DominatorTree>(); 25280f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel SE = &getAnalysis<ScalarEvolution>(); 25290f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel TD = &getAnalysis<TargetData>(); 25300f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel UIntPtrTy = TD->getIntPtrType(); 25313fea643fb427e72907b7a40e940d7949d1d57e76Dan Gohman Changed = false; 25320f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel 2533b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // Find all uses of induction variables in this loop, and categorize 2534169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // them by stride. Start by finding all of the PHI nodes in the header for 2535169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // this loop. If they are induction variables, inspect their uses. 2536168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SmallPtrSet<Instruction*,16> Processed; // Don't reprocess instructions. 2537169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) 25383416e5f645186a7e3321f927eab662d0ecef404bChris Lattner AddUsersIfInteresting(I, L, Processed); 2539169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 25401ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng if (!IVUsesByStride.empty()) { 25411ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Optimize induction variables. Some indvar uses can be transformed to use 25421ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // strides that will be needed for other purposes. A common example of this 25431ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // is the exit test for the loop, which can often be rewritten to use the 25441ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // computation of some other indvar to decide when to terminate the loop. 25451ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng OptimizeIndvars(L); 2546010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 25471ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // FIXME: We can widen subreg IV's here for RISC targets. e.g. instead of 25481ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // doing computation in byte values, promote to 32-bit values if safe. 2549010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 25501ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // FIXME: Attempt to reuse values across multiple IV's. In particular, we 25511ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // could have something like "for(i) { foo(i*8); bar(i*16) }", which should 25521ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // be codegened as "for (j = 0;; j+=8) { foo(j); bar(j+j); }" on X86/PPC. 25531ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Need to be careful that IV's are all the same type. Only works for 25541ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // intptr_t indvars. 2555169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 25561ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // If we only have one stride, we can more aggressively eliminate some 25571ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // things. 25581ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng bool HasOneStride = IVUsesByStride.size() == 1; 2559d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 2560d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng#ifndef NDEBUG 25611ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng DOUT << "\nLSR on "; 25621ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng DEBUG(L->dump()); 2563d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng#endif 2564d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 25651ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // IVsByStride keeps IVs for one particular loop. 25661ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng assert(IVsByStride.empty() && "Stale entries in IVsByStride?"); 25671ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 25681ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Sort the StrideOrder so we process larger strides first. 25691ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng std::stable_sort(StrideOrder.begin(), StrideOrder.end(), StrideCompare()); 25701ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 25711ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Note: this processes each stride/type pair individually. All users 25721ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // passed into StrengthReduceStridedIVUsers have the same type AND stride. 25731ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Also, note that we iterate over IVUsesByStride indirectly by using 25741ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // StrideOrder. This extra layer of indirection makes the ordering of 25751ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // strides deterministic - not dependent on map order. 25761ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng for (unsigned Stride = 0, e = StrideOrder.size(); Stride != e; ++Stride) { 25771ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 25781ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng IVUsesByStride.find(StrideOrder[Stride]); 25791ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng assert(SI != IVUsesByStride.end() && "Stride doesn't exist!"); 25801ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng StrengthReduceStridedIVUsers(SI->first, SI->second, L, HasOneStride); 25811ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng } 25827305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner } 2583eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2584010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman // We're done analyzing this loop; release all the state we built up for it. 2585010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman CastedPointers.clear(); 2586010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman IVUsesByStride.clear(); 2587010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman IVsByStride.clear(); 2588010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman StrideOrder.clear(); 25892f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned i=0; i<GEPlist.size(); i++) 25902f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SE->deleteValueFromRecords(GEPlist[i]); 25912f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen GEPlist.clear(); 2592010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman 2593eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman // Clean up after ourselves 2594eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman if (!DeadInsts.empty()) { 2595a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeleteTriviallyDeadInstructions(); 2596eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2597169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman BasicBlock::iterator I = L->getHeader()->begin(); 2598cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman while (PHINode *PN = dyn_cast<PHINode>(I++)) { 2599cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // At this point, we know that we have killed one or more IV users. 2600bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner // It is worth checking to see if the cannonical indvar is also 2601cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // dead, so that we can remove it as well. 2602cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // 2603cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // We can remove a PHI if it is on a cycle in the def-use graph 2604cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // where each node in the cycle has degree one, i.e. only one use, 2605cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // and is an instruction with no side effects. 2606cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // 2607169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // FIXME: this needs to eliminate an induction variable even if it's being 2608169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // compared against some value to decide loop termination. 2609a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner if (!PN->hasOneUse()) 2610a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner continue; 2611a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner 2612a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner SmallPtrSet<PHINode *, 4> PHIs; 2613a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner for (Instruction *J = dyn_cast<Instruction>(*PN->use_begin()); 2614a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner J && J->hasOneUse() && !J->mayWriteToMemory(); 2615a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner J = dyn_cast<Instruction>(*J->use_begin())) { 2616a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // If we find the original PHI, we've discovered a cycle. 2617a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner if (J == PN) { 2618a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // Break the cycle and mark the PHI for deletion. 2619a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner SE->deleteValueFromRecords(PN); 2620a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner PN->replaceAllUsesWith(UndefValue::get(PN->getType())); 262109fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(PN); 2622a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner Changed = true; 2623a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner break; 26247e608bbb5dfe4f827e64e91b0bb68a1d95d737aeChris Lattner } 2625a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // If we find a PHI more than once, we're on a cycle that 2626a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // won't prove fruitful. 2627a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner if (isa<PHINode>(J) && !PHIs.insert(cast<PHINode>(J))) 2628a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner break; 2629169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 2630eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 2631a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeleteTriviallyDeadInstructions(); 2632eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 26331ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng return Changed; 2634eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 2635