LoopStrengthReduce.cpp revision bc10b8c6c7e85e44d8231dfb2fb41a60300857e3
1eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===- LoopStrengthReduce.cpp - Strength Reduce GEPs in Loops -------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// The LLVM Compiler Infrastructure 4eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 9eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 10eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// This pass performs a strength reduction on array references inside loops that 11eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// have as one or more of their components the loop induction variable. This is 12eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// accomplished by creating a new Value to hold the initial value of the array 13eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// access for the first iteration, and then creating a new GEP instruction in 14eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// the loop to increment the value by the appropriate amount. 15eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 16eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 17eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 18be3e5212e23edc9210f774fac18d801de252e906Chris Lattner#define DEBUG_TYPE "loop-reduce" 19eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Scalar.h" 20eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Constants.h" 21eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Instructions.h" 22e5b01bea7b9b7dce7c24484d2d915b0c118d4d07Dan Gohman#include "llvm/IntrinsicInst.h" 23eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Type.h" 242f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/DerivedTypes.h" 25eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Analysis/Dominators.h" 26eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Analysis/LoopInfo.h" 270f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel#include "llvm/Analysis/LoopPass.h" 28169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Analysis/ScalarEvolutionExpander.h" 29d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng#include "llvm/Transforms/Utils/AddrModeMatcher.h" 30e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner#include "llvm/Transforms/Utils/BasicBlockUtils.h" 31eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Utils/Local.h" 322f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/Target/TargetData.h" 33168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng#include "llvm/ADT/SmallPtrSet.h" 34eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/ADT/Statistic.h" 35d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng#include "llvm/Support/CFG.h" 36169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/Debug.h" 37a4f0b3a084d120cfc5b5bb06f64b222f5cb72740Chris Lattner#include "llvm/Support/Compiler.h" 38c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman#include "llvm/Support/CommandLine.h" 39d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng#include "llvm/Support/GetElementPtrTypeIterator.h" 40d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng#include "llvm/Target/TargetLowering.h" 41cfb1d4235fe3291028341e6acf4139723b4749e3Jeff Cohen#include <algorithm> 42eaa13851a7fe604363577350c5cf65c257c4d41aNate Begemanusing namespace llvm; 43eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 44cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumReduced , "Number of GEPs strength reduced"); 45cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumInserted, "Number of PHIs inserted"); 46cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumVariable, "Number of PHIs with variable strides"); 47541532724e29203e28c2fe0136cf6eabd49d4532Devang PatelSTATISTIC(NumEliminated, "Number of strides eliminated"); 48541532724e29203e28c2fe0136cf6eabd49d4532Devang PatelSTATISTIC(NumShadow, "Number of Shadow IVs optimized"); 49d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan ChengSTATISTIC(NumImmSunk, "Number of common expr immediates sunk into uses"); 50eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 51c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic cl::opt<bool> EnableFullLSRMode("enable-full-lsr", 52c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cl::init(false), 53c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cl::Hidden); 54c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 550e5f499638c8d277b9dc4a4385712177c53b5681Chris Lattnernamespace { 56dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 57c01a53007a4f4f9a601f1cc83ff4e2935405b905Jeff Cohen struct BasedUser; 58dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 59ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// IVStrideUse - Keep track of one use of a strided induction variable, where 60ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// the stride is stored externally. The Offset member keeps track of the 619330c3af45453d0a3dd9caef40910a9a053e4de5Dan Gohman /// offset from the IV, User is the actual user of the operand, and 629330c3af45453d0a3dd9caef40910a9a053e4de5Dan Gohman /// 'OperandValToReplace' is the operand of the User that is the use. 639133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVStrideUse { 64ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner SCEVHandle Offset; 65ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Instruction *User; 66ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Value *OperandValToReplace; 67010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 68010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // isUseOfPostIncrementedValue - True if this should use the 69010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // post-incremented version of this IV, not the preincremented version. 70010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // This can only be set in special cases, such as the terminating setcc 71c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // instruction for a loop or uses dominated by the loop. 72010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner bool isUseOfPostIncrementedValue; 73ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner 74ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner IVStrideUse(const SCEVHandle &Offs, Instruction *U, Value *O) 75010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner : Offset(Offs), User(U), OperandValToReplace(O), 76010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner isUseOfPostIncrementedValue(false) {} 77ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner }; 78ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner 79ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// IVUsersOfOneStride - This structure keeps track of all instructions that 80ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// have an operand that is based on the trip count multiplied by some stride. 81ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// The stride for all of these users is common and kept external to this 82ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// structure. 839133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVUsersOfOneStride { 84169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Users - Keep track of all of the users of this stride as well as the 85ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// initial value and the operand that uses the IV. 86ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner std::vector<IVStrideUse> Users; 87ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner 88ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner void addUser(const SCEVHandle &Offset,Instruction *User, Value *Operand) { 89ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Users.push_back(IVStrideUse(Offset, User, Operand)); 90169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 91169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman }; 92169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 93d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVInfo - This structure keeps track of one IV expression inserted during 9421495775e710d37003e100862cdc647cbdc3b999Evan Cheng /// StrengthReduceStridedIVUsers. It contains the stride, the common base, as 9521495775e710d37003e100862cdc647cbdc3b999Evan Cheng /// well as the PHI node and increment value created for rewrite. 969133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVExpr { 9721495775e710d37003e100862cdc647cbdc3b999Evan Cheng SCEVHandle Stride; 98d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng SCEVHandle Base; 99d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng PHINode *PHI; 100d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng Value *IncV; 101d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 10221495775e710d37003e100862cdc647cbdc3b999Evan Cheng IVExpr(const SCEVHandle &stride, const SCEVHandle &base, PHINode *phi, 10321495775e710d37003e100862cdc647cbdc3b999Evan Cheng Value *incv) 10421495775e710d37003e100862cdc647cbdc3b999Evan Cheng : Stride(stride), Base(base), PHI(phi), IncV(incv) {} 105d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng }; 106d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 107d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVsOfOneStride - This structure keeps track of all IV expression inserted 108d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// during StrengthReduceStridedIVUsers for a particular stride of the IV. 1099133fe28954d498fc4de13064c7d65bd811de02cReid Spencer struct VISIBILITY_HIDDEN IVsOfOneStride { 110d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng std::vector<IVExpr> IVs; 111d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 11221495775e710d37003e100862cdc647cbdc3b999Evan Cheng void addIV(const SCEVHandle &Stride, const SCEVHandle &Base, PHINode *PHI, 11321495775e710d37003e100862cdc647cbdc3b999Evan Cheng Value *IncV) { 11421495775e710d37003e100862cdc647cbdc3b999Evan Cheng IVs.push_back(IVExpr(Stride, Base, PHI, IncV)); 115d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng } 116d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng }; 117169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 1180f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel class VISIBILITY_HIDDEN LoopStrengthReduce : public LoopPass { 119eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman LoopInfo *LI; 120b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel DominatorTree *DT; 121169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman ScalarEvolution *SE; 122169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman const TargetData *TD; 123169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman const Type *UIntPtrTy; 124eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman bool Changed; 1257e608bbb5dfe4f827e64e91b0bb68a1d95d737aeChris Lattner 126169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// IVUsesByStride - Keep track of all uses of induction variables that we 127169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// are interested in. The key of the map is the stride of the access. 12850fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner std::map<SCEVHandle, IVUsersOfOneStride> IVUsesByStride; 129169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 130d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVsByStride - Keep track of all IVs that have been inserted for a 131d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// particular stride. 132d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng std::map<SCEVHandle, IVsOfOneStride> IVsByStride; 133d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1347305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner /// StrideOrder - An ordering of the keys in IVUsesByStride that is stable: 1357305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner /// We use this to iterate over the IVUsesByStride collection without being 1367305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner /// dependent on random ordering of pointers in the process. 1378392772727ed9105c92fe4514d53dab74c333edcEvan Cheng SmallVector<SCEVHandle, 16> StrideOrder; 1387305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner 1392f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// GEPlist - A list of the GEP's that have been remembered in the SCEV 1402f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// data structures. SCEV does not know to update these when the operands 1412f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// of the GEP are changed, which means we cannot leave them live across 1422f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen /// loops. 1432f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SmallVector<GetElementPtrInst *, 16> GEPlist; 1442f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 14549f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// CastedValues - As we need to cast values to uintptr_t, this keeps track 14649f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// of the casted version of each value. This is accessed by 14749f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// getCastedVersionOf. 1488392772727ed9105c92fe4514d53dab74c333edcEvan Cheng DenseMap<Value*, Value*> CastedPointers; 149169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 150169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// DeadInsts - Keep track of instructions we may have made dead, so that 151169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// we can remove them after we are done working. 15209fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVector<Instruction*, 16> DeadInsts; 153d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng 154d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng /// TLI - Keep a pointer of a TargetLowering to consult for determining 155d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng /// transformation profitability. 156d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng const TargetLowering *TLI; 157d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng 158eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman public: 1591997473cf72957d0e70322e2fe6fe2ab141c58a6Devang Patel static char ID; // Pass ID, replacement for typeid 160c2bbfc18e9adbbdcf5b3375d8d25e2452f7df7f1Dan Gohman explicit LoopStrengthReduce(const TargetLowering *tli = NULL) : 161ae73dc1448d25b02cabc7c64c86c64371453dda8Dan Gohman LoopPass(&ID), TLI(tli) { 1622f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen } 1632f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen 1640f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel bool runOnLoop(Loop *L, LPPassManager &LPM); 165eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 166eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman virtual void getAnalysisUsage(AnalysisUsage &AU) const { 167aa96ae780afa5475e62df284855a971216289212Chris Lattner // We split critical edges, so we change the CFG. However, we do update 168aa96ae780afa5475e62df284855a971216289212Chris Lattner // many analyses if they are around. 169aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreservedID(LoopSimplifyID); 170aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<LoopInfo>(); 171aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<DominanceFrontier>(); 172aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<DominatorTree>(); 173aa96ae780afa5475e62df284855a971216289212Chris Lattner 174f465db6c6a5a877aa791abfc3837d62c491dacd5Jeff Cohen AU.addRequiredID(LoopSimplifyID); 175eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman AU.addRequired<LoopInfo>(); 176b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel AU.addRequired<DominatorTree>(); 1772f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen AU.addRequired<TargetData>(); 178169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman AU.addRequired<ScalarEvolution>(); 179a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel AU.addPreserved<ScalarEvolution>(); 180eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 18149f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 18249f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// getCastedVersionOf - Return the specified value casted to uintptr_t. 18349f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner /// 1843ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Value *getCastedVersionOf(Instruction::CastOps opcode, Value *V); 18549f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattnerprivate: 1863416e5f645186a7e3321f927eab662d0ecef404bChris Lattner bool AddUsersIfInteresting(Instruction *I, Loop *L, 187168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SmallPtrSet<Instruction*,16> &Processed); 1888480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman SCEVHandle GetExpressionSCEV(Instruction *E); 189cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst *ChangeCompareStride(Loop *L, ICmpInst *Cond, 190cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVStrideUse* &CondUse, 191cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng const SCEVHandle* &CondStride); 192010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner void OptimizeIndvars(Loop *L); 193a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 194a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /// OptimizeShadowIV - If IV is used in a int-to-float cast 195a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /// inside the loop then try to eliminate the cast opeation. 196a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel void OptimizeShadowIV(Loop *L); 197a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 198ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman /// OptimizeSMax - Rewrite the loop's terminating condition 199ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman /// if it uses an smax computation. 200ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman ICmpInst *OptimizeSMax(Loop *L, ICmpInst *Cond, 201ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman IVStrideUse* &CondUse); 202ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 203c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse, 204a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const SCEVHandle *&CondStride); 2055f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool RequiresTypeConversion(const Type *Ty, const Type *NewTy); 2062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SCEVHandle CheckForIVReuse(bool, bool, bool, const SCEVHandle&, 20702e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman IVExpr&, const Type*, 208dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess); 20902e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman bool ValidStride(bool, int64_t, 21002e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman const std::vector<BasedUser>& UsersToProcess); 2115f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng SCEVHandle CollectIVUsers(const SCEVHandle &Stride, 2125f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 2135f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 2145f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool &AllUsesAreAddresses, 215b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool &AllUsesAreOutsideLoop, 2165f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> &UsersToProcess); 217c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool ShouldUseFullStrengthReductionMode( 218c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const std::vector<BasedUser> &UsersToProcess, 219c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 220c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool AllUsesAreAddresses, 221c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride); 222c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceFully( 223c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 224c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 225c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 226c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 227c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter); 228c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceFromSmallerStride( 229c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 230c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 231c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const IVExpr &ReuseIV, 232c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Instruction *PreInsertPt); 233c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceWithNewPhi( 234c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 235c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 236c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 237c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 238c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 239c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter); 24050fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner void StrengthReduceStridedIVUsers(const SCEVHandle &Stride, 24150fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner IVUsersOfOneStride &Uses, 242ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Loop *L, bool isOnlyStride); 243a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner void DeleteTriviallyDeadInstructions(); 244eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman }; 245eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 246eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 247844731a7f1909f55935e3514c9e713a62d67662eDan Gohmanchar LoopStrengthReduce::ID = 0; 248844731a7f1909f55935e3514c9e713a62d67662eDan Gohmanstatic RegisterPass<LoopStrengthReduce> 249844731a7f1909f55935e3514c9e713a62d67662eDan GohmanX("loop-reduce", "Loop Strength Reduction"); 250844731a7f1909f55935e3514c9e713a62d67662eDan Gohman 251394f0441e06dafca29f0752cf400990a5b8fe4b1Daniel DunbarPass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) { 252d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng return new LoopStrengthReduce(TLI); 253eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 254eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2554da49122f3f3c8da68a52723d846b88c72166a68Reid Spencer/// getCastedVersionOf - Return the specified value casted to uintptr_t. This 2564da49122f3f3c8da68a52723d846b88c72166a68Reid Spencer/// assumes that the Value* V is of integer or pointer type only. 25749f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner/// 2583ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid SpencerValue *LoopStrengthReduce::getCastedVersionOf(Instruction::CastOps opcode, 2593ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Value *V) { 26049f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner if (V->getType() == UIntPtrTy) return V; 26149f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner if (Constant *CB = dyn_cast<Constant>(V)) 2623ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer return ConstantExpr::getCast(opcode, CB, UIntPtrTy); 26349f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 26449f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner Value *&New = CastedPointers[V]; 26549f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner if (New) return New; 26649f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 2673ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer New = SCEVExpander::InsertCastOfTo(opcode, V, UIntPtrTy); 26809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(cast<Instruction>(New)); 2697db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return New; 27049f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner} 27149f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 27249f72e68cf6eb77b5791310513d94dc64dc6ea7dChris Lattner 273eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// DeleteTriviallyDeadInstructions - If any of the instructions is the 274eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// specified set are trivially dead, delete them and see if this makes any of 275eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// their operands subsequently dead. 276a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattnervoid LoopStrengthReduce::DeleteTriviallyDeadInstructions() { 27709fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (DeadInsts.empty()) return; 27809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 27909fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // Sort the deadinsts list so that we can trivially eliminate duplicates as we 28009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // go. The code below never adds a non-dead instruction to the worklist, but 28109fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // callers may not be so careful. 28299d0015735f8e2aee1a4b99e39ffdaadc8a1dba8Chris Lattner array_pod_sort(DeadInsts.begin(), DeadInsts.end()); 28309fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 28409fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner // Drop duplicate instructions and those with uses. 28509fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner for (unsigned i = 0, e = DeadInsts.size()-1; i < e; ++i) { 28609fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner Instruction *I = DeadInsts[i]; 28709fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (!I->use_empty()) DeadInsts[i] = 0; 28846a879ebd384fb39cb680e4c3c182fefcdef5778Chris Lattner while (i != e && DeadInsts[i+1] == I) 28909fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts[++i] = 0; 29009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner } 29109fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 292a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner while (!DeadInsts.empty()) { 293a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner Instruction *I = DeadInsts.back(); 294a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeadInsts.pop_back(); 29509fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 29609fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (I == 0 || !isInstructionTriviallyDead(I)) 297bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner continue; 298bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner 299bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner SE->deleteValueFromRecords(I); 300411052bb96fb4a30035de6cafa492fa4f598e824Bill Wendling 30109fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) { 30209fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (Instruction *U = dyn_cast<Instruction>(*OI)) { 30309fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner *OI = 0; 304bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner if (U->use_empty()) 30509fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(U); 306bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner } 307eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 308bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner 309bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner I->eraseFromParent(); 310bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner Changed = true; 311eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 312eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 313eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 314169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 3153416e5f645186a7e3321f927eab662d0ecef404bChris Lattner/// GetExpressionSCEV - Compute and return the SCEV for the specified 3163416e5f645186a7e3321f927eab662d0ecef404bChris Lattner/// instruction. 3178480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan GohmanSCEVHandle LoopStrengthReduce::GetExpressionSCEV(Instruction *Exp) { 318da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen // Pointer to pointer bitcast instructions return the same value as their 319da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen // operand. 320da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen if (BitCastInst *BCI = dyn_cast<BitCastInst>(Exp)) { 321da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen if (SE->hasSCEV(BCI) || !isa<Instruction>(BCI->getOperand(0))) 322da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen return SE->getSCEV(BCI); 3238480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman SCEVHandle R = GetExpressionSCEV(cast<Instruction>(BCI->getOperand(0))); 324da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen SE->setSCEV(BCI, R); 325da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen return R; 326da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen } 327da91f49751b9b55a41d315b9fbdbb14614587b03Dale Johannesen 32887265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // Scalar Evolutions doesn't know how to compute SCEV's for GEP instructions. 32987265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // If this is a GEP that SE doesn't know about, compute it now and insert it. 33087265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // If this is not a GEP, or if we have already done this computation, just let 33187265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner // SE figure it out. 3323416e5f645186a7e3321f927eab662d0ecef404bChris Lattner GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Exp); 33387265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner if (!GEP || SE->hasSCEV(GEP)) 3343416e5f645186a7e3321f927eab662d0ecef404bChris Lattner return SE->getSCEV(Exp); 3353416e5f645186a7e3321f927eab662d0ecef404bChris Lattner 336169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Analyze all of the subscripts of this getelementptr instruction, looking 3378480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman // for uses that are determined by the trip count of the loop. First, skip 3388480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman // all operands the are not dependent on the IV. 339169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 340169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Build up the base expression. Insert an LLVM cast of the pointer to 341169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // uintptr_t first. 342246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle GEPVal = SE->getUnknown( 3433ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer getCastedVersionOf(Instruction::PtrToInt, GEP->getOperand(0))); 344169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 345169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman gep_type_iterator GTI = gep_type_begin(GEP); 3463416e5f645186a7e3321f927eab662d0ecef404bChris Lattner 3476725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end(); 3486725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif i != e; ++i, ++GTI) { 349169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // If this is a use of a recurrence that we can analyze, and it comes before 350169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Op does in the GEP operand list, we will handle this when we process this 351169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // operand. 352169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman if (const StructType *STy = dyn_cast<StructType>(*GTI)) { 353169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman const StructLayout *SL = TD->getStructLayout(STy); 3546725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif unsigned Idx = cast<ConstantInt>(*i)->getZExtValue(); 355b1919e2f08ecb37140af676fd2916f8d5ed7df3dChris Lattner uint64_t Offset = SL->getElementOffset(Idx); 356246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman GEPVal = SE->getAddExpr(GEPVal, 357246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SE->getIntegerSCEV(Offset, UIntPtrTy)); 3582f62fdc9a71d790d381d7f17d16e7098e30217e3Chris Lattner } else { 3593ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer unsigned GEPOpiBits = 3606725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif (*i)->getType()->getPrimitiveSizeInBits(); 3613ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer unsigned IntPtrBits = UIntPtrTy->getPrimitiveSizeInBits(); 3623ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::CastOps opcode = (GEPOpiBits < IntPtrBits ? 3633ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::SExt : (GEPOpiBits > IntPtrBits ? Instruction::Trunc : 3643ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::BitCast)); 3656725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif Value *OpVal = getCastedVersionOf(opcode, *i); 3667db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner SCEVHandle Idx = SE->getSCEV(OpVal); 3677db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 368ceb4d1aecb9deffe59b3dcdc9a783ffde8477be9Duncan Sands uint64_t TypeSize = TD->getTypePaddedSize(GTI.getIndexedType()); 3693416e5f645186a7e3321f927eab662d0ecef404bChris Lattner if (TypeSize != 1) 370246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Idx = SE->getMulExpr(Idx, 371246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SE->getConstant(ConstantInt::get(UIntPtrTy, 372246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman TypeSize))); 373246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman GEPVal = SE->getAddExpr(GEPVal, Idx); 3742f62fdc9a71d790d381d7f17d16e7098e30217e3Chris Lattner } 375eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 376169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 37787265abffc4942bbcfb9c75de31b6c7dffb82be1Chris Lattner SE->setSCEV(GEP, GEPVal); 3782f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen GEPlist.push_back(GEP); 3793416e5f645186a7e3321f927eab662d0ecef404bChris Lattner return GEPVal; 380169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 381169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 3822f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// containsAddRecFromDifferentLoop - Determine whether expression S involves a 3832f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// subexpression that is an AddRec from a loop other than L. An outer loop 3842f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// of L is OK, but not an inner loop nor a disjoint loop. 3852f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesenstatic bool containsAddRecFromDifferentLoop(SCEVHandle S, Loop *L) { 3862f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // This is very common, put it first. 3872f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (isa<SCEVConstant>(S)) 3882f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 3892f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVCommutativeExpr *AE = dyn_cast<SCEVCommutativeExpr>(S)) { 3902f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned int i=0; i< AE->getNumOperands(); i++) 3912f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (containsAddRecFromDifferentLoop(AE->getOperand(i), L)) 3922f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return true; 3932f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 3942f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 3952f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVAddRecExpr *AE = dyn_cast<SCEVAddRecExpr>(S)) { 3962f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (const Loop *newLoop = AE->getLoop()) { 3972f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (newLoop == L) 3982f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 3992f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // if newLoop is an outer loop of L, this is OK. 4002f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!LoopInfoBase<BasicBlock>::isNotAlreadyContainedIn(L, newLoop)) 4012f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 4022f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 4032f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return true; 4042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 4052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVUDivExpr *DE = dyn_cast<SCEVUDivExpr>(S)) 4062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(DE->getLHS(), L) || 4072f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen containsAddRecFromDifferentLoop(DE->getRHS(), L); 4082f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen#if 0 4092f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // SCEVSDivExpr has been backed out temporarily, but will be back; we'll 4102f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // need this when it is. 4112f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVSDivExpr *DE = dyn_cast<SCEVSDivExpr>(S)) 4122f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(DE->getLHS(), L) || 4132f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen containsAddRecFromDifferentLoop(DE->getRHS(), L); 4142f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen#endif 4152f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVTruncateExpr *TE = dyn_cast<SCEVTruncateExpr>(S)) 4162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(TE->getOperand(), L); 4172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVZeroExtendExpr *ZE = dyn_cast<SCEVZeroExtendExpr>(S)) 4182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(ZE->getOperand(), L); 4192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVSignExtendExpr *SE = dyn_cast<SCEVSignExtendExpr>(S)) 4202f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(SE->getOperand(), L); 4212f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 4222f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen} 4232f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 4247db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner/// getSCEVStartAndStride - Compute the start and stride of this expression, 4257db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner/// returning false if the expression is not a start/stride pair, or true if it 4267db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner/// is. The stride must be a loop invariant expression, but the start may be 4272f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// a mix of loop invariant and loop variant expressions. The start cannot, 4282f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// however, contain an AddRec from a different loop, unless that loop is an 4292f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// outer loop of the current loop. 4307db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattnerstatic bool getSCEVStartAndStride(const SCEVHandle &SH, Loop *L, 431246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle &Start, SCEVHandle &Stride, 4328f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng ScalarEvolution *SE, DominatorTree *DT) { 4337db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner SCEVHandle TheAddRec = Start; // Initialize to zero. 4347db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4357db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // If the outer level is an AddExpr, the operands are all start values except 4367db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // for a nested AddRecExpr. 4377db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(SH)) { 4387db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner for (unsigned i = 0, e = AE->getNumOperands(); i != e; ++i) 4397db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (SCEVAddRecExpr *AddRec = 4407db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner dyn_cast<SCEVAddRecExpr>(AE->getOperand(i))) { 4417db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (AddRec->getLoop() == L) 442246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman TheAddRec = SE->getAddExpr(AddRec, TheAddRec); 4437db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner else 4447db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return false; // Nested IV of some sort? 4457db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } else { 446246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Start = SE->getAddExpr(Start, AE->getOperand(i)); 4477db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } 4487db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4493ed469ccd7b028a030b550d84b7336d146f5d8faReid Spencer } else if (isa<SCEVAddRecExpr>(SH)) { 4507db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner TheAddRec = SH; 4517db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } else { 4527db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return false; // not analyzable. 4537db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } 4547db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4557db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(TheAddRec); 4567db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (!AddRec || AddRec->getLoop() != L) return false; 4577db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4587db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // FIXME: Generalize to non-affine IV's. 4597db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (!AddRec->isAffine()) return false; 4607db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4612f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If Start contains an SCEVAddRecExpr from a different loop, other than an 4621de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // outer loop of the current loop, reject it. SCEV has no concept of 4631de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // operating on one loop at a time so don't confuse it with such expressions. 4649194e8b0c827c97f396b209c7f4eeb75759e7068Dan Gohman if (containsAddRecFromDifferentLoop(AddRec->getOperand(0), L)) 4652f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 4662f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 467246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Start = SE->getAddExpr(Start, AddRec->getOperand(0)); 4687db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4698f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng if (!isa<SCEVConstant>(AddRec->getOperand(1))) { 4705a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng // If stride is an instruction, make sure it dominates the loop preheader. 4718f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng // Otherwise we could end up with a use before def situation. 4725a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng BasicBlock *Preheader = L->getLoopPreheader(); 4735a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng if (!AddRec->getOperand(1)->dominates(Preheader, DT)) 4745a6c1a840ad343c0ed2fa54a0edb50b61f828f0fEvan Cheng return false; 4758f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng 476b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling DOUT << "[" << L->getHeader()->getName() 477b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling << "] Variable stride: " << *AddRec << "\n"; 4788f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng } 47950fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner 48050fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner Stride = AddRec->getOperand(1); 4817db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return true; 4827db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner} 4837db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 4840ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// IVUseShouldUsePostIncValue - We have discovered a "User" of an IV expression 4850ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// and now we need to decide whether the user should use the preinc or post-inc 4860ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// value. If this user should use the post-inc version of the IV, return true. 4870ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// 4880ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// Choosing wrong here can break dominance properties (if we choose to use the 4890ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// post-inc value when we cannot) or it can end up adding extra live-ranges to 4900ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// the loop, resulting in reg-reg copies (if we use the pre-inc value when we 4910ae33eb243417982fbdca792460bdd986108ac09Chris Lattner/// should use the post-inc value). 4920ae33eb243417982fbdca792460bdd986108ac09Chris Lattnerstatic bool IVUseShouldUsePostIncValue(Instruction *User, Instruction *IV, 4930e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng Loop *L, DominatorTree *DT, Pass *P, 49409fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVectorImpl<Instruction*> &DeadInsts){ 4950ae33eb243417982fbdca792460bdd986108ac09Chris Lattner // If the user is in the loop, use the preinc value. 4960ae33eb243417982fbdca792460bdd986108ac09Chris Lattner if (L->contains(User->getParent())) return false; 4970ae33eb243417982fbdca792460bdd986108ac09Chris Lattner 4985e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner BasicBlock *LatchBlock = L->getLoopLatch(); 4995e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5005e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // Ok, the user is outside of the loop. If it is dominated by the latch 5015e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // block, use the post-inc value. 502b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel if (DT->dominates(LatchBlock, User->getParent())) 5035e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner return true; 5045e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5055e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // There is one case we have to be careful of: PHI nodes. These little guys 5065e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // can live in blocks that do not dominate the latch block, but (since their 5075e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // uses occur in the predecessor block, not the block the PHI lives in) should 5085e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // still use the post-inc value. Check for this case now. 5095e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner PHINode *PN = dyn_cast<PHINode>(User); 5105e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (!PN) return false; // not a phi, not dominated by latch block. 5115e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5125e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // Look at all of the uses of IV by the PHI node. If any use corresponds to 5135e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // a block that is not dominated by the latch block, give up and use the 5145e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // preincremented value. 5155e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner unsigned NumUses = 0; 5165e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 5175e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (PN->getIncomingValue(i) == IV) { 5185e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner ++NumUses; 519b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel if (!DT->dominates(LatchBlock, PN->getIncomingBlock(i))) 5205e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner return false; 5215e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner } 5225e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5235e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // Okay, all uses of IV by PN are in predecessor blocks that really are 5245e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // dominated by the latch block. Split the critical edges and use the 5255e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner // post-incremented value. 5265e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 5275e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (PN->getIncomingValue(i) == IV) { 5288392772727ed9105c92fe4514d53dab74c333edcEvan Cheng SplitCriticalEdge(PN->getIncomingBlock(i), PN->getParent(), P, false); 5291b9c8e73b5d620cf2d9a8e150b179fe95ddb8c4eChris Lattner // Splitting the critical edge can reduce the number of entries in this 5301b9c8e73b5d620cf2d9a8e150b179fe95ddb8c4eChris Lattner // PHI. 5311b9c8e73b5d620cf2d9a8e150b179fe95ddb8c4eChris Lattner e = PN->getNumIncomingValues(); 5325e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner if (--NumUses == 0) break; 5335e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner } 5340e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 5350e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng // PHI node might have become a constant value after SplitCriticalEdge. 53609fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(User); 5375e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner 5385e8ca66914fc9da9848cf6b1a29dd5075f4c721fChris Lattner return true; 5390ae33eb243417982fbdca792460bdd986108ac09Chris Lattner} 5400ae33eb243417982fbdca792460bdd986108ac09Chris Lattner 541f284ce203b656f1d6f933b67ad096a93d3cd7290Dan Gohman/// isAddressUse - Returns true if the specified instruction is using the 542203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// specified value as an address. 543203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstatic bool isAddressUse(Instruction *Inst, Value *OperandVal) { 544203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen bool isAddress = isa<LoadInst>(Inst); 545203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 546203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (SI->getOperand(1) == OperandVal) 547203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen isAddress = true; 548203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 549203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Addressing modes can also be folded into prefetches and a variety 550203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // of intrinsics. 551203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen switch (II->getIntrinsicID()) { 552203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen default: break; 553203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::prefetch: 554203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_loadu_dq: 555203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_loadu_pd: 556203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse_loadu_ps: 557203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse_storeu_ps: 558203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storeu_pd: 559203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storeu_dq: 560203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storel_dq: 561203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (II->getOperand(1) == OperandVal) 562203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen isAddress = true; 563203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen break; 564203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 565203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 566203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen return isAddress; 567203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen} 5680ae33eb243417982fbdca792460bdd986108ac09Chris Lattner 569169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// AddUsersIfInteresting - Inspect the specified instruction. If it is a 570169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// reducible SCEV, recursively add its users to the IVUsesByStride set and 571169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// return true. Otherwise, return false. 5723416e5f645186a7e3321f927eab662d0ecef404bChris Lattnerbool LoopStrengthReduce::AddUsersIfInteresting(Instruction *I, Loop *L, 573168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SmallPtrSet<Instruction*,16> &Processed) { 57442a75517250017a52afb03a0ade03cbd49559fe5Chris Lattner if (!I->getType()->isInteger() && !isa<PointerType>(I->getType())) 5754a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman return false; // Void and FP expressions cannot be reduced. 576168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng if (!Processed.insert(I)) 5773416e5f645186a7e3321f927eab662d0ecef404bChris Lattner return true; // Instruction already handled. 5783416e5f645186a7e3321f927eab662d0ecef404bChris Lattner 5797db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // Get the symbolic expression for this instruction. 5808480bc5b5b0d75c2cfa08d71c29dc98c20e5882cDan Gohman SCEVHandle ISE = GetExpressionSCEV(I); 5817db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (isa<SCEVCouldNotCompute>(ISE)) return false; 5827db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner 5837db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // Get the start and stride for this expression. 584246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Start = SE->getIntegerSCEV(0, ISE->getType()); 58550fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner SCEVHandle Stride = Start; 5868f40afe62db871230ac149de1c4ccc16ff48253aEvan Cheng if (!getSCEVStartAndStride(ISE, L, Start, Stride, SE, DT)) 5877db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner return false; // Non-reducible symbolic expression, bail out. 5884fe26582c09ec19873753cb4e9bb2ac3cdace88aDevang Patel 5892a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel std::vector<Instruction *> IUsers; 5902a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel // Collect all I uses now because IVUseShouldUsePostIncValue may 5912a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel // invalidate use_iterator. 5922a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); UI != E; ++UI) 5932a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel IUsers.push_back(cast<Instruction>(*UI)); 594169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 5952a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel for (unsigned iused_index = 0, iused_size = IUsers.size(); 5962a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel iused_index != iused_size; ++iused_index) { 5972a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel 5982a5fa189972c1ecc29f3682a732d15f94b9498f1Devang Patel Instruction *User = IUsers[iused_index]; 5994fe26582c09ec19873753cb4e9bb2ac3cdace88aDevang Patel 600169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Do not infinitely recurse on PHI nodes. 601396b2baf3ca0ee46e696713d34943035206d7a46Chris Lattner if (isa<PHINode>(User) && Processed.count(User)) 602169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman continue; 603169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 6042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Descend recursively, but not into PHI nodes outside the current loop. 6052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // It's important to see the entire expression outside the loop to get 6062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // choices that depend on addressing mode use right, although we won't 6072f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // consider references ouside the loop in all cases. 6082f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If User is already in Processed, we don't want to recurse into it again, 6092f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // but do want to record a second reference in the same instruction. 6107db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner bool AddUserToIVUsers = false; 611f9186596f06f2c1c5357420d44e2fe2f38f98068Chris Lattner if (LI->getLoopFor(User->getParent()) != L) { 6122f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (isa<PHINode>(User) || Processed.count(User) || 6132f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !AddUsersIfInteresting(User, L, Processed)) { 6142f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen DOUT << "FOUND USER in other loop: " << *User 6152f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen << " OF SCEV: " << *ISE << "\n"; 6162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen AddUserToIVUsers = true; 6172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 6182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } else if (Processed.count(User) || 6192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !AddUsersIfInteresting(User, L, Processed)) { 620b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling DOUT << "FOUND USER: " << *User 621b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling << " OF SCEV: " << *ISE << "\n"; 6227db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner AddUserToIVUsers = true; 6237db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner } 624fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 6257db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner if (AddUserToIVUsers) { 6267305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner IVUsersOfOneStride &StrideUses = IVUsesByStride[Stride]; 627b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen if (StrideUses.Users.empty()) // First occurrence of this stride? 6287305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideOrder.push_back(Stride); 6297305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner 630a4479ad25f7f184fc4600beb1d39fd1e71849c4dChris Lattner // Okay, we found a user that we cannot reduce. Analyze the instruction 631c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // and decide what to do with it. If we are a use inside of the loop, use 632c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // the value before incrementation, otherwise use it after incrementation. 6330e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng if (IVUseShouldUsePostIncValue(User, I, L, DT, this, DeadInsts)) { 634c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // The value used will be incremented by the stride more than we are 635c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // expecting, so subtract this off. 636246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle NewStart = SE->getMinusSCEV(Start, Stride); 6377305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideUses.addUser(NewStart, User, I); 6387305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideUses.Users.back().isUseOfPostIncrementedValue = true; 639b7427031372337e6d67f9573ec6c722ab5ea913eBill Wendling DOUT << " USING POSTINC SCEV, START=" << *NewStart<< "\n"; 6400ae33eb243417982fbdca792460bdd986108ac09Chris Lattner } else { 6417305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner StrideUses.addUser(Start, User, I); 642c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner } 643169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 644169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 645169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman return true; 646169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 647169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 648169974856781a1ce27af9ce6220c390b20c9e6ddNate Begemannamespace { 649169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// BasedUser - For a particular base value, keep information about how we've 650169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// partitioned the expression so far. 651169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman struct BasedUser { 652246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman /// SE - The current ScalarEvolution object. 653246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE; 654246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman 655a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// Base - The Base value for the PHI node that needs to be inserted for 656a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// this use. As the use is processed, information gets moved from this 657a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// field to the Imm field (below). BasedUser values are sorted by this 658a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// field. 659a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner SCEVHandle Base; 660a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 661169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Inst - The instruction using the induction variable. 662169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman Instruction *Inst; 663169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 664ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// OperandValToReplace - The operand value of Inst to replace with the 665ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// EmittedBase. 666ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Value *OperandValToReplace; 667169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 668169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Imm - The immediate value that should be added to the base immediately 669169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// before Inst, because it will be folded into the imm field of the 67033e3a36f0a116b758708ef8fd0695d3a4cb21531Dan Gohman /// instruction. This is also sometimes used for loop-variant values that 67133e3a36f0a116b758708ef8fd0695d3a4cb21531Dan Gohman /// must be added inside the loop. 672169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman SCEVHandle Imm; 673169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 674c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// Phi - The induction variable that performs the striding that 675c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// should be used for this user. 676c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *Phi; 677c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 678c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// IncV - The post-incremented value of Phi. 679c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *IncV; 680c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 681010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // isUseOfPostIncrementedValue - True if this should use the 682010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // post-incremented version of this IV, not the preincremented version. 683010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // This can only be set in special cases, such as the terminating setcc 684c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // instruction for a loop and uses outside the loop that are dominated by 685c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // the loop. 686010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner bool isUseOfPostIncrementedValue; 687a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 688246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman BasedUser(IVStrideUse &IVSU, ScalarEvolution *se) 689246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman : SE(se), Base(IVSU.Offset), Inst(IVSU.User), 690a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner OperandValToReplace(IVSU.OperandValToReplace), 691308f24d4525a6365f8d65ba821786189c080c0ceDale Johannesen Imm(SE->getIntegerSCEV(0, Base->getType())), 692a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner isUseOfPostIncrementedValue(IVSU.isUseOfPostIncrementedValue) {} 693169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 6942114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Once we rewrite the code to insert the new IVs we want, update the 6952114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // operands of Inst to use the new expression 'NewBase', with 'Imm' added 6962114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // to it. 6971bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner void RewriteInstructionToUseNewBase(const SCEVHandle &NewBase, 698f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *InsertPt, 6990e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng SCEVExpander &Rewriter, Loop *L, Pass *P, 70009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVectorImpl<Instruction*> &DeadInsts); 701221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 702221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Value *InsertCodeForBaseAtPosition(const SCEVHandle &NewBase, 703221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVExpander &Rewriter, 704221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Instruction *IP, Loop *L); 705169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman void dump() const; 706169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman }; 707169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 708169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 709169974856781a1ce27af9ce6220c390b20c9e6ddNate Begemanvoid BasedUser::dump() const { 710e81561909d128c6e2d8033cb5465a49b2596b26aBill Wendling cerr << " Base=" << *Base; 711e81561909d128c6e2d8033cb5465a49b2596b26aBill Wendling cerr << " Imm=" << *Imm; 712e81561909d128c6e2d8033cb5465a49b2596b26aBill Wendling cerr << " Inst: " << *Inst; 713169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 714169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 715221fc3c6d69bd3854e9121f51e3283492c222ab7Chris LattnerValue *BasedUser::InsertCodeForBaseAtPosition(const SCEVHandle &NewBase, 716221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVExpander &Rewriter, 717221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Instruction *IP, Loop *L) { 718221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Figure out where we *really* want to insert this code. In particular, if 719221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // the user is inside of a loop that is nested inside of L, we really don't 720221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // want to insert this expression before the user, we'd rather pull it out as 721221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // many loops as possible. 722221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner LoopInfo &LI = Rewriter.getLoopInfo(); 723221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Instruction *BaseInsertPt = IP; 724221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 725221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Figure out the most-nested loop that IP is in. 726221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Loop *InsertLoop = LI.getLoopFor(IP->getParent()); 727221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 728221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If InsertLoop is not L, and InsertLoop is nested inside of L, figure out 729221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // the preheader of the outer-most loop where NewBase is not loop invariant. 730eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen if (L->contains(IP->getParent())) 731eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen while (InsertLoop && NewBase->isLoopInvariant(InsertLoop)) { 732eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen BaseInsertPt = InsertLoop->getLoopPreheader()->getTerminator(); 733eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen InsertLoop = InsertLoop->getParentLoop(); 734eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen } 735221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 7362f09f519542202b8af227c2a524f8fe82378a934Dan Gohman Value *Base = Rewriter.expandCodeFor(NewBase, BaseInsertPt); 7372f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 738221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If there is no immediate value, skip the next part. 739cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman if (Imm->isZero()) 7402f09f519542202b8af227c2a524f8fe82378a934Dan Gohman return Base; 741b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner 742b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner // If we are inserting the base and imm values in the same block, make sure to 743b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner // adjust the IP position if insertion reused a result. 744b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner if (IP == BaseInsertPt) 745b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner IP = Rewriter.getInsertionPoint(); 746221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 747221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Always emit the immediate (if non-zero) into the same block as the user. 748246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle NewValSCEV = SE->getAddExpr(SE->getUnknown(Base), Imm); 749d19534add90a2a894af61523b830887097bb780bDan Gohman return Rewriter.expandCodeFor(NewValSCEV, IP); 750b47f6124f4893c00573d1a5be56a6e4cf877faebChris Lattner 751221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner} 752221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 753221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 7542114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner// Once we rewrite the code to insert the new IVs we want, update the 7552114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner// operands of Inst to use the new expression 'NewBase', with 'Imm' added 756f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// to it. NewBasePt is the last instruction which contributes to the 757f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// value of NewBase in the case that it's a diffferent instruction from 758f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// the PHI that NewBase is computed from, or null otherwise. 759f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// 7601bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattnervoid BasedUser::RewriteInstructionToUseNewBase(const SCEVHandle &NewBase, 761f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *NewBasePt, 7620e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng SCEVExpander &Rewriter, Loop *L, Pass *P, 76309fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner SmallVectorImpl<Instruction*> &DeadInsts){ 7642114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner if (!isa<PHINode>(Inst)) { 765c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // By default, insert code at the user instruction. 766c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner BasicBlock::iterator InsertPt = Inst; 767c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner 768c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // However, if the Operand is itself an instruction, the (potentially 769c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // complex) inserted code may be shared by many users. Because of this, we 770c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // want to emit code for the computation of the operand right before its old 771c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // computation. This is usually safe, because we obviously used to use the 772c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // computation when it was computed in its current block. However, in some 773c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // cases (e.g. use of a post-incremented induction variable) the NewBase 774c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // value will be pinned to live somewhere after the original computation. 775c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // In this case, we have to back off. 776f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // 777f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If this is a use outside the loop (which means after, since it is based 778f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // on a loop indvar) we use the post-incremented value, so that we don't 779f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // artificially make the preinc value live out the bottom of the loop. 780589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (!isUseOfPostIncrementedValue && L->contains(Inst->getParent())) { 781ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman if (NewBasePt && isa<PHINode>(OperandValToReplace)) { 782f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman InsertPt = NewBasePt; 783f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman ++InsertPt; 7846725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif } else if (Instruction *OpInst 7856725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif = dyn_cast<Instruction>(OperandValToReplace)) { 786c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner InsertPt = OpInst; 787c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner while (isa<PHINode>(InsertPt)) ++InsertPt; 788c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner } 789c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner } 790c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner Value *NewVal = InsertCodeForBaseAtPosition(NewBase, Rewriter, InsertPt, L); 791a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman // Adjust the type back to match the Inst. Note that we can't use InsertPt 792a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman // here because the SCEVExpander may have inserted the instructions after 793a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman // that point, in its efforts to avoid inserting redundant expressions. 794d19534add90a2a894af61523b830887097bb780bDan Gohman if (isa<PointerType>(OperandValToReplace->getType())) { 795a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman NewVal = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr, 796a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman NewVal, 797a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman OperandValToReplace->getType()); 798d19534add90a2a894af61523b830887097bb780bDan Gohman } 7992114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Replace the use of the operand Value with the new Phi we just created. 8002114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner Inst->replaceUsesOfWith(OperandValToReplace, NewVal); 8012f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8022f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Replacing with "; 8034a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, NewVal, /*PrintType=*/false)); 8042f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << ", which has value " << *NewBase << " plus IMM " << *Imm << "\n"; 8052114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner return; 8062114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 8072f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8082114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // PHI nodes are more complex. We have to insert one copy of the NewBase+Imm 809c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // expression into each operand block that uses it. Note that PHI nodes can 810c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // have multiple entries for the same predecessor. We use a map to make sure 811c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // that a PHI node only has a single Value* for each predecessor (which also 812c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // prevents us from inserting duplicate code in some blocks). 8138392772727ed9105c92fe4514d53dab74c333edcEvan Cheng DenseMap<BasicBlock*, Value*> InsertedCode; 8142114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner PHINode *PN = cast<PHINode>(Inst); 8152114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 8162114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner if (PN->getIncomingValue(i) == OperandValToReplace) { 8172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If the original expression is outside the loop, put the replacement 8182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // code in the same place as the original expression, 8192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // which need not be an immediate predecessor of this PHI. This way we 8202f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // need only one copy of it even if it is referenced multiple times in 8212f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // the PHI. We don't do this when the original expression is inside the 8221de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // loop because multiple copies sometimes do useful sinking of code in 8231de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // that case(?). 8242f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen Instruction *OldLoc = dyn_cast<Instruction>(OperandValToReplace); 8252f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(OldLoc->getParent())) { 8261de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // If this is a critical edge, split the edge so that we do not insert 8271de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // the code on all predecessor/successor paths. We do this unless this 8281de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // is the canonical backedge for this loop, as this can make some 8291de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // inserted code be in an illegal position. 8302f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen BasicBlock *PHIPred = PN->getIncomingBlock(i); 8312f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (e != 1 && PHIPred->getTerminator()->getNumSuccessors() > 1 && 8322f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen (PN->getParent() != L->getHeader() || !L->contains(PHIPred))) { 8332f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 8342f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // First step, split the critical edge. 8352f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SplitCriticalEdge(PHIPred, PN->getParent(), P, false); 8362f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 8372f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Next step: move the basic block. In particular, if the PHI node 8382f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // is outside of the loop, and PredTI is in the loop, we want to 8392f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // move the block to be immediately before the PHI block, not 8402f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // immediately after PredTI. 8412f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(PHIPred) && !L->contains(PN->getParent())) { 8422f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen BasicBlock *NewBB = PN->getIncomingBlock(i); 8432f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen NewBB->moveBefore(PN->getParent()); 8442f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 8452f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 8462f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Splitting the edge can reduce the number of PHI entries we have. 8472f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen e = PN->getNumIncomingValues(); 848e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner } 849e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner } 850c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner Value *&Code = InsertedCode[PN->getIncomingBlock(i)]; 851c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner if (!Code) { 852c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // Insert the code into the end of the predecessor block. 8532f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen Instruction *InsertPt = (L->contains(OldLoc->getParent())) ? 8542f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen PN->getIncomingBlock(i)->getTerminator() : 8552f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen OldLoc->getParent()->getTerminator(); 856221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Code = InsertCodeForBaseAtPosition(NewBase, Rewriter, InsertPt, L); 857d19534add90a2a894af61523b830887097bb780bDan Gohman 858684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // Adjust the type back to match the PHI. Note that we can't use 859684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // InsertPt here because the SCEVExpander may have inserted its 860684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // instructions after that point, in its efforts to avoid inserting 861684b22df79c51114a12289e10a4063d5f02259a9Chris Lattner // redundant expressions. 862d19534add90a2a894af61523b830887097bb780bDan Gohman if (isa<PointerType>(PN->getType())) { 863a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman Code = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr, 864a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman Code, 865a9cfed77b6bc4930ffb07cbd2877cef5c44d93b2Dan Gohman PN->getType()); 866d19534add90a2a894af61523b830887097bb780bDan Gohman } 8672f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8682f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Changing PHI use to "; 8694a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, Code, /*PrintType=*/false)); 8702f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << ", which has value " << *NewBase << " plus IMM " << *Imm << "\n"; 871c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner } 8722f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 8732114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Replace the use of the operand Value with the new Phi we just created. 874c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner PN->setIncomingValue(i, Code); 8752114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner Rewriter.clear(); 8762114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 8772114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 8780e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 8790e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng // PHI node might have become a constant value after SplitCriticalEdge. 88009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(Inst); 8812114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner} 8822114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner 8832114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner 884203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// fitsInAddressMode - Return true if V can be subsumed within an addressing 885203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// mode, and does not need to be put in a register first. 886203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstatic bool fitsInAddressMode(const SCEVHandle &V, const Type *UseTy, 887203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const TargetLowering *TLI, bool HasBaseReg) { 8883821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner if (SCEVConstant *SC = dyn_cast<SCEVConstant>(V)) { 8895eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t VC = SC->getValue()->getSExtValue(); 890579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (TLI) { 891579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 892579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseOffs = VC; 893203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen AM.HasBaseReg = HasBaseReg; 894579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner return TLI->isLegalAddressingMode(AM, UseTy); 895579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } else { 896d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng // Defaults to PPC. PPC allows a sign-extended 16-bit immediate field. 8975eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng return (VC > -(1 << 16) && VC < (1 << 16)-1); 898579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } 8993821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner } 900d29b6aa608d69f19b57ebd2ae630b040b1c4951dJeff Cohen 901169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman if (SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) 902169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman if (ConstantExpr *CE = dyn_cast<ConstantExpr>(SU->getValue())) 903579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (TLI && CE->getOpcode() == Instruction::PtrToInt) { 904d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng Constant *Op0 = CE->getOperand(0); 905579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (GlobalValue *GV = dyn_cast<GlobalValue>(Op0)) { 906579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 907579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseGV = GV; 908203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen AM.HasBaseReg = HasBaseReg; 909579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner return TLI->isLegalAddressingMode(AM, UseTy); 910579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } 911d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng } 912169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman return false; 913169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 914169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 915544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen/// MoveLoopVariantsToImmediateField - Move any subexpressions from Val that are 91644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner/// loop varying to the Imm operand. 917544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesenstatic void MoveLoopVariantsToImmediateField(SCEVHandle &Val, SCEVHandle &Imm, 918246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Loop *L, ScalarEvolution *SE) { 91944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (Val->isLoopInvariant(L)) return; // Nothing to do. 92044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 92144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (SCEVAddExpr *SAE = dyn_cast<SCEVAddExpr>(Val)) { 92244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner std::vector<SCEVHandle> NewOps; 92344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner NewOps.reserve(SAE->getNumOperands()); 92444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 92544b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner for (unsigned i = 0; i != SAE->getNumOperands(); ++i) 92644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (!SAE->getOperand(i)->isLoopInvariant(L)) { 92744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // If this is a loop-variant expression, it must stay in the immediate 92844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // field of the expression. 929246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, SAE->getOperand(i)); 93044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else { 93144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner NewOps.push_back(SAE->getOperand(i)); 93244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 93344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 93444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (NewOps.empty()) 935246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 93644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner else 937246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddExpr(NewOps); 93844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else if (SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Val)) { 93944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Try to pull immediates out of the start value of nested addrec's. 94044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner SCEVHandle Start = SARE->getStart(); 941544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen MoveLoopVariantsToImmediateField(Start, Imm, L, SE); 94244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 94344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner std::vector<SCEVHandle> Ops(SARE->op_begin(), SARE->op_end()); 94444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner Ops[0] = Start; 945246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddRecExpr(Ops, SARE->getLoop()); 94644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else { 94744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Otherwise, all of Val is variant, move the whole thing over. 948246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, Val); 949246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 95044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 95144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner} 95244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 95344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 95426d91f16464db56283087176a73981048331dd2dChris Lattner/// MoveImmediateValues - Look at Val, and pull out any additions of constants 955169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// that can fit into the immediate field of instructions in the target. 95626d91f16464db56283087176a73981048331dd2dChris Lattner/// Accumulate these immediate values into the Imm value. 957d277f2c66914aecb619c12855f6afae4c7ef883bEvan Chengstatic void MoveImmediateValues(const TargetLowering *TLI, 958d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng const Type *UseTy, 959d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng SCEVHandle &Val, SCEVHandle &Imm, 960246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman bool isAddress, Loop *L, 961246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE) { 9627a65839f4118c99fb1636c3cbb41b0bf7ef27665Chris Lattner if (SCEVAddExpr *SAE = dyn_cast<SCEVAddExpr>(Val)) { 96326d91f16464db56283087176a73981048331dd2dChris Lattner std::vector<SCEVHandle> NewOps; 96426d91f16464db56283087176a73981048331dd2dChris Lattner NewOps.reserve(SAE->getNumOperands()); 96526d91f16464db56283087176a73981048331dd2dChris Lattner 966221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner for (unsigned i = 0; i != SAE->getNumOperands(); ++i) { 967221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVHandle NewOp = SAE->getOperand(i); 968d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng MoveImmediateValues(TLI, UseTy, NewOp, Imm, isAddress, L, SE); 969221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 970221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner if (!NewOp->isLoopInvariant(L)) { 9717db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // If this is a loop-variant expression, it must stay in the immediate 9727db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // field of the expression. 973246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, NewOp); 97426d91f16464db56283087176a73981048331dd2dChris Lattner } else { 975221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner NewOps.push_back(NewOp); 976169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 977221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 97826d91f16464db56283087176a73981048331dd2dChris Lattner 97926d91f16464db56283087176a73981048331dd2dChris Lattner if (NewOps.empty()) 980246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 98126d91f16464db56283087176a73981048331dd2dChris Lattner else 982246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddExpr(NewOps); 98326d91f16464db56283087176a73981048331dd2dChris Lattner return; 9847a65839f4118c99fb1636c3cbb41b0bf7ef27665Chris Lattner } else if (SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Val)) { 9857a65839f4118c99fb1636c3cbb41b0bf7ef27665Chris Lattner // Try to pull immediates out of the start value of nested addrec's. 98626d91f16464db56283087176a73981048331dd2dChris Lattner SCEVHandle Start = SARE->getStart(); 987d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng MoveImmediateValues(TLI, UseTy, Start, Imm, isAddress, L, SE); 98826d91f16464db56283087176a73981048331dd2dChris Lattner 98926d91f16464db56283087176a73981048331dd2dChris Lattner if (Start != SARE->getStart()) { 99026d91f16464db56283087176a73981048331dd2dChris Lattner std::vector<SCEVHandle> Ops(SARE->op_begin(), SARE->op_end()); 99126d91f16464db56283087176a73981048331dd2dChris Lattner Ops[0] = Start; 992246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddRecExpr(Ops, SARE->getLoop()); 99326d91f16464db56283087176a73981048331dd2dChris Lattner } 99426d91f16464db56283087176a73981048331dd2dChris Lattner return; 995221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } else if (SCEVMulExpr *SME = dyn_cast<SCEVMulExpr>(Val)) { 996221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Transform "8 * (4 + v)" -> "32 + 8*V" if "32" fits in the immed field. 997203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (isAddress && fitsInAddressMode(SME->getOperand(0), UseTy, TLI, false) && 998221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SME->getNumOperands() == 2 && SME->isLoopInvariant(L)) { 999221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 1000246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle SubImm = SE->getIntegerSCEV(0, Val->getType()); 1001221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVHandle NewOp = SME->getOperand(1); 1002d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng MoveImmediateValues(TLI, UseTy, NewOp, SubImm, isAddress, L, SE); 1003221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 1004221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If we extracted something out of the subexpressions, see if we can 1005221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // simplify this! 1006221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner if (NewOp != SME->getOperand(1)) { 1007221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Scale SubImm up by "8". If the result is a target constant, we are 1008221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // good. 1009246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SubImm = SE->getMulExpr(SubImm, SME->getOperand(0)); 1010203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (fitsInAddressMode(SubImm, UseTy, TLI, false)) { 1011221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Accumulate the immediate. 1012246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, SubImm); 1013221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 1014221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Update what is left of 'Val'. 1015246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getMulExpr(SME->getOperand(0), NewOp); 1016221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner return; 1017221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 1018221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 1019221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 1020169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 1021169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 102226d91f16464db56283087176a73981048331dd2dChris Lattner // Loop-variant expressions must stay in the immediate field of the 102326d91f16464db56283087176a73981048331dd2dChris Lattner // expression. 1024203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if ((isAddress && fitsInAddressMode(Val, UseTy, TLI, false)) || 102526d91f16464db56283087176a73981048331dd2dChris Lattner !Val->isLoopInvariant(L)) { 1026246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, Val); 1027246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 102826d91f16464db56283087176a73981048331dd2dChris Lattner return; 10297a2ca56ef3bdda6874bcd4df2910fb5a33999f85Chris Lattner } 103026d91f16464db56283087176a73981048331dd2dChris Lattner 103126d91f16464db56283087176a73981048331dd2dChris Lattner // Otherwise, no immediates to move. 1032169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 1033169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 1034d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Chengstatic void MoveImmediateValues(const TargetLowering *TLI, 1035d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng Instruction *User, 1036d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng SCEVHandle &Val, SCEVHandle &Imm, 1037d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng bool isAddress, Loop *L, 1038d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng ScalarEvolution *SE) { 1039d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng const Type *UseTy = User->getType(); 1040d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (StoreInst *SI = dyn_cast<StoreInst>(User)) 1041d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng UseTy = SI->getOperand(0)->getType(); 1042d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng MoveImmediateValues(TLI, UseTy, Val, Imm, isAddress, L, SE); 1043d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng} 1044934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 10457e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// SeparateSubExprs - Decompose Expr into all of the subexpressions that are 10467e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// added together. This is used to reassociate common addition subexprs 10477e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// together for maximal sharing when rewriting bases. 1048934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattnerstatic void SeparateSubExprs(std::vector<SCEVHandle> &SubExprs, 1049246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Expr, 1050246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE) { 1051934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(Expr)) { 1052934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned j = 0, e = AE->getNumOperands(); j != e; ++j) 1053246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, AE->getOperand(j), SE); 1054934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } else if (SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Expr)) { 1055246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Zero = SE->getIntegerSCEV(0, Expr->getType()); 1056934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SARE->getOperand(0) == Zero) { 1057934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.push_back(Expr); 1058934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } else { 1059934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Compute the addrec with zero as its base. 1060934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner std::vector<SCEVHandle> Ops(SARE->op_begin(), SARE->op_end()); 1061934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner Ops[0] = Zero; // Start with zero base. 1062246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SubExprs.push_back(SE->getAddRecExpr(Ops, SARE->getLoop())); 1063934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1064934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1065246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, SARE->getOperand(0), SE); 1066934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1067cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman } else if (!Expr->isZero()) { 1068934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Do not add zero. 1069934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.push_back(Expr); 1070934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1071934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner} 1072934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1073203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen// This is logically local to the following function, but C++ says we have 1074203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen// to make it file scope. 1075203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstruct SubExprUseData { unsigned Count; bool notAllUsesAreFree; }; 1076934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1077203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// RemoveCommonExpressionsFromUseBases - Look through all of the Bases of all 1078203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// the Uses, removing any common subexpressions, except that if all such 1079203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// subexpressions can be folded into an addressing mode for all uses inside 1080203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// the loop (this case is referred to as "free" in comments herein) we do 1081203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// not remove anything. This looks for things like (a+b+c) and 1082f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner/// (a+c+d) and computes the common (a+c) subexpression. The common expression 1083f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner/// is *removed* from the Bases and returned. 10841bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattnerstatic SCEVHandle 1085246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan GohmanRemoveCommonExpressionsFromUseBases(std::vector<BasedUser> &Uses, 1086203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen ScalarEvolution *SE, Loop *L, 1087203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const TargetLowering *TLI) { 10881bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner unsigned NumUses = Uses.size(); 10891bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1090f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Only one use? This is a very common case, so we handle it specially and 1091f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // cheaply. 1092246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle Zero = SE->getIntegerSCEV(0, Uses[0].Base->getType()); 10931bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner SCEVHandle Result = Zero; 1094203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SCEVHandle FreeResult = Zero; 10951bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner if (NumUses == 1) { 1096f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If the use is inside the loop, use its base, regardless of what it is: 1097f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // it is clearly shared across all the IV's. If the use is outside the loop 1098f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // (which means after it) we don't want to factor anything *into* the loop, 1099f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // so just use 0 as the base. 1100589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (L->contains(Uses[0].Inst->getParent())) 1101589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen std::swap(Result, Uses[0].Base); 11021bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner return Result; 11031bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner } 11041bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 11051bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // To find common subexpressions, count how many of Uses use each expression. 11061bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If any subexpressions are used Uses.size() times, they are common. 1107203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Also track whether all uses of each expression can be moved into an 1108203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // an addressing mode "for free"; such expressions are left within the loop. 1109203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // struct SubExprUseData { unsigned Count; bool notAllUsesAreFree; }; 1110203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen std::map<SCEVHandle, SubExprUseData> SubExpressionUseData; 11111bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1112d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // UniqueSubExprs - Keep track of all of the subexpressions we see in the 1113d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // order we see them. 1114d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner std::vector<SCEVHandle> UniqueSubExprs; 1115d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner 1116934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner std::vector<SCEVHandle> SubExprs; 1117f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner unsigned NumUsesInsideLoop = 0; 1118934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned i = 0; i != NumUses; ++i) { 1119f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If the user is outside the loop, just ignore it for base computation. 1120f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Since the user is outside the loop, it must be *after* the loop (if it 1121f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // were before, it could not be based on the loop IV). We don't want users 1122f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // after the loop to affect base computation of values *inside* the loop, 1123f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // because we can always add their offsets to the result IV after the loop 1124f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // is done, ensuring we get good code inside the loop. 1125589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (!L->contains(Uses[i].Inst->getParent())) 1126589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen continue; 1127589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen NumUsesInsideLoop++; 1128589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 1129934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // If the base is zero (which is common), return zero now, there are no 1130934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // CSEs we can find. 1131934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (Uses[i].Base == Zero) return Zero; 1132934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1133203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // If this use is as an address we may be able to put CSEs in the addressing 1134203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // mode rather than hoisting them. 1135203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen bool isAddrUse = isAddressUse(Uses[i].Inst, Uses[i].OperandValToReplace); 1136203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We may need the UseTy below, but only when isAddrUse, so compute it 1137203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // only in that case. 1138203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const Type *UseTy = 0; 1139203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (isAddrUse) { 1140203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen UseTy = Uses[i].Inst->getType(); 1141203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (StoreInst *SI = dyn_cast<StoreInst>(Uses[i].Inst)) 1142203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen UseTy = SI->getOperand(0)->getType(); 1143203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1144203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 1145934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Split the expression into subexprs. 1146246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, Uses[i].Base, SE); 1147203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Add one to SubExpressionUseData.Count for each subexpr present, and 1148203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // if the subexpr is not a valid immediate within an addressing mode use, 1149203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // set SubExpressionUseData.notAllUsesAreFree. We definitely want to 1150203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // hoist these out of the loop (if they are common to all uses). 1151203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) { 1152203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (++SubExpressionUseData[SubExprs[j]].Count == 1) 1153d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner UniqueSubExprs.push_back(SubExprs[j]); 1154203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!isAddrUse || !fitsInAddressMode(SubExprs[j], UseTy, TLI, false)) 1155203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData[SubExprs[j]].notAllUsesAreFree = true; 1156203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1157934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.clear(); 1158934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1159934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1160d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // Now that we know how many times each is used, build Result. Iterate over 1161d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // UniqueSubexprs so that we have a stable ordering. 1162d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner for (unsigned i = 0, e = UniqueSubExprs.size(); i != e; ++i) { 1163203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen std::map<SCEVHandle, SubExprUseData>::iterator I = 1164203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.find(UniqueSubExprs[i]); 1165203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen assert(I != SubExpressionUseData.end() && "Entry not found?"); 1166203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (I->second.Count == NumUsesInsideLoop) { // Found CSE! 1167203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (I->second.notAllUsesAreFree) 1168203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen Result = SE->getAddExpr(Result, I->first); 1169203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen else 1170203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen FreeResult = SE->getAddExpr(FreeResult, I->first); 1171203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } else 1172203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Remove non-cse's from SubExpressionUseData. 1173203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.erase(I); 1174d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner } 1175203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 1176203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (FreeResult != Zero) { 1177203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We have some subexpressions that can be subsumed into addressing 1178203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // modes in every use inside the loop. However, it's possible that 1179203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // there are so many of them that the combined FreeResult cannot 1180203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // be subsumed, or that the target cannot handle both a FreeResult 1181203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // and a Result in the same instruction (for example because it would 1182203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // require too many registers). Check this. 1183203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned i=0; i<NumUses; ++i) { 1184203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!L->contains(Uses[i].Inst->getParent())) 1185203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen continue; 1186203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We know this is an addressing mode use; if there are any uses that 1187203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // are not, FreeResult would be Zero. 1188203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const Type *UseTy = Uses[i].Inst->getType(); 1189203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (StoreInst *SI = dyn_cast<StoreInst>(Uses[i].Inst)) 1190203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen UseTy = SI->getOperand(0)->getType(); 1191203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!fitsInAddressMode(FreeResult, UseTy, TLI, Result!=Zero)) { 1192203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // FIXME: could split up FreeResult into pieces here, some hoisted 1193b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // and some not. There is no obvious advantage to this. 1194203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen Result = SE->getAddExpr(Result, FreeResult); 1195203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen FreeResult = Zero; 1196203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen break; 1197203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1198203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1199203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1200203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 12011bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If we found no CSE's, return now. 12021bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner if (Result == Zero) return Result; 12031bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1204203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // If we still have a FreeResult, remove its subexpressions from 1205203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // SubExpressionUseData. This means they will remain in the use Bases. 1206203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (FreeResult != Zero) { 1207203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SeparateSubExprs(SubExprs, FreeResult, SE); 1208203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) { 1209203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen std::map<SCEVHandle, SubExprUseData>::iterator I = 1210203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.find(SubExprs[j]); 1211203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.erase(I); 1212203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1213203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExprs.clear(); 1214203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 1215203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 12161bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Otherwise, remove all of the CSE's we found from each of the base values. 1217934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned i = 0; i != NumUses; ++i) { 1218fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // Uses outside the loop don't necessarily include the common base, but 1219fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // the final IV value coming into those uses does. Instead of trying to 1220fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // remove the pieces of the common base, which might not be there, 1221fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // subtract off the base to compensate for this. 1222fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen if (!L->contains(Uses[i].Inst->getParent())) { 1223fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen Uses[i].Base = SE->getMinusSCEV(Uses[i].Base, Result); 1224589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen continue; 1225fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen } 1226589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 1227934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Split the expression into subexprs. 1228246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, Uses[i].Base, SE); 1229934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1230934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Remove any common subexpressions. 1231934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) 1232203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (SubExpressionUseData.count(SubExprs[j])) { 1233934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.erase(SubExprs.begin()+j); 1234934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner --j; --e; 1235934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 1236934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 1237f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Finally, add the non-shared expressions together. 1238934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SubExprs.empty()) 12391bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Uses[i].Base = Zero; 1240934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner else 1241246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Uses[i].Base = SE->getAddExpr(SubExprs); 124227e5142309946ca12c633b265673af0c24684427Chris Lattner SubExprs.clear(); 1243934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 12441bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 12451bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner return Result; 12461bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner} 12471bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1248dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen/// ValidStride - Check whether the given Scale is valid for all loads and 1249579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner/// stores in UsersToProcess. 1250dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen/// 125102e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohmanbool LoopStrengthReduce::ValidStride(bool HasBaseReg, 125202e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman int64_t Scale, 1253dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess) { 1254d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (!TLI) 1255d6b62a572210aff965a55626cf36a68821838844Evan Cheng return true; 1256d6b62a572210aff965a55626cf36a68821838844Evan Cheng 12578e59e163db8cd3e7b4c96e438fbedf78bff06707Dale Johannesen for (unsigned i=0, e = UsersToProcess.size(); i!=e; ++i) { 12581ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner // If this is a load or other access, pass the type of the access in. 12591ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner const Type *AccessTy = Type::VoidTy; 12601ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner if (StoreInst *SI = dyn_cast<StoreInst>(UsersToProcess[i].Inst)) 12611ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner AccessTy = SI->getOperand(0)->getType(); 12621ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner else if (LoadInst *LI = dyn_cast<LoadInst>(UsersToProcess[i].Inst)) 12631ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner AccessTy = LI->getType(); 126455e641b766a18878b51551d626d5a566102e487eEvan Cheng else if (isa<PHINode>(UsersToProcess[i].Inst)) 126555e641b766a18878b51551d626d5a566102e487eEvan Cheng continue; 12661ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner 1267579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 1268579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (SCEVConstant *SC = dyn_cast<SCEVConstant>(UsersToProcess[i].Imm)) 1269579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseOffs = SC->getValue()->getSExtValue(); 1270cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman AM.HasBaseReg = HasBaseReg || !UsersToProcess[i].Base->isZero(); 1271579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.Scale = Scale; 1272579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner 1273579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner // If load[imm+r*scale] is illegal, bail out. 1274d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (!TLI->isLegalAddressingMode(AM, AccessTy)) 1275dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen return false; 12768e59e163db8cd3e7b4c96e438fbedf78bff06707Dale Johannesen } 1277dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen return true; 1278dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen} 12791bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 12801de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen/// RequiresTypeConversion - Returns true if converting Ty1 to Ty2 is not 12815f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// a nop. 12822bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Chengbool LoopStrengthReduce::RequiresTypeConversion(const Type *Ty1, 12832bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng const Type *Ty2) { 12842bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (Ty1 == Ty2) 12855f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return false; 12861de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (Ty1->canLosslesslyBitCastTo(Ty2)) 12871de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return false; 12882bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (TLI && TLI->isTruncateFree(Ty1, Ty2)) 12892bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng return false; 12901de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (isa<PointerType>(Ty2) && Ty1->canLosslesslyBitCastTo(UIntPtrTy)) 12911de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return false; 12921de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (isa<PointerType>(Ty1) && Ty2->canLosslesslyBitCastTo(UIntPtrTy)) 12931de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return false; 12941de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return true; 12955f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 12965f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1297eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng/// CheckForIVReuse - Returns the multiple if the stride is the multiple 1298eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng/// of a previous stride and it is a legal value for the target addressing 129902e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman/// mode scale component and optional base reg. This allows the users of 130002e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman/// this stride to be rewritten as prev iv * factor. It returns 0 if no 1301b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen/// reuse is possible. Factors can be negative on same targets, e.g. ARM. 13022f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// 13032f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// If all uses are outside the loop, we don't require that all multiplies 13042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// be folded into the addressing mode, nor even that the factor be constant; 13052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// a multiply (executed once) outside the loop is better than another IV 13062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// within. Well, usually. 13072f46bb8178e30e3b845859a44b57c048db06ef84Dale JohannesenSCEVHandle LoopStrengthReduce::CheckForIVReuse(bool HasBaseReg, 13082bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng bool AllUsesAreAddresses, 1309b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool AllUsesAreOutsideLoop, 131002e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman const SCEVHandle &Stride, 1311dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen IVExpr &IV, const Type *Ty, 1312dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess) { 1313eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng if (SCEVConstant *SC = dyn_cast<SCEVConstant>(Stride)) { 1314502db93a8ab376730164db43ca3ce8032b72bd59Reid Spencer int64_t SInt = SC->getValue()->getSExtValue(); 1315b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen for (unsigned NewStride = 0, e = StrideOrder.size(); NewStride != e; 1316b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen ++NewStride) { 1317b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen std::map<SCEVHandle, IVsOfOneStride>::iterator SI = 1318b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen IVsByStride.find(StrideOrder[NewStride]); 13192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end() || !isa<SCEVConstant>(SI->first)) 1320b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen continue; 13215eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 13222bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (SI->first != Stride && 13231d31290634eccc3b360c427282d59780d76b9169Chris Lattner (unsigned(abs(SInt)) < SSInt || (SInt % SSInt) != 0)) 1324eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng continue; 13255eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t Scale = SInt / SSInt; 1326dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // Check that this stride is valid for all the types used for loads and 1327dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // stores; if it can be used for some and not others, we might as well use 1328dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // the original stride everywhere, since we have to create the IV for it 1329aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman // anyway. If the scale is 1, then we don't need to worry about folding 1330aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman // multiplications. 1331aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman if (Scale == 1 || 1332aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman (AllUsesAreAddresses && 1333aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman ValidStride(HasBaseReg, Scale, UsersToProcess))) 13345eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 13355eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng IE = SI->second.IVs.end(); II != IE; ++II) 13365eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng // FIXME: Only handle base == 0 for now. 13375eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng // Only reuse previous IV if it would not require a type conversion. 1338cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman if (II->Base->isZero() && 13392bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng !RequiresTypeConversion(II->Base->getType(), Ty)) { 13405eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng IV = *II; 13412f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(Scale, Stride->getType()); 13425eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng } 1343eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 13442f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } else if (AllUsesAreOutsideLoop) { 13452f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonconstant strides here; it is really really right to substitute 13462f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // an existing IV if we can. 13472f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned NewStride = 0, e = StrideOrder.size(); NewStride != e; 13482f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen ++NewStride) { 13492f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen std::map<SCEVHandle, IVsOfOneStride>::iterator SI = 13502f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IVsByStride.find(StrideOrder[NewStride]); 13512f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end() || !isa<SCEVConstant>(SI->first)) 13522f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 13532f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 13542f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI->first != Stride && SSInt != 1) 13552f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 13562f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 13572f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IE = SI->second.IVs.end(); II != IE; ++II) 13582f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonzero base here. 13592f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Only reuse previous IV if it would not require a type conversion. 13602f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!RequiresTypeConversion(II->Base->getType(), Ty)) { 13612f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IV = *II; 13622f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return Stride; 13632f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 13642f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 13652f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Special case, old IV is -1*x and this one is x. Can treat this one as 13662f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // -1*old. 13672f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned NewStride = 0, e = StrideOrder.size(); NewStride != e; 13682f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen ++NewStride) { 13692f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen std::map<SCEVHandle, IVsOfOneStride>::iterator SI = 13702f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IVsByStride.find(StrideOrder[NewStride]); 13712f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end()) 13722f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 13732f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(SI->first)) 13742f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SCEVConstant *SC = dyn_cast<SCEVConstant>(ME->getOperand(0))) 13752f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (Stride == ME->getOperand(1) && 13762f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SC->getValue()->getSExtValue() == -1LL) 13772f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 13782f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IE = SI->second.IVs.end(); II != IE; ++II) 13792f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonzero base here. 13802f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Only reuse previous IV if it would not require type conversion. 13812f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!RequiresTypeConversion(II->Base->getType(), Ty)) { 13822f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IV = *II; 13832f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(-1LL, Stride->getType()); 13842f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 13852f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 1386eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 13872f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(0, Stride->getType()); 1388eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng} 1389eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 13907e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// PartitionByIsUseOfPostIncrementedValue - Simple boolean predicate that 13917e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// returns true if Val's isUseOfPostIncrementedValue is true. 13927e79b3898ddd919170d367a516f51296017146c2Chris Lattnerstatic bool PartitionByIsUseOfPostIncrementedValue(const BasedUser &Val) { 13937e79b3898ddd919170d367a516f51296017146c2Chris Lattner return Val.isUseOfPostIncrementedValue; 13947e79b3898ddd919170d367a516f51296017146c2Chris Lattner} 1395eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 13964a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman/// isNonConstantNegative - Return true if the specified scev is negated, but 1397fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner/// not a constant. 1398fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattnerstatic bool isNonConstantNegative(const SCEVHandle &Expr) { 1399fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Expr); 1400fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner if (!Mul) return false; 1401fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 1402fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner // If there is a constant factor, it will be first. 1403fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 1404fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner if (!SC) return false; 1405fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 1406fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner // Return true if the value is negative, this matches things like (-42 * V). 1407fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner return SC->getValue()->getValue().isNegative(); 1408fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner} 1409fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 14105f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng// CollectIVUsers - Transform our list of users and offsets to a bit more 141173b43b9b549a75fb0015c825df68abd95705a67cDan Gohman// complex table. In this new vector, each 'BasedUser' contains 'Base', the base 141273b43b9b549a75fb0015c825df68abd95705a67cDan Gohman// of the strided accesses, as well as the old information from Uses. We 14135f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng// progressively move information from the Base field to the Imm field, until 14145f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng// we eventually have the full access expression to rewrite the use. 14155f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan ChengSCEVHandle LoopStrengthReduce::CollectIVUsers(const SCEVHandle &Stride, 14165f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 14175f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 14185f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool &AllUsesAreAddresses, 1419b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool &AllUsesAreOutsideLoop, 14205f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> &UsersToProcess) { 1421169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman UsersToProcess.reserve(Uses.Users.size()); 1422a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner for (unsigned i = 0, e = Uses.Users.size(); i != e; ++i) { 1423246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess.push_back(BasedUser(Uses.Users[i], SE)); 1424a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 142567c79892949332568b082f124d9598971fa3277fDale Johannesen // Move any loop variant operands from the offset field to the immediate 1426a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner // field of the use, so that we don't try to use something before it is 1427a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner // computed. 1428544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen MoveLoopVariantsToImmediateField(UsersToProcess.back().Base, 1429246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess.back().Imm, L, SE); 1430a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner assert(UsersToProcess.back().Base->isLoopInvariant(L) && 143126d91f16464db56283087176a73981048331dd2dChris Lattner "Base value is not loop invariant!"); 14322461dff0700d0e34b9854d96a5cc03921b375525Chris Lattner } 1433eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 143431e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // We now have a whole bunch of uses of like-strided induction variables, but 143531e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // they might all have different bases. We want to emit one PHI node for this 143631e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // stride which we fold as many common expressions (between the IVs) into as 143731e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // possible. Start by identifying the common expressions in the base values 143831e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // for the strides (e.g. if we have "A+C+B" and "A+B+D" as our bases, find 143931e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // "A+B"), emit it to the preheader, then remove the expression from the 144031e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // UsersToProcess base values. 144131e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng SCEVHandle CommonExprs = 1442203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen RemoveCommonExpressionsFromUseBases(UsersToProcess, SE, L, TLI); 144302e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman 144444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Next, figure out what we can represent in the immediate fields of 144544b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // instructions. If we can represent anything there, move it to the imm 14461bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // fields of the BasedUsers. We do this so that it increases the commonality 14471bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // of the remaining uses. 144832e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng unsigned NumPHI = 0; 1449d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng bool HasAddress = false; 145044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 145180b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // If the user is not in the current loop, this means it is using the exit 145280b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // value of the IV. Do not put anything in the base, make sure it's all in 145380b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // the immediate field to allow as much factoring as possible. 145480b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner if (!L->contains(UsersToProcess[i].Inst->getParent())) { 1455246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Imm = SE->getAddExpr(UsersToProcess[i].Imm, 1456246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Base); 14578385e51e210fb5c1e7e48eae150b31679b3e137dChris Lattner UsersToProcess[i].Base = 1458246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SE->getIntegerSCEV(0, UsersToProcess[i].Base->getType()); 145980b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner } else { 1460d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // Not all uses are outside the loop. 1461d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AllUsesAreOutsideLoop = false; 1462d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 146380b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // Addressing modes can be folded into loads and stores. Be careful that 146480b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // the store is through the expression, not of the expression though. 146532e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng bool isPHI = false; 1466d6b62a572210aff965a55626cf36a68821838844Evan Cheng bool isAddress = isAddressUse(UsersToProcess[i].Inst, 1467d6b62a572210aff965a55626cf36a68821838844Evan Cheng UsersToProcess[i].OperandValToReplace); 1468d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (isa<PHINode>(UsersToProcess[i].Inst)) { 146932e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng isPHI = true; 147032e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng ++NumPHI; 14712acc7601650654d03cd53faeece8d7685a203105Dan Gohman } 147202e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman 1473d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng if (isAddress) 1474d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng HasAddress = true; 1475b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen 147602e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman // If this use isn't an address, then not all uses are addresses. 147755e641b766a18878b51551d626d5a566102e487eEvan Cheng if (!isAddress && !isPHI) 147802e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman AllUsesAreAddresses = false; 147980b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner 14801d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng MoveImmediateValues(TLI, UsersToProcess[i].Inst, UsersToProcess[i].Base, 1481246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Imm, isAddress, L, SE); 148280b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner } 148344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 1484d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1485d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // If one of the use is a PHI node and all other uses are addresses, still 148632e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng // allow iv reuse. Essentially we are trading one constant multiplication 148732e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng // for one fewer iv. 148832e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng if (NumPHI > 1) 148932e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng AllUsesAreAddresses = false; 1490d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 1491d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng // There are no in-loop address uses. 1492d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng if (AllUsesAreAddresses && (!HasAddress && !AllUsesAreOutsideLoop)) 1493d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng AllUsesAreAddresses = false; 149432e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng 14955f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return CommonExprs; 14965f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 14975f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1498c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// ShouldUseFullStrengthReductionMode - Test whether full strength-reduction 1499c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// is valid and profitable for the given set of users of a stride. In 1500c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// full strength-reduction mode, all addresses at the current stride are 1501c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// strength-reduced all the way down to pointer arithmetic. 1502c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1503c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanbool LoopStrengthReduce::ShouldUseFullStrengthReductionMode( 1504c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const std::vector<BasedUser> &UsersToProcess, 1505c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1506c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool AllUsesAreAddresses, 1507c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride) { 1508c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!EnableFullLSRMode) 1509c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1510c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1511c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // The heuristics below aim to avoid increasing register pressure, but 1512c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // fully strength-reducing all the addresses increases the number of 1513c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // add instructions, so don't do this when optimizing for size. 1514c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: If the loop is large, the savings due to simpler addresses 1515c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // may oughtweight the costs of the extra increment instructions. 1516c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (L->getHeader()->getParent()->hasFnAttr(Attribute::OptimizeForSize)) 1517c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1518c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1519c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: For now, don't do full strength reduction if there could 1520c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // potentially be greater-stride multiples of the current stride 1521c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // which could reuse the current stride IV. 1522c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (StrideOrder.back() != Stride) 1523c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1524c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1525c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Iterate through the uses to find conditions that automatically rule out 1526c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // full-lsr mode. 1527c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ) { 1528c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEV *Base = UsersToProcess[i].Base; 1529c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEV *Imm = UsersToProcess[i].Imm; 1530c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If any users have a loop-variant component, they can't be fully 1531c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength-reduced. 1532c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (Imm && !Imm->isLoopInvariant(L)) 1533c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1534c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If there are to users with the same base and the difference between 1535c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // the two Imm values can't be folded into the address, full 1536c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength reduction would increase register pressure. 1537c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman do { 1538c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEV *CurImm = UsersToProcess[i].Imm; 1539a04af4380d6814cb908d80ff8f52badc926a8bdcDan Gohman if ((CurImm || Imm) && CurImm != Imm) { 1540c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!CurImm) CurImm = SE->getIntegerSCEV(0, Stride->getType()); 1541c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!Imm) Imm = SE->getIntegerSCEV(0, Stride->getType()); 1542c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Instruction *Inst = UsersToProcess[i].Inst; 1543c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Type *UseTy = Inst->getType(); 1544c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) 1545c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UseTy = SI->getOperand(0)->getType(); 1546c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Diff = SE->getMinusSCEV(UsersToProcess[i].Imm, Imm); 1547c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!Diff->isZero() && 1548c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (!AllUsesAreAddresses || 1549c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman !fitsInAddressMode(Diff, UseTy, TLI, /*HasBaseReg=*/true))) 1550c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1551c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1552c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } while (++i != e && Base == UsersToProcess[i].Base); 1553c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1554c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1555c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If there's exactly one user in this stride, fully strength-reducing it 1556c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // won't increase register pressure. If it's starting from a non-zero base, 1557c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // it'll be simpler this way. 1558c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (UsersToProcess.size() == 1 && !UsersToProcess[0].Base->isZero()) 1559c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return true; 1560c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1561c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Otherwise, if there are any users in this stride that don't require 1562c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // a register for their base, full strength-reduction will increase 1563c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // register pressure. 1564c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) 1565f0baa6e9cba77f54cee438e9a10e41c37789403eDan Gohman if (UsersToProcess[i].Base->isZero()) 1566c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1567c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1568c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Otherwise, go for it. 1569c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return true; 1570c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1571c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1572c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// InsertAffinePhi Create and insert a PHI node for an induction variable 1573c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// with the specified start and step values in the specified loop. 1574c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1575c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// If NegateStride is true, the stride should be negated by using a 1576c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// subtract instead of an add. 1577c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1578c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// Return the created phi node, and return the step instruction by 1579c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// reference in IncV. 1580c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1581c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic PHINode *InsertAffinePhi(SCEVHandle Start, SCEVHandle Step, 1582c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1583c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &Rewriter, 1584c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *&IncV) { 1585c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(Start->isLoopInvariant(L) && "New PHI start is not loop invariant!"); 1586c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(Step->isLoopInvariant(L) && "New PHI stride is not loop invariant!"); 1587c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1588c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman BasicBlock *Header = L->getHeader(); 1589c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman BasicBlock *Preheader = L->getLoopPreheader(); 1590c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1591c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PHINode *PN = PHINode::Create(Start->getType(), "lsr.iv", Header->begin()); 1592c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PN->addIncoming(Rewriter.expandCodeFor(Start, Preheader->getTerminator()), 1593c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Preheader); 1594c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1595c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman pred_iterator HPI = pred_begin(Header); 1596c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(HPI != pred_end(Header) && "Loop with zero preds???"); 1597c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!L->contains(*HPI)) ++HPI; 1598c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(HPI != pred_end(Header) && L->contains(*HPI) && 1599c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "No backedge in loop?"); 1600c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1601c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If the stride is negative, insert a sub instead of an add for the 1602c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // increment. 1603c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool isNegative = isNonConstantNegative(Step); 1604c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle IncAmount = Step; 1605c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isNegative) 1606c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncAmount = Rewriter.SE.getNegativeSCEV(Step); 1607c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1608c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Insert an add instruction right before the terminator corresponding 1609c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // to the back-edge. 1610c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *StepV = Rewriter.expandCodeFor(IncAmount, Preheader->getTerminator()); 1611c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isNegative) { 1612c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV = BinaryOperator::CreateSub(PN, StepV, "lsr.iv.next", 1613c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (*HPI)->getTerminator()); 1614c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } else { 1615c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV = BinaryOperator::CreateAdd(PN, StepV, "lsr.iv.next", 1616c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (*HPI)->getTerminator()); 1617c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1618c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!isa<ConstantInt>(StepV)) ++NumVariable; 1619c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1620c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman pred_iterator PI = pred_begin(Header); 1621c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (*PI == L->getLoopPreheader()) 1622c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++PI; 1623c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PN->addIncoming(IncV, *PI); 1624c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1625c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++NumInserted; 1626c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return PN; 1627c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1628c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1629c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic void SortUsersToProcess(std::vector<BasedUser> &UsersToProcess) { 1630c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We want to emit code for users inside the loop first. To do this, we 1631c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // rearrange BasedUser so that the entries at the end have 1632c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // isUseOfPostIncrementedValue = false, because we pop off the end of the 1633c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // vector (so we handle them first). 1634c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::partition(UsersToProcess.begin(), UsersToProcess.end(), 1635c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PartitionByIsUseOfPostIncrementedValue); 1636c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1637c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Sort this by base, so that things with the same base are handled 1638c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // together. By partitioning first and stable-sorting later, we are 1639c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // guaranteed that within each base we will pop off users from within the 1640c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // loop before users outside of the loop with a particular base. 1641c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // 1642c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We would like to use stable_sort here, but we can't. The problem is that 1643c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // SCEVHandle's don't have a deterministic ordering w.r.t to each other, so 1644c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // we don't have anything to do a '<' comparison on. Because we think the 1645c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // number of uses is small, do a horrible bubble sort which just relies on 1646c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // ==. 1647c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1648c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Get a base value. 1649c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Base = UsersToProcess[i].Base; 1650c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1651c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Compact everything with this base to be consecutive with this one. 1652c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned j = i+1; j != e; ++j) { 1653c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (UsersToProcess[j].Base == Base) { 1654c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::swap(UsersToProcess[i+1], UsersToProcess[j]); 1655c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++i; 1656c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1657c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1658c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1659c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1660c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 16616b38e29f13cb8700146e4b170567e2828553db9aDan Gohman/// PrepareToStrengthReduceFully - Prepare to fully strength-reduce 16626b38e29f13cb8700146e4b170567e2828553db9aDan Gohman/// UsersToProcess, meaning lowering addresses all the way down to direct 16636b38e29f13cb8700146e4b170567e2828553db9aDan Gohman/// pointer arithmetic. 1664c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1665c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1666c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceFully( 1667c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1668c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 1669c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 1670c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1671c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter) { 1672c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " Fully reducing all users\n"; 1673c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1674c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Rewrite the UsersToProcess records, creating a separate PHI for each 1675c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // unique Base value. 1676c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ) { 1677c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: The uses are grouped by base, but not sorted. We arbitrarily 1678c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // pick the first Imm value here to start with, and adjust it for the 1679c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // other uses. 1680c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Imm = UsersToProcess[i].Imm; 1681c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Base = UsersToProcess[i].Base; 1682c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Start = SE->getAddExpr(CommonExprs, Base, Imm); 1683c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *IncV; 1684c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PHINode *Phi = InsertAffinePhi(Start, Stride, L, 1685c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PreheaderRewriter, 1686c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV); 1687c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Loop over all the users with the same base. 1688c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman do { 1689c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Base = SE->getIntegerSCEV(0, Stride->getType()); 1690c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Imm = SE->getMinusSCEV(UsersToProcess[i].Imm, Imm); 1691c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = Phi; 1692c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].IncV = IncV; 1693c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(UsersToProcess[i].Imm->isLoopInvariant(L) && 1694c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "ShouldUseFullStrengthReductionMode should reject this!"); 1695c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } while (++i != e && Base == UsersToProcess[i].Base); 1696c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1697c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1698c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1699c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// PrepareToStrengthReduceWithNewPhi - Insert a new induction variable for the 1700c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// given users to share. 1701c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1702c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1703c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceWithNewPhi( 1704c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1705c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle Stride, 1706c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle CommonExprs, 1707c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 1708c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1709c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter) { 1710c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " Inserting new PHI:\n"; 1711c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1712c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *IncV; 1713c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PHINode *Phi = InsertAffinePhi(SE->getUnknown(CommonBaseV), 1714c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Stride, L, 1715c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PreheaderRewriter, 1716c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV); 1717c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1718c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Remember this in case a later stride is multiple of this. 1719c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IVsByStride[Stride].addIV(Stride, CommonExprs, Phi, IncV); 1720c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1721c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // All the users will share this new IV. 1722c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1723c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = Phi; 1724c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].IncV = IncV; 1725c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1726c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1727c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " IV="; 1728c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DEBUG(WriteAsOperand(*DOUT, Phi, /*PrintType=*/false)); 1729c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << ", INC="; 1730c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DEBUG(WriteAsOperand(*DOUT, IncV, /*PrintType=*/false)); 1731c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << "\n"; 1732c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1733c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1734c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// PrepareToStrengthReduceWithNewPhi - Prepare for the given users to reuse 1735c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// an induction variable with a stride that is a factor of the current 1736c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// induction variable. 1737c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1738c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1739c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceFromSmallerStride( 1740c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1741c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 1742c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const IVExpr &ReuseIV, 1743c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Instruction *PreInsertPt) { 1744c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman DOUT << " Rewriting in terms of existing IV of STRIDE " << *ReuseIV.Stride 1745c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman << " and BASE " << *ReuseIV.Base << "\n"; 1746c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1747c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // All the users will share the reused IV. 1748c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1749c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = ReuseIV.PHI; 1750c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].IncV = ReuseIV.IncV; 1751c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1752c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1753c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Constant *C = dyn_cast<Constant>(CommonBaseV); 1754c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (C && 1755c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (!C->isNullValue() && 1756c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman !fitsInAddressMode(SE->getUnknown(CommonBaseV), CommonBaseV->getType(), 1757c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman TLI, false))) 1758c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We want the common base emitted into the preheader! This is just 1759c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // using cast as a copy so BitCast (no-op cast) is appropriate 1760c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman CommonBaseV = new BitCastInst(CommonBaseV, CommonBaseV->getType(), 1761c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "commonbase", PreInsertPt); 1762c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1763c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1764d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Chengstatic bool IsImmFoldedIntoAddrMode(GlobalValue *GV, int64_t Offset, 1765d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng const Type *ReplacedTy, 1766d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng std::vector<BasedUser> &UsersToProcess, 1767d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng const TargetLowering *TLI) { 1768d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng SmallVector<Instruction*, 16> AddrModeInsts; 1769d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1770d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (UsersToProcess[i].isUseOfPostIncrementedValue) 1771d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng continue; 1772d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng ExtAddrMode AddrMode = 1773d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AddressingModeMatcher::Match(UsersToProcess[i].OperandValToReplace, 1774d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng ReplacedTy, UsersToProcess[i].Inst, 1775d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AddrModeInsts, *TLI); 1776d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (GV && GV != AddrMode.BaseGV) 1777d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng return false; 1778d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (Offset && !AddrMode.BaseOffs) 1779d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // FIXME: How to accurate check it's immediate offset is folded. 1780d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng return false; 1781d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AddrModeInsts.clear(); 1782d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1783d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng return true; 1784d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng} 1785d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 17865f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// StrengthReduceStridedIVUsers - Strength reduce all of the users of a single 17875f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// stride of IV. All of the users may have different starting values, and this 17885f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// may not be the only stride (we know it is if isOnlyStride is true). 17895f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Chengvoid LoopStrengthReduce::StrengthReduceStridedIVUsers(const SCEVHandle &Stride, 17905f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 17915f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 17925f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool isOnlyStride) { 17935f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // If all the users are moved to another stride, then there is nothing to do. 1794303595942502f17c087fa28874c2b89117148c45Dan Gohman if (Uses.Users.empty()) 17955f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return; 17965f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 17975f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // Keep track if every use in UsersToProcess is an address. If they all are, 17985f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // we may be able to rewrite the entire collection of them in terms of a 17995f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // smaller-stride IV. 18005f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool AllUsesAreAddresses = true; 18015f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1802b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // Keep track if every use of a single stride is outside the loop. If so, 1803b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // we want to be more aggressive about reusing a smaller-stride IV; a 1804b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // multiply outside the loop is better than another IV inside. Well, usually. 1805b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool AllUsesAreOutsideLoop = true; 1806b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen 18075f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // Transform our list of users and offsets to a bit more complex table. In 18085f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // this new vector, each 'BasedUser' contains 'Base' the base of the 18095f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // strided accessas well as the old information from Uses. We progressively 18105f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // move information from the Base field to the Imm field, until we eventually 18115f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // have the full access expression to rewrite the use. 18125f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> UsersToProcess; 18135f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng SCEVHandle CommonExprs = CollectIVUsers(Stride, Uses, L, AllUsesAreAddresses, 1814b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen AllUsesAreOutsideLoop, 18155f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng UsersToProcess); 18165f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1817c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Sort the UsersToProcess array so that users with common bases are 1818c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // next to each other. 1819c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SortUsersToProcess(UsersToProcess); 1820c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 18215f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // If we managed to find some expressions in common, we'll need to carry 18225f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // their value in a register and add it in for each use. This will take up 18235f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // a register operand, which potentially restricts what stride values are 18245f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // valid. 1825cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman bool HaveCommonExprs = !CommonExprs->isZero(); 1826c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1827fe35555d09ecdad4b42da1138a1d40ea4b83898eChris Lattner const Type *ReplacedTy = CommonExprs->getType(); 1828c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1829d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // If all uses are addresses, consider sinking the immediate part of the 1830d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // common expression back into uses if they can fit in the immediate fields. 18313cd389de398e95b1633d310fc9c4af5b9f84b7afEvan Cheng if (TLI && HaveCommonExprs && AllUsesAreAddresses) { 1832d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng SCEVHandle NewCommon = CommonExprs; 1833d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng SCEVHandle Imm = SE->getIntegerSCEV(0, ReplacedTy); 1834d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng MoveImmediateValues(TLI, ReplacedTy, NewCommon, Imm, true, L, SE); 1835d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (!Imm->isZero()) { 1836d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng bool DoSink = true; 1837d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 1838d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // If the immediate part of the common expression is a GV, check if it's 1839d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // possible to fold it into the target addressing mode. 1840d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng GlobalValue *GV = 0; 1841d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (SCEVUnknown *SU = dyn_cast<SCEVUnknown>(Imm)) { 1842d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (ConstantExpr *CE = dyn_cast<ConstantExpr>(SU->getValue())) 1843d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (CE->getOpcode() == Instruction::PtrToInt) 1844d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng GV = dyn_cast<GlobalValue>(CE->getOperand(0)); 1845d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1846d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng int64_t Offset = 0; 1847d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (SCEVConstant *SC = dyn_cast<SCEVConstant>(Imm)) 1848d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng Offset = SC->getValue()->getSExtValue(); 1849d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (GV || Offset) 1850d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng DoSink = IsImmFoldedIntoAddrMode(GV, Offset, ReplacedTy, 1851d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng UsersToProcess, TLI); 1852d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 1853d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (DoSink) { 1854d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng DOUT << " Sinking " << *Imm << " back down into uses\n"; 1855d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) 1856d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng UsersToProcess[i].Imm = SE->getAddExpr(UsersToProcess[i].Imm, Imm); 1857d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng CommonExprs = NewCommon; 1858d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng HaveCommonExprs = !CommonExprs->isZero(); 1859d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng ++NumImmSunk; 1860d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1861d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1862d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1863d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 18641bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Now that we know what we need to do, insert the PHI node itself. 18651bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // 18662f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << "LSR: Examining IVs of TYPE " << *ReplacedTy << " of STRIDE " 18672f09f519542202b8af227c2a524f8fe82378a934Dan Gohman << *Stride << ":\n" 18682f09f519542202b8af227c2a524f8fe82378a934Dan Gohman << " Common base: " << *CommonExprs << "\n"; 1869d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 18701bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner SCEVExpander Rewriter(*SE, *LI); 18711bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner SCEVExpander PreheaderRewriter(*SE, *LI); 1872c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 18731bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner BasicBlock *Preheader = L->getLoopPreheader(); 18741bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Instruction *PreInsertPt = Preheader->getTerminator(); 187512b50410cd0a8cd81a7f528f862005e80921cf58Chris Lattner BasicBlock *LatchBlock = L->getLoopLatch(); 1876d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1877c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV = ConstantInt::get(ReplacedTy, 0); 1878eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 1879c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVHandle RewriteFactor = SE->getIntegerSCEV(0, ReplacedTy); 1880c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IVExpr ReuseIV(SE->getIntegerSCEV(0, Type::Int32Ty), 1881c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SE->getIntegerSCEV(0, Type::Int32Ty), 1882c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 0, 0); 18831bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1884c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// Choose a strength-reduction strategy and prepare for it by creating 1885c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// the necessary PHIs and adjusting the bookkeeping. 1886c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (ShouldUseFullStrengthReductionMode(UsersToProcess, L, 1887c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman AllUsesAreAddresses, Stride)) { 1888c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceFully(UsersToProcess, Stride, CommonExprs, L, 1889c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PreheaderRewriter); 1890c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } else { 1891c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Emit the initial base value into the loop preheader. 1892c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman CommonBaseV = PreheaderRewriter.expandCodeFor(CommonExprs, PreInsertPt); 1893c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1894c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If all uses are addresses, check if it is possible to reuse an IV with a 1895c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // stride that is a factor of this stride. And that the multiple is a number 1896c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // that can be encoded in the scale field of the target addressing mode. And 18976b38e29f13cb8700146e4b170567e2828553db9aDan Gohman // that we will have a valid instruction after this substition, including 18986b38e29f13cb8700146e4b170567e2828553db9aDan Gohman // the immediate field, if any. 1899c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman RewriteFactor = CheckForIVReuse(HaveCommonExprs, AllUsesAreAddresses, 1900c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman AllUsesAreOutsideLoop, 1901c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Stride, ReuseIV, CommonExprs->getType(), 1902c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess); 1903c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isa<SCEVConstant>(RewriteFactor) && 1904c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cast<SCEVConstant>(RewriteFactor)->isZero()) 1905c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceWithNewPhi(UsersToProcess, Stride, CommonExprs, 1906c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman CommonBaseV, L, PreheaderRewriter); 1907c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman else 1908c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceFromSmallerStride(UsersToProcess, CommonBaseV, 1909c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ReuseIV, PreInsertPt); 19107e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 19117e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1912c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Process all the users now, replacing their strided uses with 1913c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength-reduced forms. This outer loop handles all bases, the inner 19147e79b3898ddd919170d367a516f51296017146c2Chris Lattner // loop handles all users of a particular base. 1915169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman while (!UsersToProcess.empty()) { 19167b445c521bc191d0d25799b289e17b45f202a1afChris Lattner SCEVHandle Base = UsersToProcess.back().Base; 19172f09f519542202b8af227c2a524f8fe82378a934Dan Gohman Instruction *Inst = UsersToProcess.back().Inst; 1918be3e5212e23edc9210f774fac18d801de252e906Chris Lattner 19191bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Emit the code for Base into the preheader. 1920d19534add90a2a894af61523b830887097bb780bDan Gohman Value *BaseV = PreheaderRewriter.expandCodeFor(Base, PreInsertPt); 19217d8ed8a94118ace05962e88689a4089c2f49366dChris Lattner 19222f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Examining uses with BASE "; 19234a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, BaseV, /*PrintType=*/false)); 19242f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << ":\n"; 19257d8ed8a94118ace05962e88689a4089c2f49366dChris Lattner 19261bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If BaseV is a constant other than 0, make sure that it gets inserted into 19271bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // the preheader, instead of being forward substituted into the uses. We do 19283da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // this by forcing a BitCast (noop cast) to be inserted into the preheader 19293da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // in this case. 19307e79b3898ddd919170d367a516f51296017146c2Chris Lattner if (Constant *C = dyn_cast<Constant>(BaseV)) { 1931203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!C->isNullValue() && !fitsInAddressMode(Base, ReplacedTy, 1932203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen TLI, false)) { 19333da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // We want this constant emitted into the preheader! This is just 19343da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // using cast as a copy so BitCast (no-op cast) is appropriate 19353da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer BaseV = new BitCastInst(BaseV, BaseV->getType(), "preheaderinsert", 19364a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman PreInsertPt); 19371bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner } 19387e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 19397e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1940169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Emit the code to add the immediate offset to the Phi value, just before 19412351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner // the instructions that we identified as using this stride and base. 19427b445c521bc191d0d25799b289e17b45f202a1afChris Lattner do { 19437e79b3898ddd919170d367a516f51296017146c2Chris Lattner // FIXME: Use emitted users to emit other users. 19447b445c521bc191d0d25799b289e17b45f202a1afChris Lattner BasedUser &User = UsersToProcess.back(); 19452351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 19462f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " Examining use "; 19474a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman DEBUG(WriteAsOperand(*DOUT, UsersToProcess.back().OperandValToReplace, 19484a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman /*PrintType=*/false)); 19492f09f519542202b8af227c2a524f8fe82378a934Dan Gohman DOUT << " in Inst: " << *Inst; 19502f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 1951010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // If this instruction wants to use the post-incremented value, move it 1952010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // after the post-inc and use its value instead of the PHI. 1953c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *RewriteOp = User.Phi; 1954010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (User.isUseOfPostIncrementedValue) { 1955c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman RewriteOp = User.IncV; 1956c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner 1957c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // If this user is in the loop, make sure it is the last thing in the 1958c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // loop to ensure it is dominated by the increment. 1959c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner if (L->contains(User.Inst->getParent())) 1960c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner User.Inst->moveBefore(LatchBlock->getTerminator()); 1961010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 19623ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer if (RewriteOp->getType() != ReplacedTy) { 19633ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer Instruction::CastOps opcode = Instruction::Trunc; 19643ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer if (ReplacedTy->getPrimitiveSizeInBits() == 19653ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer RewriteOp->getType()->getPrimitiveSizeInBits()) 19663ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer opcode = Instruction::BitCast; 19673ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer RewriteOp = SCEVExpander::InsertCastOfTo(opcode, RewriteOp, ReplacedTy); 19683ba68b9eef2851dae8a9d1b18928c6fa2e3c5f87Reid Spencer } 196986c75d31133319a88216c1b1cd26a789e4023000Evan Cheng 1970246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SCEVHandle RewriteExpr = SE->getUnknown(RewriteOp); 19711bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 1972b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // If we had to insert new instructions for RewriteOp, we have to 1973f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // consider that they may not have been able to end up immediately 1974f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // next to RewriteOp, because non-PHI instructions may never precede 1975f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // PHI instructions in a block. In this case, remember where the last 1976ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // instruction was inserted so that if we're replacing a different 1977ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // PHI node, we can use the later point to expand the final 1978ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // RewriteExpr. 1979f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *NewBasePt = dyn_cast<Instruction>(RewriteOp); 1980c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (RewriteOp == User.Phi) NewBasePt = 0; 1981f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman 19821bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Clear the SCEVExpander's expression map so that we are guaranteed 19831bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // to have the code emitted where we expect it. 19841bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Rewriter.clear(); 1985d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1986d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng // If we are reusing the iv, then it must be multiplied by a constant 19871de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // factor to take advantage of the addressing mode scale component. 19882f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!isa<SCEVConstant>(RewriteFactor) || 19892f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !cast<SCEVConstant>(RewriteFactor)->isZero()) { 19902f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If we're reusing an IV with a nonzero base (currently this happens 19912f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // only when all reuses are outside the loop) subtract that base here. 19922f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // The base has been used to initialize the PHI node but we don't want 19932f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // it here. 19941de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (!ReuseIV.Base->isZero()) { 19951de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen SCEVHandle typedBase = ReuseIV.Base; 19961de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (RewriteExpr->getType()->getPrimitiveSizeInBits() != 19971de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen ReuseIV.Base->getType()->getPrimitiveSizeInBits()) { 19981de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // It's possible the original IV is a larger type than the new IV, 19991de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // in which case we have to truncate the Base. We checked in 20001de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // RequiresTypeConversion that this is valid. 20011de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen assert (RewriteExpr->getType()->getPrimitiveSizeInBits() < 20021de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen ReuseIV.Base->getType()->getPrimitiveSizeInBits() && 20031de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen "Unexpected lengthening conversion!"); 20041de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen typedBase = SE->getTruncateExpr(ReuseIV.Base, 20051de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen RewriteExpr->getType()); 20061de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen } 20071de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen RewriteExpr = SE->getMinusSCEV(RewriteExpr, typedBase); 20081de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen } 20092f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 20102f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Multiply old variable, with base removed, by new scale factor. 20112f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getMulExpr(RewriteFactor, 20128392772727ed9105c92fe4514d53dab74c333edcEvan Cheng RewriteExpr); 2013eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 2014eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // The common base is emitted in the loop preheader. But since we 2015eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // are reusing an IV, it has not been used to initialize the PHI node. 2016eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // Add it to the expression used to rewrite the uses. 20172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // When this use is outside the loop, we earlier subtracted the 20182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // common base, and are adding it back here. Use the same expression 20192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // as before, rather than CommonBaseV, so DAGCombiner will zap it. 2020eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng if (!isa<ConstantInt>(CommonBaseV) || 20212f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen !cast<ConstantInt>(CommonBaseV)->isZero()) { 20222f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(User.Inst->getParent())) 20232f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getAddExpr(RewriteExpr, 2024b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen SE->getUnknown(CommonBaseV)); 20252f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen else 20262f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getAddExpr(RewriteExpr, CommonExprs); 20272f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 2028eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 2029d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 20301bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Now that we know what we need to do, insert code before User for the 20311bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // immediate and any loop-variant expressions. 2032bee0f663d842f5aa41286c22815446e2d22de95aReid Spencer if (!isa<ConstantInt>(BaseV) || !cast<ConstantInt>(BaseV)->isZero()) 20331bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Add BaseV to the PHI value if needed. 2034246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman RewriteExpr = SE->getAddExpr(RewriteExpr, SE->getUnknown(BaseV)); 2035d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 2036f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman User.RewriteInstructionToUseNewBase(RewriteExpr, NewBasePt, 2037f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Rewriter, L, this, 20380e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng DeadInsts); 20392351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 2040a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner // Mark old value we replaced as possibly dead, so that it is eliminated 20412351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner // if we just replaced the last use of that value. 204209fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(cast<Instruction>(User.OperandValToReplace)); 20432351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 20447b445c521bc191d0d25799b289e17b45f202a1afChris Lattner UsersToProcess.pop_back(); 20452351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner ++NumReduced; 20467b445c521bc191d0d25799b289e17b45f202a1afChris Lattner 20477e79b3898ddd919170d367a516f51296017146c2Chris Lattner // If there are any more users to process with the same base, process them 20487e79b3898ddd919170d367a516f51296017146c2Chris Lattner // now. We sorted by base above, so we just have to check the last elt. 20497b445c521bc191d0d25799b289e17b45f202a1afChris Lattner } while (!UsersToProcess.empty() && UsersToProcess.back().Base == Base); 2050169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // TODO: Next, find out which base index is the most common, pull it out. 2051169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 2052169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 2053169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // IMPORTANT TODO: Figure out how to partition the IV's with this stride, but 2054169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // different starting values, into different PHIs. 2055eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 2056eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2057c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel/// FindIVUserForCond - If Cond has an operand that is an expression of an IV, 2058aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner/// set the IV user and stride information and return true, otherwise return 2059aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner/// false. 2060c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patelbool LoopStrengthReduce::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse, 2061aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner const SCEVHandle *&CondStride) { 2062aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner for (unsigned Stride = 0, e = StrideOrder.size(); Stride != e && !CondUse; 2063aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner ++Stride) { 2064aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 2065aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner IVUsesByStride.find(StrideOrder[Stride]); 2066aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner assert(SI != IVUsesByStride.end() && "Stride doesn't exist!"); 2067aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 2068aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner for (std::vector<IVStrideUse>::iterator UI = SI->second.Users.begin(), 2069aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner E = SI->second.Users.end(); UI != E; ++UI) 2070aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner if (UI->User == Cond) { 2071aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // NOTE: we could handle setcc instructions with multiple uses here, but 2072aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // InstCombine does it as well for simple uses, it's not clear that it 2073aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // occurs enough in real life to handle. 2074aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner CondUse = &*UI; 2075aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner CondStride = &SI->first; 2076aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return true; 2077aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner } 2078aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner } 2079aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return false; 2080aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner} 2081aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 2082cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Chengnamespace { 2083cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Constant strides come first which in turns are sorted by their absolute 2084cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // values. If absolute values are the same, then positive strides comes first. 2085cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // e.g. 2086cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // 4, -1, X, 1, 2 ==> 1, -1, 2, 4, X 2087cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng struct StrideCompare { 2088cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng bool operator()(const SCEVHandle &LHS, const SCEVHandle &RHS) { 2089cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS); 2090cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS); 2091cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (LHSC && RHSC) { 2092cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t LV = LHSC->getValue()->getSExtValue(); 2093cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t RV = RHSC->getValue()->getSExtValue(); 2094cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng uint64_t ALV = (LV < 0) ? -LV : LV; 2095cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng uint64_t ARV = (RV < 0) ? -RV : RV; 2096bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman if (ALV == ARV) { 2097bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman if (LV != RV) 2098bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return LV > RV; 2099bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman } else { 2100cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return ALV < ARV; 2101bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman } 2102bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman 2103bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // If it's the same value but different type, sort by bit width so 2104bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // that we emit larger induction variables before smaller 2105bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // ones, letting the smaller be re-written in terms of larger ones. 2106bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return RHS->getBitWidth() < LHS->getBitWidth(); 2107cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2108bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return LHSC && !RHSC; 2109cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2110cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng }; 2111cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 2112cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2113cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ChangeCompareStride - If a loop termination compare instruction is the 2114cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// only use of its stride, and the compaison is against a constant value, 2115cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// try eliminate the stride by moving the compare instruction to another 2116cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// stride and change its constant operand accordingly. e.g. 2117cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// 2118cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// loop: 2119cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ... 2120cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v1 = v1 + 3 2121cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v2 = v2 + 1 2122cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// if (v2 < 10) goto loop 2123cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// => 2124cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// loop: 2125cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ... 2126cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v1 = v1 + 3 2127cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// if (v1 < 30) goto loop 2128cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengICmpInst *LoopStrengthReduce::ChangeCompareStride(Loop *L, ICmpInst *Cond, 21290e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng IVStrideUse* &CondUse, 2130cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng const SCEVHandle* &CondStride) { 2131cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (StrideOrder.size() < 2 || 2132cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVUsesByStride[*CondStride].Users.size() != 1) 2133cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return Cond; 2134cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng const SCEVConstant *SC = dyn_cast<SCEVConstant>(*CondStride); 2135cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (!SC) return Cond; 2136cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2137cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst::Predicate Predicate = Cond->getPredicate(); 2138cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t CmpSSInt = SC->getValue()->getSExtValue(); 2139c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman unsigned BitWidth = (*CondStride)->getBitWidth(); 2140168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng uint64_t SignBit = 1ULL << (BitWidth-1); 2141c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman const Type *CmpTy = Cond->getOperand(0)->getType(); 2142168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng const Type *NewCmpTy = NULL; 2143af62c09437675369afcc481e88ed9cd5806491feEvan Cheng unsigned TyBits = CmpTy->getPrimitiveSizeInBits(); 2144af62c09437675369afcc481e88ed9cd5806491feEvan Cheng unsigned NewTyBits = 0; 2145cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng SCEVHandle *NewStride = NULL; 2146ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman Value *NewCmpLHS = NULL; 2147ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman Value *NewCmpRHS = NULL; 2148cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t Scale = 1; 2149c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman SCEVHandle NewOffset = SE->getIntegerSCEV(0, UIntPtrTy); 2150c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman std::stable_sort(StrideOrder.begin(), StrideOrder.end(), StrideCompare()); 2151cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2152c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (ConstantInt *C = dyn_cast<ConstantInt>(Cond->getOperand(1))) { 2153c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman int64_t CmpVal = C->getValue().getSExtValue(); 2154d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel 2155c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Check stride constant and the comparision constant signs to detect 2156c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // overflow. 2157c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if ((CmpVal & SignBit) != (CmpSSInt & SignBit)) 2158c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman return Cond; 2159cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2160c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Look for a suitable stride / iv as replacement. 2161c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman for (unsigned i = 0, e = StrideOrder.size(); i != e; ++i) { 2162c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 2163c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman IVUsesByStride.find(StrideOrder[i]); 2164c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (!isa<SCEVConstant>(SI->first)) 2165c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2166c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 2167c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (abs(SSInt) <= abs(CmpSSInt) || (SSInt % CmpSSInt) != 0) 2168c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2169168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2170c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman Scale = SSInt / CmpSSInt; 2171c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman int64_t NewCmpVal = CmpVal * Scale; 2172c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman APInt Mul = APInt(BitWidth, NewCmpVal); 2173c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Check for overflow. 2174c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (Mul.getSExtValue() != NewCmpVal) 2175c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2176cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2177c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Watch out for overflow. 2178c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (ICmpInst::isSignedPredicate(Predicate) && 2179c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman (CmpVal & SignBit) != (NewCmpVal & SignBit)) 2180c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2181af62c09437675369afcc481e88ed9cd5806491feEvan Cheng 2182c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (NewCmpVal == CmpVal) 2183c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2184c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Pick the best iv to use trying to avoid a cast. 2185c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpLHS = NULL; 2186c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman for (std::vector<IVStrideUse>::iterator UI = SI->second.Users.begin(), 2187c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman E = SI->second.Users.end(); UI != E; ++UI) { 2188c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpLHS = UI->OperandValToReplace; 2189c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (NewCmpLHS->getType() == CmpTy) 2190c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman break; 21915f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng } 2192c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (!NewCmpLHS) 2193cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng continue; 2194cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2195c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpTy = NewCmpLHS->getType(); 2196c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewTyBits = isa<PointerType>(NewCmpTy) 2197c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman ? UIntPtrTy->getPrimitiveSizeInBits() 2198c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman : NewCmpTy->getPrimitiveSizeInBits(); 2199c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (RequiresTypeConversion(NewCmpTy, CmpTy)) { 2200c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Check if it is possible to rewrite it using 2201c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // an iv / stride of a smaller integer type. 2202c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman bool TruncOk = false; 2203c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (NewCmpTy->isInteger()) { 2204c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman unsigned Bits = NewTyBits; 2205c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (ICmpInst::isSignedPredicate(Predicate)) 2206c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman --Bits; 2207c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman uint64_t Mask = (1ULL << Bits) - 1; 2208c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (((uint64_t)NewCmpVal & Mask) == (uint64_t)NewCmpVal) 2209c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman TruncOk = true; 2210c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman } 2211c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (!TruncOk) 2212c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2213c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman } 2214ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman 2215c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Don't rewrite if use offset is non-constant and the new type is 2216c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // of a different type. 2217c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // FIXME: too conservative? 2218c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (NewTyBits != TyBits && !isa<SCEVConstant>(CondUse->Offset)) 2219c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2220cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2221c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman bool AllUsesAreAddresses = true; 2222c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman bool AllUsesAreOutsideLoop = true; 2223c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman std::vector<BasedUser> UsersToProcess; 2224c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman SCEVHandle CommonExprs = CollectIVUsers(SI->first, SI->second, L, 2225c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman AllUsesAreAddresses, 2226c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman AllUsesAreOutsideLoop, 2227c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman UsersToProcess); 2228c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Avoid rewriting the compare instruction with an iv of new stride 2229c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // if it's likely the new stride uses will be rewritten using the 2230c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // stride of the compare instruction. 2231c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (AllUsesAreAddresses && 2232c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman ValidStride(!CommonExprs->isZero(), Scale, UsersToProcess)) 2233c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2234c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman 2235c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // If scale is negative, use swapped predicate unless it's testing 2236c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // for equality. 2237c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (Scale < 0 && !Cond->isEquality()) 2238c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman Predicate = ICmpInst::getSwappedPredicate(Predicate); 2239c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman 2240c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewStride = &StrideOrder[i]; 2241c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (!isa<PointerType>(NewCmpTy)) 2242c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpRHS = ConstantInt::get(NewCmpTy, NewCmpVal); 2243c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman else { 2244c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpRHS = ConstantInt::get(UIntPtrTy, NewCmpVal); 2245c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpRHS = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr, 2246c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpRHS, NewCmpTy); 2247c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman } 2248c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewOffset = TyBits == NewTyBits 2249c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman ? SE->getMulExpr(CondUse->Offset, 2250c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman SE->getConstant(ConstantInt::get(CmpTy, Scale))) 2251c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman : SE->getConstant(ConstantInt::get(NewCmpTy, 2252c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman cast<SCEVConstant>(CondUse->Offset)->getValue()->getSExtValue()*Scale)); 2253c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman break; 2254cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2255cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2256cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 22579b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // Forgo this transformation if it the increment happens to be 22589b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // unfortunately positioned after the condition, and the condition 22599b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // has multiple uses which prevent it from being moved immediately 22609b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // before the branch. See 22619b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // test/Transforms/LoopStrengthReduce/change-compare-stride-trickiness-*.ll 22629b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // for an example of this situation. 2263d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel if (!Cond->hasOneUse()) { 22649b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman for (BasicBlock::iterator I = Cond, E = Cond->getParent()->end(); 22659b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman I != E; ++I) 2266ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman if (I == NewCmpLHS) 22679b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman return Cond; 2268d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel } 22699b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman 2270ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman if (NewCmpRHS) { 2271cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Create a new compare instruction using new stride / iv. 2272cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst *OldCond = Cond; 2273168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng // Insert new compare instruction. 2274ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman Cond = new ICmpInst(Predicate, NewCmpLHS, NewCmpRHS, 2275e562b1725ee068ff525082d1e9ba885c8928c72eDan Gohman L->getHeader()->getName() + ".termcond", 2276e562b1725ee068ff525082d1e9ba885c8928c72eDan Gohman OldCond); 2277168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2278168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng // Remove the old compare instruction. The old indvar is probably dead too. 227909fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(cast<Instruction>(CondUse->OperandValToReplace)); 2280168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SE->deleteValueFromRecords(OldCond); 2281010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman OldCond->replaceAllUsesWith(Cond); 2282cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng OldCond->eraseFromParent(); 2283168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2284cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng IVUsesByStride[*CondStride].Users.pop_back(); 2285ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman IVUsesByStride[*NewStride].addUser(NewOffset, Cond, NewCmpLHS); 2286cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng CondUse = &IVUsesByStride[*NewStride].Users.back(); 2287cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng CondStride = NewStride; 2288cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ++NumEliminated; 2289cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2290cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2291cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return Cond; 2292cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 2293cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2294ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// OptimizeSMax - Rewrite the loop's terminating condition if it uses 2295ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// an smax computation. 2296ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2297ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// This is a narrow solution to a specific, but acute, problem. For loops 2298ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// like this: 2299ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2300ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2301ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2302ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2303ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i < n); 2304ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2305ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// where the comparison is signed, the trip count isn't just 'n', because 2306ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 'n' could be negative. And unfortunately this can come up even for loops 2307ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// where the user didn't use a C do-while loop. For example, seemingly 2308ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// well-behaved top-test loops will commonly be lowered like this: 2309ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman// 2310ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// if (n > 0) { 2311ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2312ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2313ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2314ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i < n); 2315ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } 2316ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2317ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// and then it's possible for subsequent optimization to obscure the if 2318ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// test in such a way that indvars can't find it. 2319ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2320ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// When indvars can't find the if test in loops like this, it creates a 2321ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// signed-max expression, which allows it to give the loop a canonical 2322ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// induction variable: 2323ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2324ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2325ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// smax = n < 1 ? 1 : n; 2326ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2327ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2328ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i != smax); 2329ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2330ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// Canonical induction variables are necessary because the loop passes 2331ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// are designed around them. The most obvious example of this is the 2332ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// LoopInfo analysis, which doesn't remember trip count values. It 2333ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// expects to be able to rediscover the trip count each time it is 2334ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// needed, and it does this using a simple analyis that only succeeds if 2335ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// the loop has a canonical induction variable. 2336ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2337ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// However, when it comes time to generate code, the maximum operation 2338ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// can be quite costly, especially if it's inside of an outer loop. 2339ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2340ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// This function solves this problem by detecting this type of loop and 2341ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting 2342ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// the instructions for the maximum computation. 2343ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2344ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan GohmanICmpInst *LoopStrengthReduce::OptimizeSMax(Loop *L, ICmpInst *Cond, 2345ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman IVStrideUse* &CondUse) { 2346ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check that the loop matches the pattern we're looking for. 2347ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (Cond->getPredicate() != CmpInst::ICMP_EQ && 2348ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->getPredicate() != CmpInst::ICMP_NE) 2349ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 2350ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2351ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); 2352ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!Sel || !Sel->hasOneUse()) return Cond; 2353ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 235446bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman SCEVHandle BackedgeTakenCount = SE->getBackedgeTakenCount(L); 235546bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 2356ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 235746bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman SCEVHandle One = SE->getIntegerSCEV(1, BackedgeTakenCount->getType()); 2358ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 235946bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman // Add one to the backedge-taken count to get the trip count. 236046bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman SCEVHandle IterationCount = SE->getAddExpr(BackedgeTakenCount, One); 2361ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2362ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check for a max calculation that matches the pattern. 2363ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(IterationCount); 2364ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!SMax || SMax != SE->getSCEV(Sel)) return Cond; 2365ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2366ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle SMaxLHS = SMax->getOperand(0); 2367ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle SMaxRHS = SMax->getOperand(1); 2368ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!SMaxLHS || SMaxLHS != One) return Cond; 2369ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2370ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check the relevant induction variable for conformance to 2371ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // the pattern. 2372ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVHandle IV = SE->getSCEV(Cond->getOperand(0)); 2373ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 2374ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!AR || !AR->isAffine() || 2375ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman AR->getStart() != One || 2376ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman AR->getStepRecurrence(*SE) != One) 2377ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 2378ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2379bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman assert(AR->getLoop() == L && 2380bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman "Loop condition operand is an addrec in a different loop!"); 2381bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman 2382ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check the right operand of the select, and remember it, as it will 2383ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // be used in the new comparison instruction. 2384ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Value *NewRHS = 0; 2385ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (SE->getSCEV(Sel->getOperand(1)) == SMaxRHS) 2386ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman NewRHS = Sel->getOperand(1); 2387ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman else if (SE->getSCEV(Sel->getOperand(2)) == SMaxRHS) 2388ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman NewRHS = Sel->getOperand(2); 2389ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!NewRHS) return Cond; 2390ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2391ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Ok, everything looks ok to change the condition into an SLT or SGE and 2392ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // delete the max calculation. 2393ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman ICmpInst *NewCond = 2394ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman new ICmpInst(Cond->getPredicate() == CmpInst::ICMP_NE ? 2395ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman CmpInst::ICMP_SLT : 2396ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman CmpInst::ICMP_SGE, 2397ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->getOperand(0), NewRHS, "scmp", Cond); 2398ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2399ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Delete the max calculation instructions. 2400586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman SE->deleteValueFromRecords(Cond); 2401ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->replaceAllUsesWith(NewCond); 2402ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->eraseFromParent(); 2403ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); 2404ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SE->deleteValueFromRecords(Sel); 2405586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman Sel->eraseFromParent(); 2406ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (Cmp->use_empty()) { 2407ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SE->deleteValueFromRecords(Cmp); 2408586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman Cmp->eraseFromParent(); 2409ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman } 2410ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman CondUse->User = NewCond; 2411ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return NewCond; 2412ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman} 2413ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2414a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel/// OptimizeShadowIV - If IV is used in a int-to-float cast 2415a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel/// inside the loop then try to eliminate the cast opeation. 2416a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patelvoid LoopStrengthReduce::OptimizeShadowIV(Loop *L) { 2417a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 241846bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman SCEVHandle BackedgeTakenCount = SE->getBackedgeTakenCount(L); 241946bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 2420a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel return; 2421a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2422a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned Stride = 0, e = StrideOrder.size(); Stride != e; 2423a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ++Stride) { 2424a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 2425a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel IVUsesByStride.find(StrideOrder[Stride]); 2426a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel assert(SI != IVUsesByStride.end() && "Stride doesn't exist!"); 2427a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!isa<SCEVConstant>(SI->first)) 2428a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2429a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2430a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (std::vector<IVStrideUse>::iterator UI = SI->second.Users.begin(), 2431a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel E = SI->second.Users.end(); UI != E; /* empty */) { 2432a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel std::vector<IVStrideUse>::iterator CandidateUI = UI; 2433541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel ++UI; 2434a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Instruction *ShadowUse = CandidateUI->User; 2435a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const Type *DestTy = NULL; 2436a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2437a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* If shadow use is a int->float cast then insert a second IV 2438541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel to eliminate this cast. 2439a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2440a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned i = 0; i < n; ++i) 2441a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel foo((double)i); 2442a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2443541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel is transformed into 2444a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2445a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel double d = 0.0; 2446a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned i = 0; i < n; ++i, ++d) 2447a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel foo(d); 2448a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel */ 2449541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->User)) 2450a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel DestTy = UCast->getDestTy(); 2451541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->User)) 2452a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel DestTy = SCast->getDestTy(); 245318bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (!DestTy) continue; 245418bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel 245518bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (TLI) { 245618bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel /* If target does not support DestTy natively then do not apply 245718bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel this transformation. */ 245818bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel MVT DVT = TLI->getValueType(DestTy); 245918bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (!TLI->isTypeLegal(DVT)) continue; 246018bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel } 246118bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel 2462a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); 2463a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!PH) continue; 2464a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (PH->getNumIncomingValues() != 2) continue; 2465a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2466a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const Type *SrcTy = PH->getType(); 2467a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel int Mantissa = DestTy->getFPMantissaWidth(); 2468a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Mantissa == -1) continue; 2469a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if ((int)TD->getTypeSizeInBits(SrcTy) > Mantissa) 2470a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2471a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2472a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel unsigned Entry, Latch; 2473a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { 2474a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Entry = 0; 2475a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Latch = 1; 2476a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } else { 2477a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Entry = 1; 2478a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Latch = 0; 2479a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2480a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2481a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); 2482a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!Init) continue; 2483a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantFP *NewInit = ConstantFP::get(DestTy, Init->getZExtValue()); 2484a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2485a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator *Incr = 2486a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); 2487a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!Incr) continue; 2488a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Incr->getOpcode() != Instruction::Add 2489a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel && Incr->getOpcode() != Instruction::Sub) 2490a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2491a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2492a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Initialize new IV, double d = 0.0 in above example. */ 2493a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantInt *C = NULL; 2494a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Incr->getOperand(0) == PH) 2495a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel C = dyn_cast<ConstantInt>(Incr->getOperand(1)); 2496a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel else if (Incr->getOperand(1) == PH) 2497a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel C = dyn_cast<ConstantInt>(Incr->getOperand(0)); 2498a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel else 2499a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2500a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2501a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!C) continue; 2502a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2503a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Add new PHINode. */ 2504a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel PHINode *NewPH = PHINode::Create(DestTy, "IV.S.", PH); 2505a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2506541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel /* create new increment. '++d' in above example. */ 2507a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantFP *CFP = ConstantFP::get(DestTy, C->getZExtValue()); 2508a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator *NewIncr = 2509a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator::Create(Incr->getOpcode(), 2510a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH, CFP, "IV.S.next.", Incr); 2511a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2512a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); 2513a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); 2514a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2515a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Remove cast operation */ 2516a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel SE->deleteValueFromRecords(ShadowUse); 2517a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ShadowUse->replaceAllUsesWith(NewPH); 2518a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ShadowUse->eraseFromParent(); 2519a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel SI->second.Users.erase(CandidateUI); 2520a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NumShadow++; 2521a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel break; 2522a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2523a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2524a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel} 2525a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2526010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner// OptimizeIndvars - Now that IVUsesByStride is set up with all of the indvar 2527010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner// uses in the loop, look to see if we can eliminate some, in favor of using 2528010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner// common indvars for the different uses. 2529010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattnervoid LoopStrengthReduce::OptimizeIndvars(Loop *L) { 2530010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // TODO: implement optzns here. 2531010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2532a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel OptimizeShadowIV(L); 2533a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2534010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Finally, get the terminating condition for the loop if possible. If we 2535010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // can, we want to change it to use a post-incremented version of its 253698d9811db263ee040d9a6a69ef9e1c4fdc8c219dChris Lattner // induction variable, to allow coalescing the live ranges for the IV into 2537010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // one register value. 2538010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner PHINode *SomePHI = cast<PHINode>(L->getHeader()->begin()); 2539010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner BasicBlock *Preheader = L->getLoopPreheader(); 2540010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner BasicBlock *LatchBlock = 2541010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner SomePHI->getIncomingBlock(SomePHI->getIncomingBlock(0) == Preheader); 2542010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner BranchInst *TermBr = dyn_cast<BranchInst>(LatchBlock->getTerminator()); 2543e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer if (!TermBr || TermBr->isUnconditional() || 2544e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer !isa<ICmpInst>(TermBr->getCondition())) 2545010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner return; 2546e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); 2547010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2548010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Search IVUsesByStride to find Cond's IVUse if there is one. 2549010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner IVStrideUse *CondUse = 0; 255050fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner const SCEVHandle *CondStride = 0; 2551010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2552c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel if (!FindIVUserForCond(Cond, CondUse, CondStride)) 2553aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return; // setcc doesn't use the IV. 2554cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2555ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // If the trip count is computed in terms of an smax (due to ScalarEvolution 2556ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // being unable to find a sufficient guard, for example), change the loop 2557ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // comparison to use SLT instead of NE. 2558ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond = OptimizeSMax(L, Cond, CondUse); 2559ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2560cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // If possible, change stride and operands of the compare instruction to 2561cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // eliminate one stride. 2562cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng Cond = ChangeCompareStride(L, Cond, CondUse, CondStride); 2563010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2564010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // It's possible for the setcc instruction to be anywhere in the loop, and 2565010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // possible for it to have multiple users. If it is not immediately before 2566010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // the latch block branch, move it. 2567010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (&*++BasicBlock::iterator(Cond) != (Instruction*)TermBr) { 2568010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (Cond->hasOneUse()) { // Condition has a single use, just move it. 2569010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner Cond->moveBefore(TermBr); 2570010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } else { 2571010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Otherwise, clone the terminating condition and insert into the loopend. 2572e4d87aa2de6e52952dca73716386db09aad5a8fdReid Spencer Cond = cast<ICmpInst>(Cond->clone()); 2573010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner Cond->setName(L->getHeader()->getName() + ".termcond"); 2574010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner LatchBlock->getInstList().insert(TermBr, Cond); 2575010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2576010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Clone the IVUse, as the old use still exists! 257750fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner IVUsesByStride[*CondStride].addUser(CondUse->Offset, Cond, 2578010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner CondUse->OperandValToReplace); 257950fad70279fa982fcd6a72c919f4e18ad99829b9Chris Lattner CondUse = &IVUsesByStride[*CondStride].Users.back(); 2580010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2581010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2582010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2583010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // If we get to here, we know that we can transform the setcc instruction to 258498d9811db263ee040d9a6a69ef9e1c4fdc8c219dChris Lattner // use the post-incremented version of the IV, allowing us to coalesce the 2585010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // live ranges for the IV correctly. 2586246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman CondUse->Offset = SE->getMinusSCEV(CondUse->Offset, *CondStride); 2587010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner CondUse->isUseOfPostIncrementedValue = true; 25881ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng Changed = true; 2589010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner} 2590169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 25910f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patelbool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager &LPM) { 2592eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 25930f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel LI = &getAnalysis<LoopInfo>(); 2594b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel DT = &getAnalysis<DominatorTree>(); 25950f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel SE = &getAnalysis<ScalarEvolution>(); 25960f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel TD = &getAnalysis<TargetData>(); 25970f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel UIntPtrTy = TD->getIntPtrType(); 25983fea643fb427e72907b7a40e940d7949d1d57e76Dan Gohman Changed = false; 25990f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel 2600b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // Find all uses of induction variables in this loop, and categorize 2601169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // them by stride. Start by finding all of the PHI nodes in the header for 2602169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // this loop. If they are induction variables, inspect their uses. 2603168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng SmallPtrSet<Instruction*,16> Processed; // Don't reprocess instructions. 2604169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) 26053416e5f645186a7e3321f927eab662d0ecef404bChris Lattner AddUsersIfInteresting(I, L, Processed); 2606169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 26071ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng if (!IVUsesByStride.empty()) { 26081ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Optimize induction variables. Some indvar uses can be transformed to use 26091ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // strides that will be needed for other purposes. A common example of this 26101ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // is the exit test for the loop, which can often be rewritten to use the 26111ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // computation of some other indvar to decide when to terminate the loop. 26121ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng OptimizeIndvars(L); 2613010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 26141ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // FIXME: We can widen subreg IV's here for RISC targets. e.g. instead of 26151ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // doing computation in byte values, promote to 32-bit values if safe. 2616010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 26171ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // FIXME: Attempt to reuse values across multiple IV's. In particular, we 26181ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // could have something like "for(i) { foo(i*8); bar(i*16) }", which should 26191ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // be codegened as "for (j = 0;; j+=8) { foo(j); bar(j+j); }" on X86/PPC. 26201ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Need to be careful that IV's are all the same type. Only works for 26211ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // intptr_t indvars. 2622169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 26231ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // If we only have one stride, we can more aggressively eliminate some 26241ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // things. 26251ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng bool HasOneStride = IVUsesByStride.size() == 1; 2626d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 2627d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng#ifndef NDEBUG 2628d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng DOUT << "\nLSR on \"" << L->getHeader()->getParent()->getNameStart() 2629d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng << "\" "; 26301ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng DEBUG(L->dump()); 2631d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng#endif 2632d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 26331ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // IVsByStride keeps IVs for one particular loop. 26341ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng assert(IVsByStride.empty() && "Stale entries in IVsByStride?"); 26351ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 26361ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Sort the StrideOrder so we process larger strides first. 26371ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng std::stable_sort(StrideOrder.begin(), StrideOrder.end(), StrideCompare()); 26381ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 26391ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Note: this processes each stride/type pair individually. All users 26401ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // passed into StrengthReduceStridedIVUsers have the same type AND stride. 26411ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Also, note that we iterate over IVUsesByStride indirectly by using 26421ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // StrideOrder. This extra layer of indirection makes the ordering of 26431ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // strides deterministic - not dependent on map order. 26441ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng for (unsigned Stride = 0, e = StrideOrder.size(); Stride != e; ++Stride) { 26451ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng std::map<SCEVHandle, IVUsersOfOneStride>::iterator SI = 26461ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng IVUsesByStride.find(StrideOrder[Stride]); 26471ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng assert(SI != IVUsesByStride.end() && "Stride doesn't exist!"); 26481ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng StrengthReduceStridedIVUsers(SI->first, SI->second, L, HasOneStride); 26491ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng } 26507305ae28df07e0b9c9a8a020a9f9596de25077aaChris Lattner } 2651eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2652010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman // We're done analyzing this loop; release all the state we built up for it. 2653010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman CastedPointers.clear(); 2654010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman IVUsesByStride.clear(); 2655010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman IVsByStride.clear(); 2656010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman StrideOrder.clear(); 26572f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned i=0; i<GEPlist.size(); i++) 26582f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SE->deleteValueFromRecords(GEPlist[i]); 26592f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen GEPlist.clear(); 2660010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman 2661eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman // Clean up after ourselves 2662eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman if (!DeadInsts.empty()) { 2663a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeleteTriviallyDeadInstructions(); 2664eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2665169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman BasicBlock::iterator I = L->getHeader()->begin(); 2666cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman while (PHINode *PN = dyn_cast<PHINode>(I++)) { 2667cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // At this point, we know that we have killed one or more IV users. 2668bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner // It is worth checking to see if the cannonical indvar is also 2669cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // dead, so that we can remove it as well. 2670cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // 2671cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // We can remove a PHI if it is on a cycle in the def-use graph 2672cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // where each node in the cycle has degree one, i.e. only one use, 2673cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // and is an instruction with no side effects. 2674cbfe5bbe88f5f2ee03a388585112f7609c8151adDan Gohman // 2675169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // FIXME: this needs to eliminate an induction variable even if it's being 2676169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // compared against some value to decide loop termination. 2677a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner if (!PN->hasOneUse()) 2678a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner continue; 2679a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner 2680a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner SmallPtrSet<PHINode *, 4> PHIs; 2681a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner for (Instruction *J = dyn_cast<Instruction>(*PN->use_begin()); 2682a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner J && J->hasOneUse() && !J->mayWriteToMemory(); 2683a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner J = dyn_cast<Instruction>(*J->use_begin())) { 2684a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // If we find the original PHI, we've discovered a cycle. 2685a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner if (J == PN) { 2686a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // Break the cycle and mark the PHI for deletion. 2687a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner SE->deleteValueFromRecords(PN); 2688a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner PN->replaceAllUsesWith(UndefValue::get(PN->getType())); 268909fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(PN); 2690a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner Changed = true; 2691a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner break; 26927e608bbb5dfe4f827e64e91b0bb68a1d95d737aeChris Lattner } 2693a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // If we find a PHI more than once, we're on a cycle that 2694a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner // won't prove fruitful. 2695a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner if (isa<PHINode>(J) && !PHIs.insert(cast<PHINode>(J))) 2696a0d4486073f6303fbf33b50ac3fbae4601ea0229Chris Lattner break; 2697169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 2698eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 2699a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeleteTriviallyDeadInstructions(); 2700eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 27011ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng return Changed; 2702eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 2703