LoopStrengthReduce.cpp revision 586f69a11881d828c056ce017b3fb432341d9657
12d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// The LLVM Compiler Infrastructure 4eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 54ee451de366474b9c228b4e5fa573795a715216dChris Lattner// This file is distributed under the University of Illinois Open Source 64ee451de366474b9c228b4e5fa573795a715216dChris Lattner// License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 9eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 10cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// This transformation analyzes and transforms the induction variables (and 11cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// computations derived from them) into forms suitable for efficient execution 12cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// on the target. 13cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// 14eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// This pass performs a strength reduction on array references inside loops that 15cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// have as one or more of their components the loop induction variable, it 16cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// rewrites expressions to take advantage of scaled-index addressing modes 17cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// available on the target, and it performs a variety of other optimizations 18cec8f9da54d3017e27c46ced38b859136655c198Dan Gohman// related to loop induction variables. 19eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman// 20eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman//===----------------------------------------------------------------------===// 21eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 22be3e5212e23edc9210f774fac18d801de252e906Chris Lattner#define DEBUG_TYPE "loop-reduce" 23eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Scalar.h" 24eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Constants.h" 25eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Instructions.h" 26e5b01bea7b9b7dce7c24484d2d915b0c118d4d07Dan Gohman#include "llvm/IntrinsicInst.h" 27eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Type.h" 282f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen#include "llvm/DerivedTypes.h" 29eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Analysis/Dominators.h" 3081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman#include "llvm/Analysis/IVUsers.h" 31eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Analysis/LoopInfo.h" 320f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel#include "llvm/Analysis/LoopPass.h" 33169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Analysis/ScalarEvolutionExpander.h" 34d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng#include "llvm/Transforms/Utils/AddrModeMatcher.h" 35e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner#include "llvm/Transforms/Utils/BasicBlockUtils.h" 36eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/Transforms/Utils/Local.h" 37eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman#include "llvm/ADT/Statistic.h" 38d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng#include "llvm/Support/CFG.h" 39169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman#include "llvm/Support/Debug.h" 40c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman#include "llvm/Support/CommandLine.h" 41afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman#include "llvm/Support/ValueHandle.h" 42460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar#include "llvm/Support/raw_ostream.h" 43d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng#include "llvm/Target/TargetLowering.h" 44cfb1d4235fe3291028341e6acf4139723b4749e3Jeff Cohen#include <algorithm> 45eaa13851a7fe604363577350c5cf65c257c4d41aNate Begemanusing namespace llvm; 46eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 4713317bc1e90efaed0c2e6da794f313aeadb226ebDan GohmanSTATISTIC(NumReduced , "Number of IV uses strength reduced"); 48cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumInserted, "Number of PHIs inserted"); 49cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengSTATISTIC(NumVariable, "Number of PHIs with variable strides"); 50541532724e29203e28c2fe0136cf6eabd49d4532Devang PatelSTATISTIC(NumEliminated, "Number of strides eliminated"); 51541532724e29203e28c2fe0136cf6eabd49d4532Devang PatelSTATISTIC(NumShadow, "Number of Shadow IVs optimized"); 52d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan ChengSTATISTIC(NumImmSunk, "Number of common expr immediates sunk into uses"); 535792f51e12d9c8685399e9857799365854ab5bf6Evan ChengSTATISTIC(NumLoopCond, "Number of loop terminating conds optimized"); 5481ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan ChengSTATISTIC(NumCountZero, "Number of count iv optimized to count toward zero"); 55eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 56c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic cl::opt<bool> EnableFullLSRMode("enable-full-lsr", 57c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cl::init(false), 58c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman cl::Hidden); 59c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 600e5f499638c8d277b9dc4a4385712177c53b5681Chris Lattnernamespace { 61dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 62c01a53007a4f4f9a601f1cc83ff4e2935405b905Jeff Cohen struct BasedUser; 63dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen 64d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVInfo - This structure keeps track of one IV expression inserted during 6521495775e710d37003e100862cdc647cbdc3b999Evan Cheng /// StrengthReduceStridedIVUsers. It contains the stride, the common base, as 6621495775e710d37003e100862cdc647cbdc3b999Evan Cheng /// well as the PHI node and increment value created for rewrite. 673e8b6631e67e01e4960a7ba4668a50c596607473Chris Lattner struct IVExpr { 680bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Stride; 690bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Base; 70d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng PHINode *PHI; 71d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 720bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman IVExpr(const SCEV *const stride, const SCEV *const base, PHINode *phi) 739d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman : Stride(stride), Base(base), PHI(phi) {} 74d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng }; 75d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 76d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVsOfOneStride - This structure keeps track of all IV expression inserted 77d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// during StrengthReduceStridedIVUsers for a particular stride of the IV. 783e8b6631e67e01e4960a7ba4668a50c596607473Chris Lattner struct IVsOfOneStride { 79d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng std::vector<IVExpr> IVs; 80d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 810bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman void addIV(const SCEV *const Stride, const SCEV *const Base, PHINode *PHI) { 829d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman IVs.push_back(IVExpr(Stride, Base, PHI)); 83d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng } 84d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng }; 85169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 863e8b6631e67e01e4960a7ba4668a50c596607473Chris Lattner class LoopStrengthReduce : public LoopPass { 8781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IVUsers *IU; 88eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman LoopInfo *LI; 89b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel DominatorTree *DT; 90169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman ScalarEvolution *SE; 91eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman bool Changed; 927e608bbb5dfe4f827e64e91b0bb68a1d95d737aeChris Lattner 93d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// IVsByStride - Keep track of all IVs that have been inserted for a 94d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng /// particular stride. 950bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVsOfOneStride> IVsByStride; 96d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 975792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng /// StrideNoReuse - Keep track of all the strides whose ivs cannot be 985792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng /// reused (nor should they be rewritten to reuse other strides). 990bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallSet<const SCEV *, 4> StrideNoReuse; 1005792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 101169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// DeadInsts - Keep track of instructions we may have made dead, so that 102169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// we can remove them after we are done working. 10381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman SmallVector<WeakVH, 16> DeadInsts; 104d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng 105d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng /// TLI - Keep a pointer of a TargetLowering to consult for determining 106d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng /// transformation profitability. 107d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng const TargetLowering *TLI; 108d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng 109eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman public: 1101997473cf72957d0e70322e2fe6fe2ab141c58a6Devang Patel static char ID; // Pass ID, replacement for typeid 111c2bbfc18e9adbbdcf5b3375d8d25e2452f7df7f1Dan Gohman explicit LoopStrengthReduce(const TargetLowering *tli = NULL) : 112ae73dc1448d25b02cabc7c64c86c64371453dda8Dan Gohman LoopPass(&ID), TLI(tli) { 1132f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen } 1142f3c9b7562bcdc1795b2bd0ca28b283a8e972826Jeff Cohen 1150f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel bool runOnLoop(Loop *L, LPPassManager &LPM); 116eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 117eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman virtual void getAnalysisUsage(AnalysisUsage &AU) const { 118aa96ae780afa5475e62df284855a971216289212Chris Lattner // We split critical edges, so we change the CFG. However, we do update 119aa96ae780afa5475e62df284855a971216289212Chris Lattner // many analyses if they are around. 120aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreservedID(LoopSimplifyID); 121aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<LoopInfo>(); 122aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<DominanceFrontier>(); 123aa96ae780afa5475e62df284855a971216289212Chris Lattner AU.addPreserved<DominatorTree>(); 124aa96ae780afa5475e62df284855a971216289212Chris Lattner 125f465db6c6a5a877aa791abfc3837d62c491dacd5Jeff Cohen AU.addRequiredID(LoopSimplifyID); 126eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman AU.addRequired<LoopInfo>(); 127b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel AU.addRequired<DominatorTree>(); 128169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman AU.addRequired<ScalarEvolution>(); 129a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel AU.addPreserved<ScalarEvolution>(); 13081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman AU.addRequired<IVUsers>(); 13181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman AU.addPreserved<IVUsers>(); 132eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 1332d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 1343d81e317244ecac40d1977221a4a521589b869f5Dan Gohman private: 135010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner void OptimizeIndvars(Loop *L); 136586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 137586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// OptimizeLoopTermCond - Change loop terminating condition to use the 138586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// postinc iv when possible. 1392d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng void OptimizeLoopTermCond(Loop *L); 1402d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng 141a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /// OptimizeShadowIV - If IV is used in a int-to-float cast 142a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /// inside the loop then try to eliminate the cast opeation. 143a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel void OptimizeShadowIV(Loop *L); 144a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 1452781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman /// OptimizeMax - Rewrite the loop's terminating condition 1462781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman /// if it uses a max computation. 1472781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman ICmpInst *OptimizeMax(Loop *L, ICmpInst *Cond, 1482781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman IVStrideUse* &CondUse); 149ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 150586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// OptimizeLoopCountIV - If, after all sharing of IVs, the IV used for 151586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// deciding when to exit the loop is used only for that purpose, try to 152586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// rearrange things so it counts down to a test against zero. 153586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool OptimizeLoopCountIV(Loop *L); 154586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool OptimizeLoopCountIVOfStride(const SCEV* &Stride, 155586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVStrideUse* &CondUse, Loop *L); 156586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 157586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// StrengthReduceIVUsersOfStride - Strength reduce all of the users of a 158586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// single stride of IV. All of the users may have different starting 159586f69a11881d828c056ce017b3fb432341d9657Evan Cheng /// values, and this may not be the only stride. 160586f69a11881d828c056ce017b3fb432341d9657Evan Cheng void StrengthReduceIVUsersOfStride(const SCEV *const &Stride, 161586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVUsersOfOneStride &Uses, 162586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Loop *L); 163586f69a11881d828c056ce017b3fb432341d9657Evan Cheng void StrengthReduceIVUsers(Loop *L); 164586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 165586f69a11881d828c056ce017b3fb432341d9657Evan Cheng ICmpInst *ChangeCompareStride(Loop *L, ICmpInst *Cond, 166586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVStrideUse* &CondUse, const SCEV* &CondStride, 167586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool PostPass = false); 168586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 169c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse, 170586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV* &CondStride); 1715f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool RequiresTypeConversion(const Type *Ty, const Type *NewTy); 1720bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CheckForIVReuse(bool, bool, bool, const SCEV *const&, 17302e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman IVExpr&, const Type*, 174dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess); 1755792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng bool ValidScale(bool, int64_t, 1765792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng const std::vector<BasedUser>& UsersToProcess); 17781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman bool ValidOffset(bool, int64_t, int64_t, 17881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman const std::vector<BasedUser>& UsersToProcess); 1790bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CollectIVUsers(const SCEV *const &Stride, 1805f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 1815f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 1825f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool &AllUsesAreAddresses, 183b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool &AllUsesAreOutsideLoop, 1845f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> &UsersToProcess); 185586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool StrideMightBeShared(const SCEV *Stride, Loop *L, bool CheckPreInc); 186c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool ShouldUseFullStrengthReductionMode( 187c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const std::vector<BasedUser> &UsersToProcess, 188c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 189c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool AllUsesAreAddresses, 1900bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Stride); 191c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceFully( 192c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1930bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Stride, 1940bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CommonExprs, 195c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 196c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter); 197c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceFromSmallerStride( 198c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 199c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 200c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const IVExpr &ReuseIV, 201c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Instruction *PreInsertPt); 202c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman void PrepareToStrengthReduceWithNewPhi( 203c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 2040bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Stride, 2050bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CommonExprs, 206c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 2075792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng Instruction *IVIncInsertPt, 208c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 209c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter); 210586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 211a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner void DeleteTriviallyDeadInstructions(); 212eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman }; 213eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 214eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 215844731a7f1909f55935e3514c9e713a62d67662eDan Gohmanchar LoopStrengthReduce::ID = 0; 216844731a7f1909f55935e3514c9e713a62d67662eDan Gohmanstatic RegisterPass<LoopStrengthReduce> 217844731a7f1909f55935e3514c9e713a62d67662eDan GohmanX("loop-reduce", "Loop Strength Reduction"); 218844731a7f1909f55935e3514c9e713a62d67662eDan Gohman 219394f0441e06dafca29f0752cf400990a5b8fe4b1Daniel DunbarPass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) { 220d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng return new LoopStrengthReduce(TLI); 221eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 222eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 223eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// DeleteTriviallyDeadInstructions - If any of the instructions is the 224eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// specified set are trivially dead, delete them and see if this makes any of 225eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman/// their operands subsequently dead. 226a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattnervoid LoopStrengthReduce::DeleteTriviallyDeadInstructions() { 22709fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (DeadInsts.empty()) return; 22809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 229a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner while (!DeadInsts.empty()) { 23081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman Instruction *I = dyn_cast_or_null<Instruction>(DeadInsts.back()); 231a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeadInsts.pop_back(); 23209fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner 23309fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (I == 0 || !isInstructionTriviallyDead(I)) 234bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner continue; 235bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner 23609fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) { 23709fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner if (Instruction *U = dyn_cast<Instruction>(*OI)) { 23809fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner *OI = 0; 239bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner if (U->use_empty()) 24009fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(U); 241bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner } 242eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 243bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner 244bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner I->eraseFromParent(); 245bfcee36cd747bf9f791ba7aa3e9e8ac3671c6822Chris Lattner Changed = true; 246eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman } 247eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 248eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2492f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// containsAddRecFromDifferentLoop - Determine whether expression S involves a 2502f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// subexpression that is an AddRec from a loop other than L. An outer loop 2512f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// of L is OK, but not an inner loop nor a disjoint loop. 2520bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanstatic bool containsAddRecFromDifferentLoop(const SCEV *S, Loop *L) { 2532f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // This is very common, put it first. 2542f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (isa<SCEVConstant>(S)) 2552f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 256890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVCommutativeExpr *AE = dyn_cast<SCEVCommutativeExpr>(S)) { 2572f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (unsigned int i=0; i< AE->getNumOperands(); i++) 2582f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (containsAddRecFromDifferentLoop(AE->getOperand(i), L)) 2592f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return true; 2602f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 2612f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 262890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVAddRecExpr *AE = dyn_cast<SCEVAddRecExpr>(S)) { 2632f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (const Loop *newLoop = AE->getLoop()) { 2642f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (newLoop == L) 2652f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 2662f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // if newLoop is an outer loop of L, this is OK. 267c8d76d5afb023a1c6b439941be3b62789fcc0ed3Dan Gohman if (!LoopInfo::isNotAlreadyContainedIn(L, newLoop)) 2682f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 2692f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 2702f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return true; 2712f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 272890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVUDivExpr *DE = dyn_cast<SCEVUDivExpr>(S)) 2732f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(DE->getLHS(), L) || 2742f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen containsAddRecFromDifferentLoop(DE->getRHS(), L); 2752f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen#if 0 2762f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // SCEVSDivExpr has been backed out temporarily, but will be back; we'll 2772f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // need this when it is. 278890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVSDivExpr *DE = dyn_cast<SCEVSDivExpr>(S)) 2792f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return containsAddRecFromDifferentLoop(DE->getLHS(), L) || 2802f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen containsAddRecFromDifferentLoop(DE->getRHS(), L); 2812f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen#endif 28284923602fdc2a81957e5dee178d5737ad8e72f55Dan Gohman if (const SCEVCastExpr *CE = dyn_cast<SCEVCastExpr>(S)) 28384923602fdc2a81957e5dee178d5737ad8e72f55Dan Gohman return containsAddRecFromDifferentLoop(CE->getOperand(), L); 2842f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return false; 2852f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen} 2862f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 287f284ce203b656f1d6f933b67ad096a93d3cd7290Dan Gohman/// isAddressUse - Returns true if the specified instruction is using the 288203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// specified value as an address. 289203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstatic bool isAddressUse(Instruction *Inst, Value *OperandVal) { 290203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen bool isAddress = isa<LoadInst>(Inst); 291203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 292203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (SI->getOperand(1) == OperandVal) 293203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen isAddress = true; 294203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 295203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Addressing modes can also be folded into prefetches and a variety 296203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // of intrinsics. 297203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen switch (II->getIntrinsicID()) { 298203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen default: break; 299203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::prefetch: 300203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_loadu_dq: 301203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_loadu_pd: 302203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse_loadu_ps: 303203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse_storeu_ps: 304203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storeu_pd: 305203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storeu_dq: 306203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen case Intrinsic::x86_sse2_storel_dq: 307203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (II->getOperand(1) == OperandVal) 308203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen isAddress = true; 309203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen break; 310203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 311203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 312203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen return isAddress; 313203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen} 3140ae33eb243417982fbdca792460bdd986108ac09Chris Lattner 31521e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman/// getAccessType - Return the type of the memory being accessed. 31621e7722868378f67974e648ab21d0e3c69a0e379Dan Gohmanstatic const Type *getAccessType(const Instruction *Inst) { 317a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman const Type *AccessTy = Inst->getType(); 31821e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) 319a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman AccessTy = SI->getOperand(0)->getType(); 32021e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { 32121e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman // Addressing modes can also be folded into prefetches and a variety 32221e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman // of intrinsics. 32321e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman switch (II->getIntrinsicID()) { 32421e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman default: break; 32521e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman case Intrinsic::x86_sse_storeu_ps: 32621e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman case Intrinsic::x86_sse2_storeu_pd: 32721e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman case Intrinsic::x86_sse2_storeu_dq: 32821e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman case Intrinsic::x86_sse2_storel_dq: 329a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman AccessTy = II->getOperand(1)->getType(); 33021e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman break; 33121e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman } 33221e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman } 333a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman return AccessTy; 33421e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman} 33521e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman 336169974856781a1ce27af9ce6220c390b20c9e6ddNate Begemannamespace { 337169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// BasedUser - For a particular base value, keep information about how we've 338169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// partitioned the expression so far. 339169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman struct BasedUser { 340246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman /// SE - The current ScalarEvolution object. 341246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE; 342246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman 343a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// Base - The Base value for the PHI node that needs to be inserted for 344a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// this use. As the use is processed, information gets moved from this 345a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// field to the Imm field (below). BasedUser values are sorted by this 346a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner /// field. 3470bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Base; 348a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 349169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Inst - The instruction using the induction variable. 350169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman Instruction *Inst; 351169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 352ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// OperandValToReplace - The operand value of Inst to replace with the 353ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner /// EmittedBase. 354ec3fb63af27b6a20f4a9ee58bb63baad5640ea9cChris Lattner Value *OperandValToReplace; 355169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 356169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// Imm - The immediate value that should be added to the base immediately 357169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman /// before Inst, because it will be folded into the imm field of the 35833e3a36f0a116b758708ef8fd0695d3a4cb21531Dan Gohman /// instruction. This is also sometimes used for loop-variant values that 35933e3a36f0a116b758708ef8fd0695d3a4cb21531Dan Gohman /// must be added inside the loop. 3600bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Imm; 361169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 362c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// Phi - The induction variable that performs the striding that 363c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman /// should be used for this user. 3649d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman PHINode *Phi; 365c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 366010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // isUseOfPostIncrementedValue - True if this should use the 367010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // post-incremented version of this IV, not the preincremented version. 368010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // This can only be set in special cases, such as the terminating setcc 369c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // instruction for a loop and uses outside the loop that are dominated by 370c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // the loop. 371010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner bool isUseOfPostIncrementedValue; 372a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner 373246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman BasedUser(IVStrideUse &IVSU, ScalarEvolution *se) 37481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman : SE(se), Base(IVSU.getOffset()), Inst(IVSU.getUser()), 37581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman OperandValToReplace(IVSU.getOperandValToReplace()), 376308f24d4525a6365f8d65ba821786189c080c0ceDale Johannesen Imm(SE->getIntegerSCEV(0, Base->getType())), 37781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman isUseOfPostIncrementedValue(IVSU.isUseOfPostIncrementedValue()) {} 378169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 3792114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Once we rewrite the code to insert the new IVs we want, update the 3802114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // operands of Inst to use the new expression 'NewBase', with 'Imm' added 3812114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // to it. 3820bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman void RewriteInstructionToUseNewBase(const SCEV *const &NewBase, 383f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *InsertPt, 3840e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng SCEVExpander &Rewriter, Loop *L, Pass *P, 3855be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman LoopInfo &LI, 38681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman SmallVectorImpl<WeakVH> &DeadInsts); 387221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 3880bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman Value *InsertCodeForBaseAtPosition(const SCEV *const &NewBase, 3892d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman const Type *Ty, 390221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVExpander &Rewriter, 3915be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman Instruction *IP, Loop *L, 3925be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman LoopInfo &LI); 393169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman void dump() const; 394169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman }; 395169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 396169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 397169974856781a1ce27af9ce6220c390b20c9e6ddNate Begemanvoid BasedUser::dump() const { 398bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner errs() << " Base=" << *Base; 399bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner errs() << " Imm=" << *Imm; 400bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner errs() << " Inst: " << *Inst; 401169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 402169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 4030bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan GohmanValue *BasedUser::InsertCodeForBaseAtPosition(const SCEV *const &NewBase, 4042d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman const Type *Ty, 405221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SCEVExpander &Rewriter, 4065be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman Instruction *IP, Loop *L, 4075be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman LoopInfo &LI) { 408221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Figure out where we *really* want to insert this code. In particular, if 409221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // the user is inside of a loop that is nested inside of L, we really don't 410221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // want to insert this expression before the user, we'd rather pull it out as 411221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // many loops as possible. 412221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Instruction *BaseInsertPt = IP; 413221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 414221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Figure out the most-nested loop that IP is in. 415221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner Loop *InsertLoop = LI.getLoopFor(IP->getParent()); 416221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 417221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If InsertLoop is not L, and InsertLoop is nested inside of L, figure out 418221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // the preheader of the outer-most loop where NewBase is not loop invariant. 419eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen if (L->contains(IP->getParent())) 420eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen while (InsertLoop && NewBase->isLoopInvariant(InsertLoop)) { 421eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen BaseInsertPt = InsertLoop->getLoopPreheader()->getTerminator(); 422eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen InsertLoop = InsertLoop->getParentLoop(); 423eccdd08d4c1a81ede221bbccf3045fe6f11e1003Dale Johannesen } 424221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 4255be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman Value *Base = Rewriter.expandCodeFor(NewBase, 0, BaseInsertPt); 42681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 4270bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *NewValSCEV = SE->getUnknown(Base); 4282f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 42940a5a1b39ee1cd40ff9d04740386b667fb27b340Dan Gohman // Always emit the immediate into the same block as the user. 43040a5a1b39ee1cd40ff9d04740386b667fb27b340Dan Gohman NewValSCEV = SE->getAddExpr(NewValSCEV, Imm); 43181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 4322d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman return Rewriter.expandCodeFor(NewValSCEV, Ty, IP); 433221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner} 434221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 435221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 4362114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner// Once we rewrite the code to insert the new IVs we want, update the 4372114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner// operands of Inst to use the new expression 'NewBase', with 'Imm' added 438f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// to it. NewBasePt is the last instruction which contributes to the 439f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// value of NewBase in the case that it's a diffferent instruction from 440f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// the PHI that NewBase is computed from, or null otherwise. 441f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman// 4420bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanvoid BasedUser::RewriteInstructionToUseNewBase(const SCEV *const &NewBase, 443f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *NewBasePt, 4440e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng SCEVExpander &Rewriter, Loop *L, Pass *P, 4455be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman LoopInfo &LI, 44681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman SmallVectorImpl<WeakVH> &DeadInsts) { 4472114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner if (!isa<PHINode>(Inst)) { 448c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // By default, insert code at the user instruction. 449c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner BasicBlock::iterator InsertPt = Inst; 450c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner 451c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // However, if the Operand is itself an instruction, the (potentially 452c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // complex) inserted code may be shared by many users. Because of this, we 453c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // want to emit code for the computation of the operand right before its old 454c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // computation. This is usually safe, because we obviously used to use the 455c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // computation when it was computed in its current block. However, in some 456c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // cases (e.g. use of a post-incremented induction variable) the NewBase 457c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // value will be pinned to live somewhere after the original computation. 458c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner // In this case, we have to back off. 459f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // 460f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If this is a use outside the loop (which means after, since it is based 461f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // on a loop indvar) we use the post-incremented value, so that we don't 462f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // artificially make the preinc value live out the bottom of the loop. 463589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (!isUseOfPostIncrementedValue && L->contains(Inst->getParent())) { 464ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman if (NewBasePt && isa<PHINode>(OperandValToReplace)) { 465f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman InsertPt = NewBasePt; 466f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman ++InsertPt; 4676725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif } else if (Instruction *OpInst 4686725cb5f1c1b59cbd71dc221623c7b4cabafafd0Gabor Greif = dyn_cast<Instruction>(OperandValToReplace)) { 469c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner InsertPt = OpInst; 470c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner while (isa<PHINode>(InsertPt)) ++InsertPt; 471c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner } 472c5494af8a90f398046c45bc2b7549ab9004c01d9Chris Lattner } 4732d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman Value *NewVal = InsertCodeForBaseAtPosition(NewBase, 4742d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman OperandValToReplace->getType(), 4755be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman Rewriter, InsertPt, L, LI); 4762114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Replace the use of the operand Value with the new Phi we just created. 4772114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner Inst->replaceUsesOfWith(OperandValToReplace, NewVal); 4782f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 479bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Replacing with "); 480bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(WriteAsOperand(errs(), NewVal, /*PrintType=*/false)); 481bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << ", which has value " << *NewBase << " plus IMM " 482bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner << *Imm << "\n"); 4832114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner return; 4842114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 4852f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 4862114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // PHI nodes are more complex. We have to insert one copy of the NewBase+Imm 487c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // expression into each operand block that uses it. Note that PHI nodes can 488c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // have multiple entries for the same predecessor. We use a map to make sure 489c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // that a PHI node only has a single Value* for each predecessor (which also 490c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // prevents us from inserting duplicate code in some blocks). 4918392772727ed9105c92fe4514d53dab74c333edcEvan Cheng DenseMap<BasicBlock*, Value*> InsertedCode; 4922114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner PHINode *PN = cast<PHINode>(Inst); 4932114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 4942114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner if (PN->getIncomingValue(i) == OperandValToReplace) { 4952f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If the original expression is outside the loop, put the replacement 4962f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // code in the same place as the original expression, 4972f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // which need not be an immediate predecessor of this PHI. This way we 4982f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // need only one copy of it even if it is referenced multiple times in 4992f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // the PHI. We don't do this when the original expression is inside the 5001de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // loop because multiple copies sometimes do useful sinking of code in 5011de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // that case(?). 5022f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen Instruction *OldLoc = dyn_cast<Instruction>(OperandValToReplace); 5035c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman BasicBlock *PHIPred = PN->getIncomingBlock(i); 5042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(OldLoc->getParent())) { 5051de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // If this is a critical edge, split the edge so that we do not insert 5061de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // the code on all predecessor/successor paths. We do this unless this 5071de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // is the canonical backedge for this loop, as this can make some 5081de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // inserted code be in an illegal position. 5092f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (e != 1 && PHIPred->getTerminator()->getNumSuccessors() > 1 && 510eb3567cd49b6767fb9b505591245f81b4341b9f2Dan Gohman !isa<IndirectBrInst>(PHIPred->getTerminator()) && 5112f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen (PN->getParent() != L->getHeader() || !L->contains(PHIPred))) { 5122f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 5132f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // First step, split the critical edge. 5145c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman BasicBlock *NewBB = SplitCriticalEdge(PHIPred, PN->getParent(), 5155c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman P, false); 5162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 5172f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Next step: move the basic block. In particular, if the PHI node 5182f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // is outside of the loop, and PredTI is in the loop, we want to 5192f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // move the block to be immediately before the PHI block, not 5202f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // immediately after PredTI. 5215c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman if (L->contains(PHIPred) && !L->contains(PN->getParent())) 5222f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen NewBB->moveBefore(PN->getParent()); 5232f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 5242f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Splitting the edge can reduce the number of PHI entries we have. 5252f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen e = PN->getNumIncomingValues(); 5265c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman PHIPred = NewBB; 5275c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman i = PN->getBasicBlockIndex(PHIPred); 528e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner } 529e0391beda88c6c441ce1aadbe223d6c0784061a2Chris Lattner } 5305c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman Value *&Code = InsertedCode[PHIPred]; 531c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner if (!Code) { 532c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner // Insert the code into the end of the predecessor block. 5332f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen Instruction *InsertPt = (L->contains(OldLoc->getParent())) ? 5345c89b5240c90eb8171f999e5f06f815502d0321cDan Gohman PHIPred->getTerminator() : 5352f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen OldLoc->getParent()->getTerminator(); 5362d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman Code = InsertCodeForBaseAtPosition(NewBase, PN->getType(), 5375be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman Rewriter, InsertPt, L, LI); 5382f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 539bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Changing PHI use to "); 540bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(WriteAsOperand(errs(), Code, /*PrintType=*/false)); 541bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << ", which has value " << *NewBase << " plus IMM " 542bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner << *Imm << "\n"); 543c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner } 5442f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 5452114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner // Replace the use of the operand Value with the new Phi we just created. 546c41e34520ab2f8c0bfe2f95546745826f6b34d59Chris Lattner PN->setIncomingValue(i, Code); 5472114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner Rewriter.clear(); 5482114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 5492114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner } 5500e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng 5510e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng // PHI node might have become a constant value after SplitCriticalEdge. 55209fb7dadf1f8f2efaae6a803c63fb29d06105df3Chris Lattner DeadInsts.push_back(Inst); 5532114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner} 5542114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner 5552114b273ef0520cec99c253ef6ddf717eaa7657aChris Lattner 556203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// fitsInAddressMode - Return true if V can be subsumed within an addressing 557203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// mode, and does not need to be put in a register first. 5580bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanstatic bool fitsInAddressMode(const SCEV *const &V, const Type *AccessTy, 559203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const TargetLowering *TLI, bool HasBaseReg) { 560890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(V)) { 5615eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t VC = SC->getValue()->getSExtValue(); 562579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner if (TLI) { 563579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 564579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseOffs = VC; 565203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen AM.HasBaseReg = HasBaseReg; 566a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman return TLI->isLegalAddressingMode(AM, AccessTy); 567579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } else { 568d277f2c66914aecb619c12855f6afae4c7ef883bEvan Cheng // Defaults to PPC. PPC allows a sign-extended 16-bit immediate field. 5695eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng return (VC > -(1 << 16) && VC < (1 << 16)-1); 570579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner } 5713821e478a574b80d7f8bc96fa42731291cfccfe8Chris Lattner } 572d29b6aa608d69f19b57ebd2ae630b040b1c4951dJeff Cohen 573890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) 5742d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman if (GlobalValue *GV = dyn_cast<GlobalValue>(SU->getValue())) { 575cc2ad058b378673e8cccba34b99857f8f72ff0e6Dan Gohman if (TLI) { 576cc2ad058b378673e8cccba34b99857f8f72ff0e6Dan Gohman TargetLowering::AddrMode AM; 577cc2ad058b378673e8cccba34b99857f8f72ff0e6Dan Gohman AM.BaseGV = GV; 578cc2ad058b378673e8cccba34b99857f8f72ff0e6Dan Gohman AM.HasBaseReg = HasBaseReg; 579a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman return TLI->isLegalAddressingMode(AM, AccessTy); 580cc2ad058b378673e8cccba34b99857f8f72ff0e6Dan Gohman } else { 581cc2ad058b378673e8cccba34b99857f8f72ff0e6Dan Gohman // Default: assume global addresses are not legal. 582cc2ad058b378673e8cccba34b99857f8f72ff0e6Dan Gohman } 5832d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman } 5842d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 585169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman return false; 586169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 587169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 588544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen/// MoveLoopVariantsToImmediateField - Move any subexpressions from Val that are 58944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner/// loop varying to the Imm operand. 5900bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanstatic void MoveLoopVariantsToImmediateField(const SCEV *&Val, const SCEV *&Imm, 5915792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng Loop *L, ScalarEvolution *SE) { 59244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (Val->isLoopInvariant(L)) return; // Nothing to do. 59344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 594890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVAddExpr *SAE = dyn_cast<SCEVAddExpr>(Val)) { 5950bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallVector<const SCEV *, 4> NewOps; 59644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner NewOps.reserve(SAE->getNumOperands()); 59744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 59844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner for (unsigned i = 0; i != SAE->getNumOperands(); ++i) 59944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (!SAE->getOperand(i)->isLoopInvariant(L)) { 60044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // If this is a loop-variant expression, it must stay in the immediate 60144b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // field of the expression. 602246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, SAE->getOperand(i)); 60344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else { 60444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner NewOps.push_back(SAE->getOperand(i)); 60544b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 60644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 60744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner if (NewOps.empty()) 608246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 60944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner else 610246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddExpr(NewOps); 611890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman } else if (const SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Val)) { 61244b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Try to pull immediates out of the start value of nested addrec's. 6130bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Start = SARE->getStart(); 614544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen MoveLoopVariantsToImmediateField(Start, Imm, L, SE); 61544b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 6160bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallVector<const SCEV *, 4> Ops(SARE->op_begin(), SARE->op_end()); 61744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner Ops[0] = Start; 618246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddRecExpr(Ops, SARE->getLoop()); 61944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } else { 62044b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Otherwise, all of Val is variant, move the whole thing over. 621246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, Val); 622246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 62344b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 62444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner} 62544b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 62644b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner 62726d91f16464db56283087176a73981048331dd2dChris Lattner/// MoveImmediateValues - Look at Val, and pull out any additions of constants 628169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman/// that can fit into the immediate field of instructions in the target. 62926d91f16464db56283087176a73981048331dd2dChris Lattner/// Accumulate these immediate values into the Imm value. 630d277f2c66914aecb619c12855f6afae4c7ef883bEvan Chengstatic void MoveImmediateValues(const TargetLowering *TLI, 631a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman const Type *AccessTy, 6320bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *&Val, const SCEV *&Imm, 633246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman bool isAddress, Loop *L, 634246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE) { 635890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVAddExpr *SAE = dyn_cast<SCEVAddExpr>(Val)) { 6360bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallVector<const SCEV *, 4> NewOps; 63726d91f16464db56283087176a73981048331dd2dChris Lattner NewOps.reserve(SAE->getNumOperands()); 63826d91f16464db56283087176a73981048331dd2dChris Lattner 639221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner for (unsigned i = 0; i != SAE->getNumOperands(); ++i) { 6400bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *NewOp = SAE->getOperand(i); 641a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman MoveImmediateValues(TLI, AccessTy, NewOp, Imm, isAddress, L, SE); 642221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 643221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner if (!NewOp->isLoopInvariant(L)) { 6447db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // If this is a loop-variant expression, it must stay in the immediate 6457db543f887c83aad2a95814a16d363a2313fc2a8Chris Lattner // field of the expression. 646246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, NewOp); 64726d91f16464db56283087176a73981048331dd2dChris Lattner } else { 648221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner NewOps.push_back(NewOp); 649169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 650221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 65126d91f16464db56283087176a73981048331dd2dChris Lattner 65226d91f16464db56283087176a73981048331dd2dChris Lattner if (NewOps.empty()) 653246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 65426d91f16464db56283087176a73981048331dd2dChris Lattner else 655246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddExpr(NewOps); 65626d91f16464db56283087176a73981048331dd2dChris Lattner return; 657890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman } else if (const SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Val)) { 6587a65839f4118c99fb1636c3cbb41b0bf7ef27665Chris Lattner // Try to pull immediates out of the start value of nested addrec's. 6590bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Start = SARE->getStart(); 660a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman MoveImmediateValues(TLI, AccessTy, Start, Imm, isAddress, L, SE); 66126d91f16464db56283087176a73981048331dd2dChris Lattner 66226d91f16464db56283087176a73981048331dd2dChris Lattner if (Start != SARE->getStart()) { 6630bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallVector<const SCEV *, 4> Ops(SARE->op_begin(), SARE->op_end()); 66426d91f16464db56283087176a73981048331dd2dChris Lattner Ops[0] = Start; 665246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getAddRecExpr(Ops, SARE->getLoop()); 66626d91f16464db56283087176a73981048331dd2dChris Lattner } 66726d91f16464db56283087176a73981048331dd2dChris Lattner return; 668890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman } else if (const SCEVMulExpr *SME = dyn_cast<SCEVMulExpr>(Val)) { 669221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Transform "8 * (4 + v)" -> "32 + 8*V" if "32" fits in the immed field. 670a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman if (isAddress && 671a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman fitsInAddressMode(SME->getOperand(0), AccessTy, TLI, false) && 672221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner SME->getNumOperands() == 2 && SME->isLoopInvariant(L)) { 673221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 6740bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *SubImm = SE->getIntegerSCEV(0, Val->getType()); 6750bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *NewOp = SME->getOperand(1); 676a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman MoveImmediateValues(TLI, AccessTy, NewOp, SubImm, isAddress, L, SE); 677221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 678221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // If we extracted something out of the subexpressions, see if we can 679221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // simplify this! 680221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner if (NewOp != SME->getOperand(1)) { 681221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Scale SubImm up by "8". If the result is a target constant, we are 682221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // good. 683246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SubImm = SE->getMulExpr(SubImm, SME->getOperand(0)); 684a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman if (fitsInAddressMode(SubImm, AccessTy, TLI, false)) { 685221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Accumulate the immediate. 686246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, SubImm); 687221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner 688221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner // Update what is left of 'Val'. 689246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getMulExpr(SME->getOperand(0), NewOp); 690221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner return; 691221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 692221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 693221fc3c6d69bd3854e9121f51e3283492c222ab7Chris Lattner } 694169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 695169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 69626d91f16464db56283087176a73981048331dd2dChris Lattner // Loop-variant expressions must stay in the immediate field of the 69726d91f16464db56283087176a73981048331dd2dChris Lattner // expression. 698a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman if ((isAddress && fitsInAddressMode(Val, AccessTy, TLI, false)) || 69926d91f16464db56283087176a73981048331dd2dChris Lattner !Val->isLoopInvariant(L)) { 700246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Imm = SE->getAddExpr(Imm, Val); 701246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Val = SE->getIntegerSCEV(0, Val->getType()); 70226d91f16464db56283087176a73981048331dd2dChris Lattner return; 7037a2ca56ef3bdda6874bcd4df2910fb5a33999f85Chris Lattner } 70426d91f16464db56283087176a73981048331dd2dChris Lattner 70526d91f16464db56283087176a73981048331dd2dChris Lattner // Otherwise, no immediates to move. 706169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman} 707169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 708d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Chengstatic void MoveImmediateValues(const TargetLowering *TLI, 709d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng Instruction *User, 7100bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *&Val, const SCEV *&Imm, 711d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng bool isAddress, Loop *L, 712d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng ScalarEvolution *SE) { 713a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman const Type *AccessTy = getAccessType(User); 714a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman MoveImmediateValues(TLI, AccessTy, Val, Imm, isAddress, L, SE); 715d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng} 716934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 7177e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// SeparateSubExprs - Decompose Expr into all of the subexpressions that are 7187e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// added together. This is used to reassociate common addition subexprs 7197e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// together for maximal sharing when rewriting bases. 7200bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanstatic void SeparateSubExprs(SmallVector<const SCEV *, 16> &SubExprs, 7210bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Expr, 722246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman ScalarEvolution *SE) { 723890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(Expr)) { 724934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned j = 0, e = AE->getNumOperands(); j != e; ++j) 725246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, AE->getOperand(j), SE); 726890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman } else if (const SCEVAddRecExpr *SARE = dyn_cast<SCEVAddRecExpr>(Expr)) { 7270bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Zero = SE->getIntegerSCEV(0, Expr->getType()); 728934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SARE->getOperand(0) == Zero) { 729934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.push_back(Expr); 730934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } else { 731934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Compute the addrec with zero as its base. 7320bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallVector<const SCEV *, 4> Ops(SARE->op_begin(), SARE->op_end()); 733934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner Ops[0] = Zero; // Start with zero base. 734246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SubExprs.push_back(SE->getAddRecExpr(Ops, SARE->getLoop())); 735934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 736934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 737246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, SARE->getOperand(0), SE); 738934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 739cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman } else if (!Expr->isZero()) { 740934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Do not add zero. 741934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.push_back(Expr); 742934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 743934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner} 744934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 745203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen// This is logically local to the following function, but C++ says we have 746203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen// to make it file scope. 747203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesenstruct SubExprUseData { unsigned Count; bool notAllUsesAreFree; }; 748934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 749203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// RemoveCommonExpressionsFromUseBases - Look through all of the Bases of all 750203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// the Uses, removing any common subexpressions, except that if all such 751203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// subexpressions can be folded into an addressing mode for all uses inside 752203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// the loop (this case is referred to as "free" in comments herein) we do 753203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen/// not remove anything. This looks for things like (a+b+c) and 754f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner/// (a+c+d) and computes the common (a+c) subexpression. The common expression 755f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner/// is *removed* from the Bases and returned. 7560bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanstatic const SCEV * 757246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan GohmanRemoveCommonExpressionsFromUseBases(std::vector<BasedUser> &Uses, 758203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen ScalarEvolution *SE, Loop *L, 759203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen const TargetLowering *TLI) { 7601bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner unsigned NumUses = Uses.size(); 7611bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 762f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Only one use? This is a very common case, so we handle it specially and 763f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // cheaply. 7640bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Zero = SE->getIntegerSCEV(0, Uses[0].Base->getType()); 7650bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Result = Zero; 7660bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *FreeResult = Zero; 7671bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner if (NumUses == 1) { 768f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If the use is inside the loop, use its base, regardless of what it is: 769f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // it is clearly shared across all the IV's. If the use is outside the loop 770f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // (which means after it) we don't want to factor anything *into* the loop, 771f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // so just use 0 as the base. 772589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (L->contains(Uses[0].Inst->getParent())) 773589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen std::swap(Result, Uses[0].Base); 7741bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner return Result; 7751bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner } 7761bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 7771bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // To find common subexpressions, count how many of Uses use each expression. 7781bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If any subexpressions are used Uses.size() times, they are common. 779203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Also track whether all uses of each expression can be moved into an 780203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // an addressing mode "for free"; such expressions are left within the loop. 781203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // struct SubExprUseData { unsigned Count; bool notAllUsesAreFree; }; 7820bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, SubExprUseData> SubExpressionUseData; 7831bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 784d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // UniqueSubExprs - Keep track of all of the subexpressions we see in the 785d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // order we see them. 7860bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallVector<const SCEV *, 16> UniqueSubExprs; 787d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner 7880bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman SmallVector<const SCEV *, 16> SubExprs; 789f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner unsigned NumUsesInsideLoop = 0; 790934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned i = 0; i != NumUses; ++i) { 791f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // If the user is outside the loop, just ignore it for base computation. 792f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Since the user is outside the loop, it must be *after* the loop (if it 793f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // were before, it could not be based on the loop IV). We don't want users 794f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // after the loop to affect base computation of values *inside* the loop, 795f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // because we can always add their offsets to the result IV after the loop 796f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // is done, ensuring we get good code inside the loop. 797589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen if (!L->contains(Uses[i].Inst->getParent())) 798589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen continue; 799589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen NumUsesInsideLoop++; 800589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 801934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // If the base is zero (which is common), return zero now, there are no 802934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // CSEs we can find. 803934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (Uses[i].Base == Zero) return Zero; 804934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 805203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // If this use is as an address we may be able to put CSEs in the addressing 806203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // mode rather than hoisting them. 807203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen bool isAddrUse = isAddressUse(Uses[i].Inst, Uses[i].OperandValToReplace); 808a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman // We may need the AccessTy below, but only when isAddrUse, so compute it 809203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // only in that case. 810a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman const Type *AccessTy = 0; 81121e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman if (isAddrUse) 812a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman AccessTy = getAccessType(Uses[i].Inst); 813203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 814934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Split the expression into subexprs. 815246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, Uses[i].Base, SE); 816203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Add one to SubExpressionUseData.Count for each subexpr present, and 817203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // if the subexpr is not a valid immediate within an addressing mode use, 818203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // set SubExpressionUseData.notAllUsesAreFree. We definitely want to 819203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // hoist these out of the loop (if they are common to all uses). 820203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) { 821203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (++SubExpressionUseData[SubExprs[j]].Count == 1) 822d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner UniqueSubExprs.push_back(SubExprs[j]); 823a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman if (!isAddrUse || !fitsInAddressMode(SubExprs[j], AccessTy, TLI, false)) 824203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData[SubExprs[j]].notAllUsesAreFree = true; 825203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 826934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.clear(); 827934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 828934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 829d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // Now that we know how many times each is used, build Result. Iterate over 830d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner // UniqueSubexprs so that we have a stable ordering. 831d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner for (unsigned i = 0, e = UniqueSubExprs.size(); i != e; ++i) { 8320bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, SubExprUseData>::iterator I = 833203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.find(UniqueSubExprs[i]); 834203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen assert(I != SubExpressionUseData.end() && "Entry not found?"); 835203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (I->second.Count == NumUsesInsideLoop) { // Found CSE! 836203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (I->second.notAllUsesAreFree) 837203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen Result = SE->getAddExpr(Result, I->first); 838203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen else 839203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen FreeResult = SE->getAddExpr(FreeResult, I->first); 840203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } else 841203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // Remove non-cse's from SubExpressionUseData. 842203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.erase(I); 843d6155e96f78a9f4344f5e697f7dd74d2f2325092Chris Lattner } 844203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 845203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (FreeResult != Zero) { 846203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We have some subexpressions that can be subsumed into addressing 847203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // modes in every use inside the loop. However, it's possible that 848203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // there are so many of them that the combined FreeResult cannot 849203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // be subsumed, or that the target cannot handle both a FreeResult 850203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // and a Result in the same instruction (for example because it would 851203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // require too many registers). Check this. 852203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned i=0; i<NumUses; ++i) { 853203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (!L->contains(Uses[i].Inst->getParent())) 854203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen continue; 855203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // We know this is an addressing mode use; if there are any uses that 856203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // are not, FreeResult would be Zero. 857a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman const Type *AccessTy = getAccessType(Uses[i].Inst); 858a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman if (!fitsInAddressMode(FreeResult, AccessTy, TLI, Result!=Zero)) { 859203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // FIXME: could split up FreeResult into pieces here, some hoisted 860b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // and some not. There is no obvious advantage to this. 861203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen Result = SE->getAddExpr(Result, FreeResult); 862203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen FreeResult = Zero; 863203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen break; 864203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 865203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 866203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 867203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 8681bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // If we found no CSE's, return now. 8691bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner if (Result == Zero) return Result; 8701bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 871203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // If we still have a FreeResult, remove its subexpressions from 872203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen // SubExpressionUseData. This means they will remain in the use Bases. 873203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (FreeResult != Zero) { 874203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SeparateSubExprs(SubExprs, FreeResult, SE); 875203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) { 8760bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, SubExprUseData>::iterator I = 877203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.find(SubExprs[j]); 878203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExpressionUseData.erase(I); 879203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 880203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen SubExprs.clear(); 881203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen } 882203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen 8831bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Otherwise, remove all of the CSE's we found from each of the base values. 884934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned i = 0; i != NumUses; ++i) { 885fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // Uses outside the loop don't necessarily include the common base, but 886fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // the final IV value coming into those uses does. Instead of trying to 887fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // remove the pieces of the common base, which might not be there, 888fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen // subtract off the base to compensate for this. 889fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen if (!L->contains(Uses[i].Inst->getParent())) { 890fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen Uses[i].Base = SE->getMinusSCEV(Uses[i].Base, Result); 891589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen continue; 892fb10cd490195128d4d36ed9d963f173a6d6ca46eDale Johannesen } 893589bf0865ccd10d36f406d622c0160be249343e1Dale Johannesen 894934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Split the expression into subexprs. 895246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SeparateSubExprs(SubExprs, Uses[i].Base, SE); 896934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 897934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner // Remove any common subexpressions. 898934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner for (unsigned j = 0, e = SubExprs.size(); j != e; ++j) 899203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen if (SubExpressionUseData.count(SubExprs[j])) { 900934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner SubExprs.erase(SubExprs.begin()+j); 901934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner --j; --e; 902934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 903934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner 904f8828eb41b2f8f0356ff6889c9924eade654e39aChris Lattner // Finally, add the non-shared expressions together. 905934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner if (SubExprs.empty()) 9061bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Uses[i].Base = Zero; 907934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner else 908246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman Uses[i].Base = SE->getAddExpr(SubExprs); 90927e5142309946ca12c633b265673af0c24684427Chris Lattner SubExprs.clear(); 910934520a747722ecf94f35768e5b88eeaf44c3b24Chris Lattner } 9111bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 9121bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner return Result; 9131bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner} 9141bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 9155792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng/// ValidScale - Check whether the given Scale is valid for all loads and 916579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner/// stores in UsersToProcess. 917dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen/// 9185792f51e12d9c8685399e9857799365854ab5bf6Evan Chengbool LoopStrengthReduce::ValidScale(bool HasBaseReg, int64_t Scale, 919dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess) { 920d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (!TLI) 921d6b62a572210aff965a55626cf36a68821838844Evan Cheng return true; 922d6b62a572210aff965a55626cf36a68821838844Evan Cheng 9235792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng for (unsigned i = 0, e = UsersToProcess.size(); i!=e; ++i) { 9241ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner // If this is a load or other access, pass the type of the access in. 9251d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson const Type *AccessTy = 9261d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson Type::getVoidTy(UsersToProcess[i].Inst->getContext()); 92721e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman if (isAddressUse(UsersToProcess[i].Inst, 92821e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman UsersToProcess[i].OperandValToReplace)) 92921e7722868378f67974e648ab21d0e3c69a0e379Dan Gohman AccessTy = getAccessType(UsersToProcess[i].Inst); 93055e641b766a18878b51551d626d5a566102e487eEvan Cheng else if (isa<PHINode>(UsersToProcess[i].Inst)) 93155e641b766a18878b51551d626d5a566102e487eEvan Cheng continue; 9321ebd89eb6b4f5df8d8171ee654a73ecdf3f66580Chris Lattner 933579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner TargetLowering::AddrMode AM; 934890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(UsersToProcess[i].Imm)) 935579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.BaseOffs = SC->getValue()->getSExtValue(); 936cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman AM.HasBaseReg = HasBaseReg || !UsersToProcess[i].Base->isZero(); 937579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner AM.Scale = Scale; 938579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner 939579633cd1006f6add1b022e9c2bc96f7f0e65777Chris Lattner // If load[imm+r*scale] is illegal, bail out. 940d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (!TLI->isLegalAddressingMode(AM, AccessTy)) 941dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen return false; 9428e59e163db8cd3e7b4c96e438fbedf78bff06707Dale Johannesen } 943dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen return true; 944dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen} 9451bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 94681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman/// ValidOffset - Check whether the given Offset is valid for all loads and 94781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman/// stores in UsersToProcess. 94881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman/// 94981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohmanbool LoopStrengthReduce::ValidOffset(bool HasBaseReg, 95081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman int64_t Offset, 95181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman int64_t Scale, 95281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman const std::vector<BasedUser>& UsersToProcess) { 95381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (!TLI) 95481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman return true; 95581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 95681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (unsigned i=0, e = UsersToProcess.size(); i!=e; ++i) { 95781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // If this is a load or other access, pass the type of the access in. 9581d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson const Type *AccessTy = 9591d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson Type::getVoidTy(UsersToProcess[i].Inst->getContext()); 96081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (isAddressUse(UsersToProcess[i].Inst, 96181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman UsersToProcess[i].OperandValToReplace)) 96281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman AccessTy = getAccessType(UsersToProcess[i].Inst); 96381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman else if (isa<PHINode>(UsersToProcess[i].Inst)) 96481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman continue; 96581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 96681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman TargetLowering::AddrMode AM; 96781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(UsersToProcess[i].Imm)) 96881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman AM.BaseOffs = SC->getValue()->getSExtValue(); 96981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman AM.BaseOffs = (uint64_t)AM.BaseOffs + (uint64_t)Offset; 97081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman AM.HasBaseReg = HasBaseReg || !UsersToProcess[i].Base->isZero(); 97181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman AM.Scale = Scale; 97281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 97381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // If load[imm+r*scale] is illegal, bail out. 97481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (!TLI->isLegalAddressingMode(AM, AccessTy)) 97581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman return false; 97681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman } 97781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman return true; 97881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman} 97981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 9801de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen/// RequiresTypeConversion - Returns true if converting Ty1 to Ty2 is not 9815f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// a nop. 9822bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Chengbool LoopStrengthReduce::RequiresTypeConversion(const Type *Ty1, 9832bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng const Type *Ty2) { 9842bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (Ty1 == Ty2) 9855f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return false; 9869f2d671b154e2b48b097d813b64a2bf2a33c547aDan Gohman Ty1 = SE->getEffectiveSCEVType(Ty1); 9879f2d671b154e2b48b097d813b64a2bf2a33c547aDan Gohman Ty2 = SE->getEffectiveSCEVType(Ty2); 9889f2d671b154e2b48b097d813b64a2bf2a33c547aDan Gohman if (Ty1 == Ty2) 989af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman return false; 9901de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (Ty1->canLosslesslyBitCastTo(Ty2)) 9911de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return false; 9922bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (TLI && TLI->isTruncateFree(Ty1, Ty2)) 9932bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng return false; 9941de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen return true; 9955f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 9965f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 997eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng/// CheckForIVReuse - Returns the multiple if the stride is the multiple 998eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng/// of a previous stride and it is a legal value for the target addressing 99902e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman/// mode scale component and optional base reg. This allows the users of 100002e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman/// this stride to be rewritten as prev iv * factor. It returns 0 if no 1001b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen/// reuse is possible. Factors can be negative on same targets, e.g. ARM. 10022f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// 10032f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// If all uses are outside the loop, we don't require that all multiplies 10042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// be folded into the addressing mode, nor even that the factor be constant; 10052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// a multiply (executed once) outside the loop is better than another IV 10062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen/// within. Well, usually. 10070bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanconst SCEV *LoopStrengthReduce::CheckForIVReuse(bool HasBaseReg, 10082bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng bool AllUsesAreAddresses, 1009b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool AllUsesAreOutsideLoop, 10100bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *const &Stride, 1011dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen IVExpr &IV, const Type *Ty, 1012dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen const std::vector<BasedUser>& UsersToProcess) { 10135792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (StrideNoReuse.count(Stride)) 10145792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng return SE->getIntegerSCEV(0, Stride->getType()); 10155792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1016890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Stride)) { 1017502db93a8ab376730164db43ca3ce8032b72bd59Reid Spencer int64_t SInt = SC->getValue()->getSExtValue(); 101881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (unsigned NewStride = 0, e = IU->StrideOrder.size(); 101981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman NewStride != e; ++NewStride) { 10200bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVsOfOneStride>::iterator SI = 102181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IVsByStride.find(IU->StrideOrder[NewStride]); 10225792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (SI == IVsByStride.end() || !isa<SCEVConstant>(SI->first) || 10235792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng StrideNoReuse.count(SI->first)) 1024b51b4b5fdfdd7a85ea66776d7900214f8b24f826Dale Johannesen continue; 1025586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // The other stride has no uses, don't reuse it. 1026586f69a11881d828c056ce017b3fb432341d9657Evan Cheng std::map<const SCEV *, IVUsersOfOneStride *>::iterator UI = 1027586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride.find(IU->StrideOrder[NewStride]); 1028586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (UI->second->Users.empty()) 1029586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 10305eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 10312bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng if (SI->first != Stride && 10327b9486ad4355a010d22e0737cee9cd7c7b747eceDale Johannesen (unsigned(abs64(SInt)) < SSInt || (SInt % SSInt) != 0)) 1033eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng continue; 10345eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng int64_t Scale = SInt / SSInt; 1035dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // Check that this stride is valid for all the types used for loads and 1036dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // stores; if it can be used for some and not others, we might as well use 1037dc42f48ea90132509e678028e7dbab5544ef0794Dale Johannesen // the original stride everywhere, since we have to create the IV for it 1038aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman // anyway. If the scale is 1, then we don't need to worry about folding 1039aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman // multiplications. 1040aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman if (Scale == 1 || 1041aa34331e7832dc1139231626516e9587eeecb0ceDan Gohman (AllUsesAreAddresses && 104281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman ValidScale(HasBaseReg, Scale, UsersToProcess))) { 104381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // Prefer to reuse an IV with a base of zero. 10445eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 10455eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng IE = SI->second.IVs.end(); II != IE; ++II) 104681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // Only reuse previous IV if it would not require a type conversion 104781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // and if the base difference can be folded. 1048cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman if (II->Base->isZero() && 10492bd122c4d934a70e031dc0ca5171719bac66c2c9Evan Cheng !RequiresTypeConversion(II->Base->getType(), Ty)) { 10505eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng IV = *II; 10512f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(Scale, Stride->getType()); 10525eef2d21a0c1cc4560a49e4dcdec9c00946a86c2Evan Cheng } 105381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // Otherwise, settle for an IV with a foldable base. 105481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (AllUsesAreAddresses) 105581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 105681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IE = SI->second.IVs.end(); II != IE; ++II) 105781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // Only reuse previous IV if it would not require a type conversion 105881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // and if the base difference can be folded. 105981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (SE->getEffectiveSCEVType(II->Base->getType()) == 106081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman SE->getEffectiveSCEVType(Ty) && 106181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman isa<SCEVConstant>(II->Base)) { 106281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman int64_t Base = 106381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman cast<SCEVConstant>(II->Base)->getValue()->getSExtValue(); 106481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (Base > INT32_MIN && Base <= INT32_MAX && 106581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman ValidOffset(HasBaseReg, -Base * Scale, 106681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman Scale, UsersToProcess)) { 106781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IV = *II; 106881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman return SE->getIntegerSCEV(Scale, Stride->getType()); 106981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman } 107081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman } 107181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman } 1072eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 10732f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } else if (AllUsesAreOutsideLoop) { 10742f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonconstant strides here; it is really really right to substitute 10752f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // an existing IV if we can. 107681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (unsigned NewStride = 0, e = IU->StrideOrder.size(); 107781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman NewStride != e; ++NewStride) { 10780bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVsOfOneStride>::iterator SI = 107981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IVsByStride.find(IU->StrideOrder[NewStride]); 10802f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end() || !isa<SCEVConstant>(SI->first)) 10812f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 10822f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 10832f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI->first != Stride && SSInt != 1) 10842f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 10852f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 10862f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IE = SI->second.IVs.end(); II != IE; ++II) 10872f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonzero base here. 10882f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Only reuse previous IV if it would not require a type conversion. 10892f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!RequiresTypeConversion(II->Base->getType(), Ty)) { 10902f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IV = *II; 10912f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return Stride; 10922f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 10932f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 10942f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Special case, old IV is -1*x and this one is x. Can treat this one as 10952f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // -1*old. 109681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (unsigned NewStride = 0, e = IU->StrideOrder.size(); 109781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman NewStride != e; ++NewStride) { 10980bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVsOfOneStride>::iterator SI = 109981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IVsByStride.find(IU->StrideOrder[NewStride]); 11002f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (SI == IVsByStride.end()) 11012f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen continue; 1102890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(SI->first)) 1103890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(ME->getOperand(0))) 11042f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (Stride == ME->getOperand(1) && 11052f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen SC->getValue()->getSExtValue() == -1LL) 11062f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen for (std::vector<IVExpr>::iterator II = SI->second.IVs.begin(), 11072f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IE = SI->second.IVs.end(); II != IE; ++II) 11082f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Accept nonzero base here. 11092f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Only reuse previous IV if it would not require type conversion. 11102f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (!RequiresTypeConversion(II->Base->getType(), Ty)) { 11112f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen IV = *II; 11122f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(-1LL, Stride->getType()); 11132f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 11142f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 1115eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 11162f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen return SE->getIntegerSCEV(0, Stride->getType()); 1117eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng} 1118eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 11197e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// PartitionByIsUseOfPostIncrementedValue - Simple boolean predicate that 11207e79b3898ddd919170d367a516f51296017146c2Chris Lattner/// returns true if Val's isUseOfPostIncrementedValue is true. 11217e79b3898ddd919170d367a516f51296017146c2Chris Lattnerstatic bool PartitionByIsUseOfPostIncrementedValue(const BasedUser &Val) { 11227e79b3898ddd919170d367a516f51296017146c2Chris Lattner return Val.isUseOfPostIncrementedValue; 11237e79b3898ddd919170d367a516f51296017146c2Chris Lattner} 1124eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 11254a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman/// isNonConstantNegative - Return true if the specified scev is negated, but 1126fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner/// not a constant. 11270bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanstatic bool isNonConstantNegative(const SCEV *const &Expr) { 1128890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Expr); 1129fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner if (!Mul) return false; 1130fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 1131fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner // If there is a constant factor, it will be first. 1132890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 1133fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner if (!SC) return false; 1134fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 1135fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner // Return true if the value is negative, this matches things like (-42 * V). 1136fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner return SC->getValue()->getValue().isNegative(); 1137fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner} 1138fb3e1190fc33c93a7185695051d5aeeaddbae0adChris Lattner 1139e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// CollectIVUsers - Transform our list of users and offsets to a bit more 1140e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// complex table. In this new vector, each 'BasedUser' contains 'Base', the base 1141e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// of the strided accesses, as well as the old information from Uses. We 1142e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// progressively move information from the Base field to the Imm field, until 1143e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// we eventually have the full access expression to rewrite the use. 11440bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanconst SCEV *LoopStrengthReduce::CollectIVUsers(const SCEV *const &Stride, 11455f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng IVUsersOfOneStride &Uses, 11465f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng Loop *L, 11475f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool &AllUsesAreAddresses, 1148b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool &AllUsesAreOutsideLoop, 11495f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> &UsersToProcess) { 115081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // FIXME: Generalize to non-affine IV's. 115181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (!Stride->isLoopInvariant(L)) 115281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman return SE->getIntegerSCEV(0, Stride->getType()); 115381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 1154169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman UsersToProcess.reserve(Uses.Users.size()); 115581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (ilist<IVStrideUse>::iterator I = Uses.Users.begin(), 115681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman E = Uses.Users.end(); I != E; ++I) { 115781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman UsersToProcess.push_back(BasedUser(*I, SE)); 115881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 115967c79892949332568b082f124d9598971fa3277fDale Johannesen // Move any loop variant operands from the offset field to the immediate 1160a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner // field of the use, so that we don't try to use something before it is 1161a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner // computed. 1162544e0d0e52e68e8bb0c4ce18bd6584602353ce1cDale Johannesen MoveLoopVariantsToImmediateField(UsersToProcess.back().Base, 11635792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng UsersToProcess.back().Imm, L, SE); 1164a553b0cc014bc0fb0d5d4820ec2725e733c60003Chris Lattner assert(UsersToProcess.back().Base->isLoopInvariant(L) && 116526d91f16464db56283087176a73981048331dd2dChris Lattner "Base value is not loop invariant!"); 11662461dff0700d0e34b9854d96a5cc03921b375525Chris Lattner } 1167eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 116831e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // We now have a whole bunch of uses of like-strided induction variables, but 116931e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // they might all have different bases. We want to emit one PHI node for this 117031e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // stride which we fold as many common expressions (between the IVs) into as 117131e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // possible. Start by identifying the common expressions in the base values 117231e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // for the strides (e.g. if we have "A+C+B" and "A+B+D" as our bases, find 117331e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // "A+B"), emit it to the preheader, then remove the expression from the 117431e773147bf13cb11c2bf343db5706fe3e2923d7Evan Cheng // UsersToProcess base values. 11750bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CommonExprs = 1176203af58aea3ae341d38e5c2c5b390b0c31d25557Dale Johannesen RemoveCommonExpressionsFromUseBases(UsersToProcess, SE, L, TLI); 117702e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman 117844b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // Next, figure out what we can represent in the immediate fields of 117944b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner // instructions. If we can represent anything there, move it to the imm 11801bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // fields of the BasedUsers. We do this so that it increases the commonality 11811bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // of the remaining uses. 118232e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng unsigned NumPHI = 0; 1183d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng bool HasAddress = false; 118444b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 118580b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // If the user is not in the current loop, this means it is using the exit 118680b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // value of the IV. Do not put anything in the base, make sure it's all in 118780b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // the immediate field to allow as much factoring as possible. 118880b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner if (!L->contains(UsersToProcess[i].Inst->getParent())) { 1189246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Imm = SE->getAddExpr(UsersToProcess[i].Imm, 1190246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Base); 11918385e51e210fb5c1e7e48eae150b31679b3e137dChris Lattner UsersToProcess[i].Base = 1192246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman SE->getIntegerSCEV(0, UsersToProcess[i].Base->getType()); 119380b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner } else { 1194d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // Not all uses are outside the loop. 1195d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AllUsesAreOutsideLoop = false; 1196d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 119780b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // Addressing modes can be folded into loads and stores. Be careful that 119880b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner // the store is through the expression, not of the expression though. 119932e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng bool isPHI = false; 1200d6b62a572210aff965a55626cf36a68821838844Evan Cheng bool isAddress = isAddressUse(UsersToProcess[i].Inst, 1201d6b62a572210aff965a55626cf36a68821838844Evan Cheng UsersToProcess[i].OperandValToReplace); 1202d6b62a572210aff965a55626cf36a68821838844Evan Cheng if (isa<PHINode>(UsersToProcess[i].Inst)) { 120332e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng isPHI = true; 120432e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng ++NumPHI; 12052acc7601650654d03cd53faeece8d7685a203105Dan Gohman } 120602e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman 1207d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng if (isAddress) 1208d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng HasAddress = true; 1209b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen 121002e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman // If this use isn't an address, then not all uses are addresses. 121155e641b766a18878b51551d626d5a566102e487eEvan Cheng if (!isAddress && !isPHI) 121202e4fa7d5fb4edd2ce9c7ede29c74d50cb126d7dDan Gohman AllUsesAreAddresses = false; 121380b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner 12141d95816db537242b3ba0e43a0ec96342ad696bf2Evan Cheng MoveImmediateValues(TLI, UsersToProcess[i].Inst, UsersToProcess[i].Base, 1215246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman UsersToProcess[i].Imm, isAddress, L, SE); 121680b32b3aab369534a25cfab6d9b7447cc4a8ff1dChris Lattner } 121744b807e3c0b358d75f153066b2b7556710d9c7ccChris Lattner } 1218d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1219d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // If one of the use is a PHI node and all other uses are addresses, still 122032e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng // allow iv reuse. Essentially we are trading one constant multiplication 122132e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng // for one fewer iv. 122232e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng if (NumPHI > 1) 122332e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng AllUsesAreAddresses = false; 1224d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 1225d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng // There are no in-loop address uses. 1226d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng if (AllUsesAreAddresses && (!HasAddress && !AllUsesAreOutsideLoop)) 1227d33cec18a9c0d9382dbf1e2b629f3daef29226eeEvan Cheng AllUsesAreAddresses = false; 122832e4c7c486084cdbed07925be4a0e9f3ab6caedeEvan Cheng 12295f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return CommonExprs; 12305f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng} 12315f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1232c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// ShouldUseFullStrengthReductionMode - Test whether full strength-reduction 1233c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// is valid and profitable for the given set of users of a stride. In 1234c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// full strength-reduction mode, all addresses at the current stride are 1235c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// strength-reduced all the way down to pointer arithmetic. 1236c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1237c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanbool LoopStrengthReduce::ShouldUseFullStrengthReductionMode( 1238c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const std::vector<BasedUser> &UsersToProcess, 1239c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1240c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool AllUsesAreAddresses, 12410bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Stride) { 1242c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!EnableFullLSRMode) 1243c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1244c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1245c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // The heuristics below aim to avoid increasing register pressure, but 1246c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // fully strength-reducing all the addresses increases the number of 1247c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // add instructions, so don't do this when optimizing for size. 1248c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: If the loop is large, the savings due to simpler addresses 1249c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // may oughtweight the costs of the extra increment instructions. 1250c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (L->getHeader()->getParent()->hasFnAttr(Attribute::OptimizeForSize)) 1251c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1252c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1253c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: For now, don't do full strength reduction if there could 1254c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // potentially be greater-stride multiples of the current stride 1255c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // which could reuse the current stride IV. 125681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (IU->StrideOrder.back() != Stride) 1257c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1258c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1259c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Iterate through the uses to find conditions that automatically rule out 1260c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // full-lsr mode. 1261c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ) { 1262622ed671b90d449496d582b3a03c7aad1e98324aDan Gohman const SCEV *Base = UsersToProcess[i].Base; 1263622ed671b90d449496d582b3a03c7aad1e98324aDan Gohman const SCEV *Imm = UsersToProcess[i].Imm; 1264c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If any users have a loop-variant component, they can't be fully 1265c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength-reduced. 1266c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (Imm && !Imm->isLoopInvariant(L)) 1267c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1268c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If there are to users with the same base and the difference between 1269c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // the two Imm values can't be folded into the address, full 1270c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength reduction would increase register pressure. 1271c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman do { 1272622ed671b90d449496d582b3a03c7aad1e98324aDan Gohman const SCEV *CurImm = UsersToProcess[i].Imm; 1273a04af4380d6814cb908d80ff8f52badc926a8bdcDan Gohman if ((CurImm || Imm) && CurImm != Imm) { 1274c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!CurImm) CurImm = SE->getIntegerSCEV(0, Stride->getType()); 1275c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!Imm) Imm = SE->getIntegerSCEV(0, Stride->getType()); 1276c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Instruction *Inst = UsersToProcess[i].Inst; 1277a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman const Type *AccessTy = getAccessType(Inst); 12780bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Diff = SE->getMinusSCEV(UsersToProcess[i].Imm, Imm); 1279c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!Diff->isZero() && 1280c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (!AllUsesAreAddresses || 1281a537bf83eda87a0eb65c278d673e51f76b0dd98bDan Gohman !fitsInAddressMode(Diff, AccessTy, TLI, /*HasBaseReg=*/true))) 1282c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1283c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1284c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } while (++i != e && Base == UsersToProcess[i].Base); 1285c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1286c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1287c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If there's exactly one user in this stride, fully strength-reducing it 1288c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // won't increase register pressure. If it's starting from a non-zero base, 1289c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // it'll be simpler this way. 1290c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (UsersToProcess.size() == 1 && !UsersToProcess[0].Base->isZero()) 1291c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return true; 1292c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1293c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Otherwise, if there are any users in this stride that don't require 1294c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // a register for their base, full strength-reduction will increase 1295c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // register pressure. 1296c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) 1297f0baa6e9cba77f54cee438e9a10e41c37789403eDan Gohman if (UsersToProcess[i].Base->isZero()) 1298c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return false; 1299c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1300c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Otherwise, go for it. 1301c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return true; 1302c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1303c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1304c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// InsertAffinePhi Create and insert a PHI node for an induction variable 1305c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// with the specified start and step values in the specified loop. 1306c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1307c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// If NegateStride is true, the stride should be negated by using a 1308c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// subtract instead of an add. 1309c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 13109d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman/// Return the created phi node. 1311c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 13120bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohmanstatic PHINode *InsertAffinePhi(const SCEV *Start, const SCEV *Step, 13135792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng Instruction *IVIncInsertPt, 1314c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 13159d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman SCEVExpander &Rewriter) { 1316c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(Start->isLoopInvariant(L) && "New PHI start is not loop invariant!"); 1317c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(Step->isLoopInvariant(L) && "New PHI stride is not loop invariant!"); 1318c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1319c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman BasicBlock *Header = L->getHeader(); 1320c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman BasicBlock *Preheader = L->getLoopPreheader(); 13210daeed270bb0318b8a1337026fe92646a40ee3d9Dan Gohman BasicBlock *LatchBlock = L->getLoopLatch(); 13222d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman const Type *Ty = Start->getType(); 1323af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman Ty = Rewriter.SE.getEffectiveSCEVType(Ty); 1324c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 13252d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman PHINode *PN = PHINode::Create(Ty, "lsr.iv", Header->begin()); 13262d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman PN->addIncoming(Rewriter.expandCodeFor(Start, Ty, Preheader->getTerminator()), 1327c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Preheader); 1328c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1329c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // If the stride is negative, insert a sub instead of an add for the 1330c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // increment. 1331c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman bool isNegative = isNonConstantNegative(Step); 13320bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *IncAmount = Step; 1333c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isNegative) 1334c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncAmount = Rewriter.SE.getNegativeSCEV(Step); 1335c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1336c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Insert an add instruction right before the terminator corresponding 13375792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // to the back-edge or just before the only use. The location is determined 13385792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // by the caller and passed in as IVIncInsertPt. 13392d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman Value *StepV = Rewriter.expandCodeFor(IncAmount, Ty, 13402d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman Preheader->getTerminator()); 13419d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman Instruction *IncV; 1342c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (isNegative) { 1343c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV = BinaryOperator::CreateSub(PN, StepV, "lsr.iv.next", 13445792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng IVIncInsertPt); 1345c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } else { 1346c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman IncV = BinaryOperator::CreateAdd(PN, StepV, "lsr.iv.next", 13475792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng IVIncInsertPt); 1348c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1349c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (!isa<ConstantInt>(StepV)) ++NumVariable; 1350c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 13510daeed270bb0318b8a1337026fe92646a40ee3d9Dan Gohman PN->addIncoming(IncV, LatchBlock); 1352c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1353c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++NumInserted; 1354c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman return PN; 1355c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1356c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1357c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanstatic void SortUsersToProcess(std::vector<BasedUser> &UsersToProcess) { 1358c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We want to emit code for users inside the loop first. To do this, we 1359c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // rearrange BasedUser so that the entries at the end have 1360c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // isUseOfPostIncrementedValue = false, because we pop off the end of the 1361c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // vector (so we handle them first). 1362c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::partition(UsersToProcess.begin(), UsersToProcess.end(), 1363c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PartitionByIsUseOfPostIncrementedValue); 1364c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1365c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Sort this by base, so that things with the same base are handled 1366c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // together. By partitioning first and stable-sorting later, we are 1367c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // guaranteed that within each base we will pop off users from within the 1368c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // loop before users outside of the loop with a particular base. 1369c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // 1370c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We would like to use stable_sort here, but we can't. The problem is that 13710bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman // const SCEV *'s don't have a deterministic ordering w.r.t to each other, so 1372c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // we don't have anything to do a '<' comparison on. Because we think the 1373c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // number of uses is small, do a horrible bubble sort which just relies on 1374c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // ==. 1375c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1376c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Get a base value. 13770bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Base = UsersToProcess[i].Base; 1378c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1379c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Compact everything with this base to be consecutive with this one. 1380c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned j = i+1; j != e; ++j) { 1381c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (UsersToProcess[j].Base == Base) { 1382c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::swap(UsersToProcess[i+1], UsersToProcess[j]); 1383c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ++i; 1384c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1385c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1386c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1387c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1388c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 13896b38e29f13cb8700146e4b170567e2828553db9aDan Gohman/// PrepareToStrengthReduceFully - Prepare to fully strength-reduce 13906b38e29f13cb8700146e4b170567e2828553db9aDan Gohman/// UsersToProcess, meaning lowering addresses all the way down to direct 13916b38e29f13cb8700146e4b170567e2828553db9aDan Gohman/// pointer arithmetic. 1392c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1393c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1394c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceFully( 1395c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 13960bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Stride, 13970bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CommonExprs, 1398c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1399c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter) { 1400bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Fully reducing all users\n"); 1401c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1402c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Rewrite the UsersToProcess records, creating a separate PHI for each 1403c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // unique Base value. 14045792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng Instruction *IVIncInsertPt = L->getLoopLatch()->getTerminator(); 1405c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ) { 1406c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // TODO: The uses are grouped by base, but not sorted. We arbitrarily 1407c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // pick the first Imm value here to start with, and adjust it for the 1408c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // other uses. 14090bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Imm = UsersToProcess[i].Imm; 14100bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Base = UsersToProcess[i].Base; 14110bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Start = SE->getAddExpr(CommonExprs, Base, Imm); 14125792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng PHINode *Phi = InsertAffinePhi(Start, Stride, IVIncInsertPt, L, 14139d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman PreheaderRewriter); 1414c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Loop over all the users with the same base. 1415c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman do { 1416c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Base = SE->getIntegerSCEV(0, Stride->getType()); 1417c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Imm = SE->getMinusSCEV(UsersToProcess[i].Imm, Imm); 1418c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = Phi; 1419c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman assert(UsersToProcess[i].Imm->isLoopInvariant(L) && 1420c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "ShouldUseFullStrengthReductionMode should reject this!"); 1421c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } while (++i != e && Base == UsersToProcess[i].Base); 1422c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } 1423c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1424c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 14255792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng/// FindIVIncInsertPt - Return the location to insert the increment instruction. 14265792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng/// If the only use if a use of postinc value, (must be the loop termination 14275792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng/// condition), then insert it just before the use. 14285792f51e12d9c8685399e9857799365854ab5bf6Evan Chengstatic Instruction *FindIVIncInsertPt(std::vector<BasedUser> &UsersToProcess, 14295792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng const Loop *L) { 14305792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (UsersToProcess.size() == 1 && 14315792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng UsersToProcess[0].isUseOfPostIncrementedValue && 14325792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng L->contains(UsersToProcess[0].Inst->getParent())) 14335792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng return UsersToProcess[0].Inst; 14345792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng return L->getLoopLatch()->getTerminator(); 14355792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng} 14365792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 1437c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// PrepareToStrengthReduceWithNewPhi - Insert a new induction variable for the 1438c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// given users to share. 1439c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1440c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1441c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceWithNewPhi( 1442c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 14430bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Stride, 14440bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CommonExprs, 1445c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 14465792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng Instruction *IVIncInsertPt, 1447c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const Loop *L, 1448c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SCEVExpander &PreheaderRewriter) { 1449bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Inserting new PHI:\n"); 1450c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1451c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PHINode *Phi = InsertAffinePhi(SE->getUnknown(CommonBaseV), 14525792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng Stride, IVIncInsertPt, L, 14539d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman PreheaderRewriter); 1454c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1455c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Remember this in case a later stride is multiple of this. 14569d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman IVsByStride[Stride].addIV(Stride, CommonExprs, Phi); 1457c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1458c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // All the users will share this new IV. 14599d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) 1460c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = Phi; 1461c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1462bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " IV="); 1463bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(WriteAsOperand(errs(), Phi, /*PrintType=*/false)); 1464bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << "\n"); 1465c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1466c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 14675792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng/// PrepareToStrengthReduceFromSmallerStride - Prepare for the given users to 14685792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng/// reuse an induction variable with a stride that is a factor of the current 1469c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// induction variable. 1470c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman/// 1471c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohmanvoid 1472c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan GohmanLoopStrengthReduce::PrepareToStrengthReduceFromSmallerStride( 1473c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman std::vector<BasedUser> &UsersToProcess, 1474c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *CommonBaseV, 1475c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman const IVExpr &ReuseIV, 1476c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Instruction *PreInsertPt) { 1477bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Rewriting in terms of existing IV of STRIDE " 1478bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner << *ReuseIV.Stride << " and BASE " << *ReuseIV.Base << "\n"); 1479c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1480c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // All the users will share the reused IV. 14819d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) 1482c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess[i].Phi = ReuseIV.PHI; 1483c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1484c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Constant *C = dyn_cast<Constant>(CommonBaseV); 1485c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (C && 1486c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman (!C->isNullValue() && 1487c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman !fitsInAddressMode(SE->getUnknown(CommonBaseV), CommonBaseV->getType(), 1488c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman TLI, false))) 1489c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // We want the common base emitted into the preheader! This is just 1490c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // using cast as a copy so BitCast (no-op cast) is appropriate 1491c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman CommonBaseV = new BitCastInst(CommonBaseV, CommonBaseV->getType(), 1492c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman "commonbase", PreInsertPt); 1493c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman} 1494c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1495d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Chengstatic bool IsImmFoldedIntoAddrMode(GlobalValue *GV, int64_t Offset, 149653f2ae268ab0e9201c0c50c28a27d451497a0371Dan Gohman const Type *AccessTy, 1497d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng std::vector<BasedUser> &UsersToProcess, 1498d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng const TargetLowering *TLI) { 1499d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng SmallVector<Instruction*, 16> AddrModeInsts; 1500d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) { 1501d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (UsersToProcess[i].isUseOfPostIncrementedValue) 1502d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng continue; 1503d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng ExtAddrMode AddrMode = 1504d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AddressingModeMatcher::Match(UsersToProcess[i].OperandValToReplace, 150553f2ae268ab0e9201c0c50c28a27d451497a0371Dan Gohman AccessTy, UsersToProcess[i].Inst, 1506d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AddrModeInsts, *TLI); 1507d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (GV && GV != AddrMode.BaseGV) 1508d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng return false; 1509d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (Offset && !AddrMode.BaseOffs) 1510d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // FIXME: How to accurate check it's immediate offset is folded. 1511d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng return false; 1512d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng AddrModeInsts.clear(); 1513d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1514d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng return true; 1515d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng} 1516d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 1517586f69a11881d828c056ce017b3fb432341d9657Evan Cheng/// StrengthReduceIVUsersOfStride - Strength reduce all of the users of a single 15185f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng/// stride of IV. All of the users may have different starting values, and this 15199f4ac31a9406c1f110e57b347689e15f64b33151Dan Gohman/// may not be the only stride. 1520586f69a11881d828c056ce017b3fb432341d9657Evan Chengvoid LoopStrengthReduce::StrengthReduceIVUsersOfStride(const SCEV *const &Stride, 1521586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVUsersOfOneStride &Uses, 1522586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Loop *L) { 15235f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // If all the users are moved to another stride, then there is nothing to do. 1524303595942502f17c087fa28874c2b89117148c45Dan Gohman if (Uses.Users.empty()) 15255f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng return; 15265f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 15275f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // Keep track if every use in UsersToProcess is an address. If they all are, 15285f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // we may be able to rewrite the entire collection of them in terms of a 15295f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // smaller-stride IV. 15305f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng bool AllUsesAreAddresses = true; 15315f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1532b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // Keep track if every use of a single stride is outside the loop. If so, 1533b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // we want to be more aggressive about reusing a smaller-stride IV; a 1534b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // multiply outside the loop is better than another IV inside. Well, usually. 1535b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen bool AllUsesAreOutsideLoop = true; 1536b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen 15375f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // Transform our list of users and offsets to a bit more complex table. In 15385f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // this new vector, each 'BasedUser' contains 'Base' the base of the 15395f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // strided accessas well as the old information from Uses. We progressively 15405f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // move information from the Base field to the Imm field, until we eventually 15415f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // have the full access expression to rewrite the use. 15425f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng std::vector<BasedUser> UsersToProcess; 15430bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *CommonExprs = CollectIVUsers(Stride, Uses, L, AllUsesAreAddresses, 154481ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng AllUsesAreOutsideLoop, 154581ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng UsersToProcess); 15465f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng 1547c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Sort the UsersToProcess array so that users with common bases are 1548c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // next to each other. 1549c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman SortUsersToProcess(UsersToProcess); 1550c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 15515f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // If we managed to find some expressions in common, we'll need to carry 15525f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // their value in a register and add it in for each use. This will take up 15535f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // a register operand, which potentially restricts what stride values are 15545f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng // valid. 1555cfeb6a450632f2a6cd05302633c8c2b8c90cfdfdDan Gohman bool HaveCommonExprs = !CommonExprs->isZero(); 1556fe35555d09ecdad4b42da1138a1d40ea4b83898eChris Lattner const Type *ReplacedTy = CommonExprs->getType(); 1557c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1558d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // If all uses are addresses, consider sinking the immediate part of the 1559d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // common expression back into uses if they can fit in the immediate fields. 15603cd389de398e95b1633d310fc9c4af5b9f84b7afEvan Cheng if (TLI && HaveCommonExprs && AllUsesAreAddresses) { 15610bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *NewCommon = CommonExprs; 15620bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Imm = SE->getIntegerSCEV(0, ReplacedTy); 15631d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson MoveImmediateValues(TLI, Type::getVoidTy( 15641d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson L->getLoopPreheader()->getContext()), 15651d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson NewCommon, Imm, true, L, SE); 1566d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (!Imm->isZero()) { 1567d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng bool DoSink = true; 1568d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 1569d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // If the immediate part of the common expression is a GV, check if it's 1570d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng // possible to fold it into the target addressing mode. 1571d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng GlobalValue *GV = 0; 1572890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(Imm)) 15732d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman GV = dyn_cast<GlobalValue>(SU->getValue()); 1574d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng int64_t Offset = 0; 1575890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Imm)) 1576d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng Offset = SC->getValue()->getSExtValue(); 1577d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (GV || Offset) 157853f2ae268ab0e9201c0c50c28a27d451497a0371Dan Gohman // Pass VoidTy as the AccessTy to be conservative, because 157953f2ae268ab0e9201c0c50c28a27d451497a0371Dan Gohman // there could be multiple access types among all the uses. 15801d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson DoSink = IsImmFoldedIntoAddrMode(GV, Offset, 15811d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson Type::getVoidTy(L->getLoopPreheader()->getContext()), 1582d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng UsersToProcess, TLI); 1583d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 1584d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng if (DoSink) { 1585bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Sinking " << *Imm << " back down into uses\n"); 1586d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng for (unsigned i = 0, e = UsersToProcess.size(); i != e; ++i) 1587d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng UsersToProcess[i].Imm = SE->getAddExpr(UsersToProcess[i].Imm, Imm); 1588d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng CommonExprs = NewCommon; 1589d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng HaveCommonExprs = !CommonExprs->isZero(); 1590d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng ++NumImmSunk; 1591d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1592d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1593d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng } 1594d9fb7124035d6f95fb08c5157ab1813fbb2a282fEvan Cheng 15951bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Now that we know what we need to do, insert the PHI node itself. 15961bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // 1597bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << "LSR: Examining IVs of TYPE " << *ReplacedTy << " of STRIDE " 1598bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner << *Stride << ":\n" 1599bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner << " Common base: " << *CommonExprs << "\n"); 1600d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 16015be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman SCEVExpander Rewriter(*SE); 16025be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman SCEVExpander PreheaderRewriter(*SE); 1603c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 16041bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner BasicBlock *Preheader = L->getLoopPreheader(); 16051bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Instruction *PreInsertPt = Preheader->getTerminator(); 160612b50410cd0a8cd81a7f528f862005e80921cf58Chris Lattner BasicBlock *LatchBlock = L->getLoopLatch(); 16075792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng Instruction *IVIncInsertPt = LatchBlock->getTerminator(); 1608d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1609a7235ea7245028a0723e8ab7fd011386b3900777Owen Anderson Value *CommonBaseV = Constant::getNullValue(ReplacedTy); 1610eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 16110bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *RewriteFactor = SE->getIntegerSCEV(0, ReplacedTy); 16121d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson IVExpr ReuseIV(SE->getIntegerSCEV(0, 16131d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson Type::getInt32Ty(Preheader->getContext())), 16141d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson SE->getIntegerSCEV(0, 16151d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson Type::getInt32Ty(Preheader->getContext())), 16169d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman 0); 16171bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 161881ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // Choose a strength-reduction strategy and prepare for it by creating 161981ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // the necessary PHIs and adjusting the bookkeeping. 1620c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (ShouldUseFullStrengthReductionMode(UsersToProcess, L, 1621c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman AllUsesAreAddresses, Stride)) { 1622c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceFully(UsersToProcess, Stride, CommonExprs, L, 1623c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PreheaderRewriter); 1624c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman } else { 1625c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Emit the initial base value into the loop preheader. 16262d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman CommonBaseV = PreheaderRewriter.expandCodeFor(CommonExprs, ReplacedTy, 16272d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman PreInsertPt); 1628c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman 1629c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // If all uses are addresses, check if it is possible to reuse an IV. The 1630c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // new IV must have a stride that is a multiple of the old stride; the 1631c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // multiple must be a number that can be encoded in the scale field of the 1632c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // target addressing mode; and we must have a valid instruction after this 1633c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // substitution, including the immediate field, if any. 1634c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman RewriteFactor = CheckForIVReuse(HaveCommonExprs, AllUsesAreAddresses, 1635c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman AllUsesAreOutsideLoop, 1636bb5b49cb8d1415685d194a7302ec6318f907160bDan Gohman Stride, ReuseIV, ReplacedTy, 1637c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman UsersToProcess); 16385792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (!RewriteFactor->isZero()) 1639c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman PrepareToStrengthReduceFromSmallerStride(UsersToProcess, CommonBaseV, 1640c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman ReuseIV, PreInsertPt); 16415792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng else { 16425792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng IVIncInsertPt = FindIVIncInsertPt(UsersToProcess, L); 16435792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng PrepareToStrengthReduceWithNewPhi(UsersToProcess, Stride, CommonExprs, 16445792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng CommonBaseV, IVIncInsertPt, 16455792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng L, PreheaderRewriter); 16465792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng } 16477e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 16487e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1649c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // Process all the users now, replacing their strided uses with 1650c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman // strength-reduced forms. This outer loop handles all bases, the inner 16517e79b3898ddd919170d367a516f51296017146c2Chris Lattner // loop handles all users of a particular base. 1652169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman while (!UsersToProcess.empty()) { 16530bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *Base = UsersToProcess.back().Base; 16542f09f519542202b8af227c2a524f8fe82378a934Dan Gohman Instruction *Inst = UsersToProcess.back().Inst; 1655be3e5212e23edc9210f774fac18d801de252e906Chris Lattner 16561bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Emit the code for Base into the preheader. 16572d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman Value *BaseV = 0; 16582d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman if (!Base->isZero()) { 16595be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman BaseV = PreheaderRewriter.expandCodeFor(Base, 0, PreInsertPt); 16602d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 1661bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " INSERTING code for BASE = " << *Base << ":"); 16622d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman if (BaseV->hasName()) 1663bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Result value name = %" << BaseV->getName()); 1664bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << "\n"); 16652d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 16662d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman // If BaseV is a non-zero constant, make sure that it gets inserted into 16672d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman // the preheader, instead of being forward substituted into the uses. We 16682d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman // do this by forcing a BitCast (noop cast) to be inserted into the 16692d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman // preheader in this case. 1670ea080be98625d86dc0a574f71d7df69e2351734cDan Gohman if (!fitsInAddressMode(Base, getAccessType(Inst), TLI, false) && 16713fe14bf15338866b21784a64b78e53b5801523c8Dan Gohman isa<Constant>(BaseV)) { 16723da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // We want this constant emitted into the preheader! This is just 16733da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer // using cast as a copy so BitCast (no-op cast) is appropriate 16743da59db637a887474c1b1346c1f3ccf53b6c4663Reid Spencer BaseV = new BitCastInst(BaseV, BaseV->getType(), "preheaderinsert", 16754a9a3e53746e3cc752d8a242ddc887a106cf5021Dan Gohman PreInsertPt); 16761bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner } 16777e79b3898ddd919170d367a516f51296017146c2Chris Lattner } 16787e79b3898ddd919170d367a516f51296017146c2Chris Lattner 1679169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // Emit the code to add the immediate offset to the Phi value, just before 16802351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner // the instructions that we identified as using this stride and base. 16817b445c521bc191d0d25799b289e17b45f202a1afChris Lattner do { 16827e79b3898ddd919170d367a516f51296017146c2Chris Lattner // FIXME: Use emitted users to emit other users. 16837b445c521bc191d0d25799b289e17b45f202a1afChris Lattner BasedUser &User = UsersToProcess.back(); 16842351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 1685bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " Examining "); 16865792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (User.isUseOfPostIncrementedValue) 1687bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << "postinc"); 16885792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng else 1689bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << "preinc"); 1690bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " use "); 1691bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(WriteAsOperand(errs(), UsersToProcess.back().OperandValToReplace, 16924a359ea2d5416706c3363687db9f03ed8daf635bDan Gohman /*PrintType=*/false)); 1693bdff548e4dd577a72094d57b282de4e765643b96Chris Lattner DEBUG(errs() << " in Inst: " << *User.Inst); 16942f09f519542202b8af227c2a524f8fe82378a934Dan Gohman 1695010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // If this instruction wants to use the post-incremented value, move it 1696010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // after the post-inc and use its value instead of the PHI. 1697c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman Value *RewriteOp = User.Phi; 1698010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (User.isUseOfPostIncrementedValue) { 16999d10086797b4dfe58f8b13a71534b28805f0713dDan Gohman RewriteOp = User.Phi->getIncomingValueForBlock(LatchBlock); 1700c6bae65b49f3b0bb457b6bdb60e29bd9a44e743aChris Lattner // If this user is in the loop, make sure it is the last thing in the 17015792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // loop to ensure it is dominated by the increment. In case it's the 17025792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // only use of the iv, the increment instruction is already before the 17035792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // use. 17045792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (L->contains(User.Inst->getParent()) && User.Inst != IVIncInsertPt) 17055792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng User.Inst->moveBefore(IVIncInsertPt); 1706010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 170786c75d31133319a88216c1b1cd26a789e4023000Evan Cheng 17080bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *RewriteExpr = SE->getUnknown(RewriteOp); 17091bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner 171081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (SE->getEffectiveSCEVType(RewriteOp->getType()) != 171181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman SE->getEffectiveSCEVType(ReplacedTy)) { 1712af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman assert(SE->getTypeSizeInBits(RewriteOp->getType()) > 1713af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman SE->getTypeSizeInBits(ReplacedTy) && 1714e616bf338a3669fad4bc3806547f70328eb2aa7eDan Gohman "Unexpected widening cast!"); 1715e616bf338a3669fad4bc3806547f70328eb2aa7eDan Gohman RewriteExpr = SE->getTruncateExpr(RewriteExpr, ReplacedTy); 1716e616bf338a3669fad4bc3806547f70328eb2aa7eDan Gohman } 1717e616bf338a3669fad4bc3806547f70328eb2aa7eDan Gohman 1718b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen // If we had to insert new instructions for RewriteOp, we have to 1719f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // consider that they may not have been able to end up immediately 1720f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // next to RewriteOp, because non-PHI instructions may never precede 1721f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman // PHI instructions in a block. In this case, remember where the last 1722ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // instruction was inserted so that if we're replacing a different 1723ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // PHI node, we can use the later point to expand the final 1724ca756ae886f1d39053ffd049123fdcc7e5c2aed3Dan Gohman // RewriteExpr. 1725f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman Instruction *NewBasePt = dyn_cast<Instruction>(RewriteOp); 1726c17e0cf6c03a36f424fafe88497b5fdf351cd50aDan Gohman if (RewriteOp == User.Phi) NewBasePt = 0; 1727f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman 17281bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Clear the SCEVExpander's expression map so that we are guaranteed 17291bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // to have the code emitted where we expect it. 17301bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner Rewriter.clear(); 1731d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1732d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng // If we are reusing the iv, then it must be multiplied by a constant 17331de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // factor to take advantage of the addressing mode scale component. 17342d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman if (!RewriteFactor->isZero()) { 17352f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // If we're reusing an IV with a nonzero base (currently this happens 17362f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // only when all reuses are outside the loop) subtract that base here. 17372f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // The base has been used to initialize the PHI node but we don't want 17382f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // it here. 17391de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen if (!ReuseIV.Base->isZero()) { 17400bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *typedBase = ReuseIV.Base; 174181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (SE->getEffectiveSCEVType(RewriteExpr->getType()) != 174281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman SE->getEffectiveSCEVType(ReuseIV.Base->getType())) { 17431de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // It's possible the original IV is a larger type than the new IV, 17441de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // in which case we have to truncate the Base. We checked in 17451de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen // RequiresTypeConversion that this is valid. 1746af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman assert(SE->getTypeSizeInBits(RewriteExpr->getType()) < 1747af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman SE->getTypeSizeInBits(ReuseIV.Base->getType()) && 174884fc33ed920d55fe59544ec83f14d1abc5bf0d63Dan Gohman "Unexpected lengthening conversion!"); 17491de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen typedBase = SE->getTruncateExpr(ReuseIV.Base, 17501de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen RewriteExpr->getType()); 17511de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen } 17521de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen RewriteExpr = SE->getMinusSCEV(RewriteExpr, typedBase); 17531de17d574c0a4503f7dd6a4a3efce6f9353bf3c5Dale Johannesen } 17542f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen 17552f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // Multiply old variable, with base removed, by new scale factor. 17562f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getMulExpr(RewriteFactor, 17578392772727ed9105c92fe4514d53dab74c333edcEvan Cheng RewriteExpr); 1758eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng 1759eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // The common base is emitted in the loop preheader. But since we 1760eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // are reusing an IV, it has not been used to initialize the PHI node. 1761eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng // Add it to the expression used to rewrite the uses. 17622f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // When this use is outside the loop, we earlier subtracted the 17632f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // common base, and are adding it back here. Use the same expression 17642f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen // as before, rather than CommonBaseV, so DAGCombiner will zap it. 17652d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman if (!CommonExprs->isZero()) { 17662f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen if (L->contains(User.Inst->getParent())) 17672f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getAddExpr(RewriteExpr, 1768b0390620d462cc3f48492febdbdbae4cb45d8bcdDale Johannesen SE->getUnknown(CommonBaseV)); 17692f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen else 17702f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen RewriteExpr = SE->getAddExpr(RewriteExpr, CommonExprs); 17712f46bb8178e30e3b845859a44b57c048db06ef84Dale Johannesen } 1772eb8f9e229740a0f292f5e8f0975bd10b7889b194Evan Cheng } 1773d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 17741bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Now that we know what we need to do, insert code before User for the 17751bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // immediate and any loop-variant expressions. 17762d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman if (BaseV) 17771bbae0cbf212d0356f564d12e8f728be455bdc47Chris Lattner // Add BaseV to the PHI value if needed. 1778246b2564d3bbbafe06ebf6a67745cd24141b5cb4Dan Gohman RewriteExpr = SE->getAddExpr(RewriteExpr, SE->getUnknown(BaseV)); 1779d1d6b5cce260808deeac0227b00f6f81a20b2c6fEvan Cheng 1780f20d70d57e1723d70b48b3f19868e17b9282bbfcDan Gohman User.RewriteInstructionToUseNewBase(RewriteExpr, NewBasePt, 17815be18e84766fb495b0bde3c8244c1df459a18683Dan Gohman Rewriter, L, this, *LI, 17820e0014d0499d6ec6402e07b71cf24af992a9d297Evan Cheng DeadInsts); 17832351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 1784a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner // Mark old value we replaced as possibly dead, so that it is eliminated 17852351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner // if we just replaced the last use of that value. 178681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman DeadInsts.push_back(User.OperandValToReplace); 17872351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner 17887b445c521bc191d0d25799b289e17b45f202a1afChris Lattner UsersToProcess.pop_back(); 17892351abaeab89ef0a0df5a715d0794ed5dec48bb3Chris Lattner ++NumReduced; 17907b445c521bc191d0d25799b289e17b45f202a1afChris Lattner 17917e79b3898ddd919170d367a516f51296017146c2Chris Lattner // If there are any more users to process with the same base, process them 17927e79b3898ddd919170d367a516f51296017146c2Chris Lattner // now. We sorted by base above, so we just have to check the last elt. 17937b445c521bc191d0d25799b289e17b45f202a1afChris Lattner } while (!UsersToProcess.empty() && UsersToProcess.back().Base == Base); 1794169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // TODO: Next, find out which base index is the most common, pull it out. 1795169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman } 1796169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 1797169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // IMPORTANT TODO: Figure out how to partition the IV's with this stride, but 1798169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman // different starting values, into different PHIs. 1799eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 1800eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 1801586f69a11881d828c056ce017b3fb432341d9657Evan Chengvoid LoopStrengthReduce::StrengthReduceIVUsers(Loop *L) { 1802586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Note: this processes each stride/type pair individually. All users 1803586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // passed into StrengthReduceIVUsersOfStride have the same type AND stride. 1804586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Also, note that we iterate over IVUsesByStride indirectly by using 1805586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // StrideOrder. This extra layer of indirection makes the ordering of 1806586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // strides deterministic - not dependent on map order. 1807586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (unsigned Stride = 0, e = IU->StrideOrder.size(); Stride != e; ++Stride) { 1808586f69a11881d828c056ce017b3fb432341d9657Evan Cheng std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI = 1809586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride.find(IU->StrideOrder[Stride]); 1810586f69a11881d828c056ce017b3fb432341d9657Evan Cheng assert(SI != IU->IVUsesByStride.end() && "Stride doesn't exist!"); 1811586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // FIXME: Generalize to non-affine IV's. 1812586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!SI->first->isLoopInvariant(L)) 1813586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 1814586f69a11881d828c056ce017b3fb432341d9657Evan Cheng StrengthReduceIVUsersOfStride(SI->first, *SI->second, L); 1815586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 1816586f69a11881d828c056ce017b3fb432341d9657Evan Cheng} 1817586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 1818c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel/// FindIVUserForCond - If Cond has an operand that is an expression of an IV, 1819aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner/// set the IV user and stride information and return true, otherwise return 1820aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner/// false. 1821c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patelbool LoopStrengthReduce::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse, 1822586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV* &CondStride) { 182381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (unsigned Stride = 0, e = IU->StrideOrder.size(); 182481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman Stride != e && !CondUse; ++Stride) { 18250bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI = 182681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IU->IVUsesByStride.find(IU->StrideOrder[Stride]); 182781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman assert(SI != IU->IVUsesByStride.end() && "Stride doesn't exist!"); 182881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 182981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (ilist<IVStrideUse>::iterator UI = SI->second->Users.begin(), 183081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman E = SI->second->Users.end(); UI != E; ++UI) 183181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (UI->getUser() == Cond) { 1832aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // NOTE: we could handle setcc instructions with multiple uses here, but 1833aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // InstCombine does it as well for simple uses, it's not clear that it 1834aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner // occurs enough in real life to handle. 183581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman CondUse = UI; 1836586f69a11881d828c056ce017b3fb432341d9657Evan Cheng CondStride = SI->first; 1837aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return true; 1838aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner } 1839aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner } 1840aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner return false; 1841aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner} 1842aed01d19315132daf68414ace410ec725b4b6d30Chris Lattner 1843cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Chengnamespace { 1844cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Constant strides come first which in turns are sorted by their absolute 1845cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // values. If absolute values are the same, then positive strides comes first. 1846cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // e.g. 1847cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // 4, -1, X, 1, 2 ==> 1, -1, 2, 4, X 1848cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng struct StrideCompare { 1849af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman const ScalarEvolution *SE; 1850af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman explicit StrideCompare(const ScalarEvolution *se) : SE(se) {} 18512d1be87ee40a4a0241d94448173879d9df2bc5b3Dan Gohman 18520bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman bool operator()(const SCEV *const &LHS, const SCEV *const &RHS) { 1853890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS); 1854890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS); 1855cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (LHSC && RHSC) { 1856cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t LV = LHSC->getValue()->getSExtValue(); 1857cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t RV = RHSC->getValue()->getSExtValue(); 1858cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng uint64_t ALV = (LV < 0) ? -LV : LV; 1859cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng uint64_t ARV = (RV < 0) ? -RV : RV; 1860bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman if (ALV == ARV) { 1861bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman if (LV != RV) 1862bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return LV > RV; 1863bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman } else { 1864cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return ALV < ARV; 1865bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman } 1866bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman 1867bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // If it's the same value but different type, sort by bit width so 1868bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // that we emit larger induction variables before smaller 1869bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman // ones, letting the smaller be re-written in terms of larger ones. 1870af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman return SE->getTypeSizeInBits(RHS->getType()) < 1871af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman SE->getTypeSizeInBits(LHS->getType()); 1872cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 1873bc511725f08c45984be6ff47d069c3773a2f2eb0Dan Gohman return LHSC && !RHSC; 1874cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 1875cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng }; 1876cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 1877cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1878cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ChangeCompareStride - If a loop termination compare instruction is the 1879cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// only use of its stride, and the compaison is against a constant value, 1880cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// try eliminate the stride by moving the compare instruction to another 1881cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// stride and change its constant operand accordingly. e.g. 1882cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// 1883cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// loop: 1884cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ... 1885cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v1 = v1 + 3 1886cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v2 = v2 + 1 1887cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// if (v2 < 10) goto loop 1888cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// => 1889cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// loop: 1890cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// ... 1891cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// v1 = v1 + 3 1892cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng/// if (v1 < 30) goto loop 1893cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan ChengICmpInst *LoopStrengthReduce::ChangeCompareStride(Loop *L, ICmpInst *Cond, 1894586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVStrideUse* &CondUse, 1895586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV* &CondStride, 1896586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool PostPass) { 189781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // If there's only one stride in the loop, there's nothing to do here. 189881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (IU->StrideOrder.size() < 2) 189981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman return Cond; 190081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // If there are other users of the condition's stride, don't bother 190181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // trying to change the condition because the stride will still 190281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // remain. 19030bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVUsersOfOneStride *>::iterator I = 1904586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride.find(CondStride); 1905586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (I == IU->IVUsesByStride.end()) 1906cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return Cond; 1907586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (I->second->Users.size() > 1) { 1908586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (ilist<IVStrideUse>::iterator II = I->second->Users.begin(), 1909586f69a11881d828c056ce017b3fb432341d9657Evan Cheng EE = I->second->Users.end(); II != EE; ++II) { 1910586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (II->getUser() == Cond) 1911586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 1912586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!isInstructionTriviallyDead(II->getUser())) 1913586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return Cond; 1914586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 1915586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 191681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // Only handle constant strides for now. 1917586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEVConstant *SC = dyn_cast<SCEVConstant>(CondStride); 1918cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng if (!SC) return Cond; 1919cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1920cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst::Predicate Predicate = Cond->getPredicate(); 1921cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t CmpSSInt = SC->getValue()->getSExtValue(); 1922586f69a11881d828c056ce017b3fb432341d9657Evan Cheng unsigned BitWidth = SE->getTypeSizeInBits(CondStride->getType()); 1923168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng uint64_t SignBit = 1ULL << (BitWidth-1); 1924c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman const Type *CmpTy = Cond->getOperand(0)->getType(); 1925168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng const Type *NewCmpTy = NULL; 1926af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman unsigned TyBits = SE->getTypeSizeInBits(CmpTy); 1927af62c09437675369afcc481e88ed9cd5806491feEvan Cheng unsigned NewTyBits = 0; 1928586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *NewStride = NULL; 1929ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman Value *NewCmpLHS = NULL; 1930ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman Value *NewCmpRHS = NULL; 1931cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng int64_t Scale = 1; 19320bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *NewOffset = SE->getIntegerSCEV(0, CmpTy); 1933cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1934c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (ConstantInt *C = dyn_cast<ConstantInt>(Cond->getOperand(1))) { 1935c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman int64_t CmpVal = C->getValue().getSExtValue(); 1936d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel 1937586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Check the relevant induction variable for conformance to 1938586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // the pattern. 1939586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *IV = SE->getSCEV(Cond->getOperand(0)); 1940586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 1941586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!AR || !AR->isAffine()) 1942586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return Cond; 1943586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 1944586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart()); 1945c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Check stride constant and the comparision constant signs to detect 1946c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // overflow. 1947586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (StartC) { 1948586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if ((StartC->getValue()->getSExtValue() < CmpVal && CmpSSInt < 0) || 1949586f69a11881d828c056ce017b3fb432341d9657Evan Cheng (StartC->getValue()->getSExtValue() > CmpVal && CmpSSInt > 0)) 1950586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return Cond; 1951586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } else { 1952586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // More restrictive check for the other cases. 1953586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if ((CmpVal & SignBit) != (CmpSSInt & SignBit)) 1954586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return Cond; 1955586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 1956cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1957c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Look for a suitable stride / iv as replacement. 195881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (unsigned i = 0, e = IU->StrideOrder.size(); i != e; ++i) { 19590bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI = 196081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IU->IVUsesByStride.find(IU->StrideOrder[i]); 1961586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!isa<SCEVConstant>(SI->first) || SI->second->Users.empty()) 1962c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 1963c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman int64_t SSInt = cast<SCEVConstant>(SI->first)->getValue()->getSExtValue(); 1964c7749b747e634f310b7f0e075f7329dcb88f58ccDan Gohman if (SSInt == CmpSSInt || 19657b9486ad4355a010d22e0737cee9cd7c7b747eceDale Johannesen abs64(SSInt) < abs64(CmpSSInt) || 1966c7749b747e634f310b7f0e075f7329dcb88f58ccDan Gohman (SSInt % CmpSSInt) != 0) 1967c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 1968168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 1969c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman Scale = SSInt / CmpSSInt; 1970c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman int64_t NewCmpVal = CmpVal * Scale; 1971586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 1972586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If old icmp value fits in icmp immediate field, but the new one doesn't 1973586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // try something else. 1974586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (TLI && 1975586f69a11881d828c056ce017b3fb432341d9657Evan Cheng TLI->isLegalICmpImmediate(CmpVal) && 1976586f69a11881d828c056ce017b3fb432341d9657Evan Cheng !TLI->isLegalICmpImmediate(NewCmpVal)) 1977586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 1978586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 1979e19c840d6c6fe075b61a21a0f6abe338afba666dDavid Greene APInt Mul = APInt(BitWidth*2, CmpVal, true); 1980e19c840d6c6fe075b61a21a0f6abe338afba666dDavid Greene Mul = Mul * APInt(BitWidth*2, Scale, true); 1981c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Check for overflow. 1982ee08da89565714ca143d4754f167e3277974fc19Evan Cheng if (!Mul.isSignedIntN(BitWidth)) 1983c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 198481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // Check for overflow in the stride's type too. 198581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (!Mul.isSignedIntN(SE->getTypeSizeInBits(SI->first->getType()))) 198681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman continue; 1987cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 1988c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Watch out for overflow. 19894a134afaef984c403f6d6b1da4d301666d899b92Nick Lewycky if (ICmpInst::isSigned(Predicate) && 1990c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman (CmpVal & SignBit) != (NewCmpVal & SignBit)) 1991c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 1992af62c09437675369afcc481e88ed9cd5806491feEvan Cheng 1993c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Pick the best iv to use trying to avoid a cast. 1994c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpLHS = NULL; 199581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (ilist<IVStrideUse>::iterator UI = SI->second->Users.begin(), 199681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman E = SI->second->Users.end(); UI != E; ++UI) { 199781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman Value *Op = UI->getOperandValToReplace(); 199881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 199981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // If the IVStrideUse implies a cast, check for an actual cast which 200081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // can be used to find the original IV expression. 200181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (SE->getEffectiveSCEVType(Op->getType()) != 200281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman SE->getEffectiveSCEVType(SI->first->getType())) { 200381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman CastInst *CI = dyn_cast<CastInst>(Op); 200481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // If it's not a simple cast, it's complicated. 200581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (!CI) 200681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman continue; 200781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // If it's a cast from a type other than the stride type, 200881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // it's complicated. 200981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (CI->getOperand(0)->getType() != SI->first->getType()) 201081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman continue; 201181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman // Ok, we found the IV expression in the stride's type. 201281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman Op = CI->getOperand(0); 201381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman } 201481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman 201581db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman NewCmpLHS = Op; 2016c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (NewCmpLHS->getType() == CmpTy) 2017c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman break; 20185f8ebaa5c0c216b7dbb16b781895e4af36bfa39aEvan Cheng } 2019c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (!NewCmpLHS) 2020cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng continue; 2021cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2022c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewCmpTy = NewCmpLHS->getType(); 2023af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman NewTyBits = SE->getTypeSizeInBits(NewCmpTy); 20241d0be15f89cb5056e20e2d24faa8d6afb1573bcaOwen Anderson const Type *NewCmpIntTy = IntegerType::get(Cond->getContext(), NewTyBits); 2025c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (RequiresTypeConversion(NewCmpTy, CmpTy)) { 2026c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Check if it is possible to rewrite it using 2027c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // an iv / stride of a smaller integer type. 202865e05b69d60e67caad9ef70b24d4033d0e68b617Dan Gohman unsigned Bits = NewTyBits; 20294a134afaef984c403f6d6b1da4d301666d899b92Nick Lewycky if (ICmpInst::isSigned(Predicate)) 203065e05b69d60e67caad9ef70b24d4033d0e68b617Dan Gohman --Bits; 203165e05b69d60e67caad9ef70b24d4033d0e68b617Dan Gohman uint64_t Mask = (1ULL << Bits) - 1; 203265e05b69d60e67caad9ef70b24d4033d0e68b617Dan Gohman if (((uint64_t)NewCmpVal & Mask) != (uint64_t)NewCmpVal) 2033c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2034c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman } 2035ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman 2036c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // Don't rewrite if use offset is non-constant and the new type is 2037c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // of a different type. 2038c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // FIXME: too conservative? 203981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (NewTyBits != TyBits && !isa<SCEVConstant>(CondUse->getOffset())) 2040c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman continue; 2041cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2042586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!PostPass) { 2043586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool AllUsesAreAddresses = true; 2044586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool AllUsesAreOutsideLoop = true; 2045586f69a11881d828c056ce017b3fb432341d9657Evan Cheng std::vector<BasedUser> UsersToProcess; 2046586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *CommonExprs = CollectIVUsers(SI->first, *SI->second, L, 2047586f69a11881d828c056ce017b3fb432341d9657Evan Cheng AllUsesAreAddresses, 2048586f69a11881d828c056ce017b3fb432341d9657Evan Cheng AllUsesAreOutsideLoop, 2049586f69a11881d828c056ce017b3fb432341d9657Evan Cheng UsersToProcess); 2050586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Avoid rewriting the compare instruction with an iv of new stride 2051586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // if it's likely the new stride uses will be rewritten using the 2052586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // stride of the compare instruction. 2053586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (AllUsesAreAddresses && 2054586f69a11881d828c056ce017b3fb432341d9657Evan Cheng ValidScale(!CommonExprs->isZero(), Scale, UsersToProcess)) 2055586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2056586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2057c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman 2058c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman // Avoid rewriting the compare instruction with an iv which has 2059c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman // implicit extension or truncation built into it. 2060c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman // TODO: This is over-conservative. 2061c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman if (SE->getTypeSizeInBits(CondUse->getOffset()->getType()) != TyBits) 2062c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman continue; 2063c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman 2064c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // If scale is negative, use swapped predicate unless it's testing 2065c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman // for equality. 2066c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (Scale < 0 && !Cond->isEquality()) 2067c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman Predicate = ICmpInst::getSwappedPredicate(Predicate); 2068c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman 2069586f69a11881d828c056ce017b3fb432341d9657Evan Cheng NewStride = IU->StrideOrder[i]; 2070c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman if (!isa<PointerType>(NewCmpTy)) 2071eed707b1e6097aac2bb6b3d47271f6300ace7f2eOwen Anderson NewCmpRHS = ConstantInt::get(NewCmpTy, NewCmpVal); 2072c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman else { 2073eed707b1e6097aac2bb6b3d47271f6300ace7f2eOwen Anderson Constant *CI = ConstantInt::get(NewCmpIntTy, NewCmpVal); 2074baf3c404409d5e47b13984a7f95bfbd6d1f2e79eOwen Anderson NewCmpRHS = ConstantExpr::getIntToPtr(CI, NewCmpTy); 2075c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman } 2076c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman NewOffset = TyBits == NewTyBits 2077c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman ? SE->getMulExpr(CondUse->getOffset(), 20786de29f8d960505421d61c80cdb738e16720b6c0eDan Gohman SE->getConstant(CmpTy, Scale)) 20796de29f8d960505421d61c80cdb738e16720b6c0eDan Gohman : SE->getConstant(NewCmpIntTy, 2080c2695eba5700c785c2ae144eede7e1a932e3f5f3Dan Gohman cast<SCEVConstant>(CondUse->getOffset())->getValue() 20816de29f8d960505421d61c80cdb738e16720b6c0eDan Gohman ->getSExtValue()*Scale); 2082c34fea3935dcc0cab13cfb5cb60471bd49089230Dan Gohman break; 2083cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2084cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2085cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 20869b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // Forgo this transformation if it the increment happens to be 20879b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // unfortunately positioned after the condition, and the condition 20889b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // has multiple uses which prevent it from being moved immediately 20899b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // before the branch. See 20909b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // test/Transforms/LoopStrengthReduce/change-compare-stride-trickiness-*.ll 20919b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman // for an example of this situation. 2092d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel if (!Cond->hasOneUse()) { 20939b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman for (BasicBlock::iterator I = Cond, E = Cond->getParent()->end(); 20949b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman I != E; ++I) 2095ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman if (I == NewCmpLHS) 20969b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman return Cond; 2097d16aba22c9eca0b2576a5fe5507f3c8ab378942eDevang Patel } 20989b93dd1f1ab3532a101a432d7000477adfe6401dDan Gohman 2099ff518c86f7daf4e7abeabb3b03a5a9237c042da9Dan Gohman if (NewCmpRHS) { 2100cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // Create a new compare instruction using new stride / iv. 2101cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ICmpInst *OldCond = Cond; 2102168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng // Insert new compare instruction. 2103333c40096561218bc3597cf153c0a3895274414cOwen Anderson Cond = new ICmpInst(OldCond, Predicate, NewCmpLHS, NewCmpRHS, 2104333c40096561218bc3597cf153c0a3895274414cOwen Anderson L->getHeader()->getName() + ".termcond"); 2105168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2106586f69a11881d828c056ce017b3fb432341d9657Evan Cheng DEBUG(errs() << " Change compare stride in Inst " << *OldCond); 2107586f69a11881d828c056ce017b3fb432341d9657Evan Cheng DEBUG(errs() << " to " << *Cond << '\n'); 2108586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2109168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng // Remove the old compare instruction. The old indvar is probably dead too. 211081db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman DeadInsts.push_back(CondUse->getOperandValToReplace()); 2111010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman OldCond->replaceAllUsesWith(Cond); 2112cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng OldCond->eraseFromParent(); 2113168a66b21e71e4c3301c9d4152d4c683a4bef889Evan Cheng 2114586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride[NewStride]->addUser(NewOffset, Cond, NewCmpLHS); 2115586f69a11881d828c056ce017b3fb432341d9657Evan Cheng CondUse = &IU->IVUsesByStride[NewStride]->Users.back(); 2116cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng CondStride = NewStride; 2117cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng ++NumEliminated; 2118afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman Changed = true; 2119cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng } 2120cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2121cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng return Cond; 2122cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng} 2123cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 21242781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// OptimizeMax - Rewrite the loop's terminating condition if it uses 21252781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// a max computation. 2126ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2127ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// This is a narrow solution to a specific, but acute, problem. For loops 2128ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// like this: 2129ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2130ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2131ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2132ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2133ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i < n); 2134ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 21352781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// the trip count isn't just 'n', because 'n' might not be positive. And 21362781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// unfortunately this can come up even for loops where the user didn't use 21372781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// a C do-while loop. For example, seemingly well-behaved top-test loops 21382781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// will commonly be lowered like this: 2139ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman// 2140ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// if (n > 0) { 2141ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 2142ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2143ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 2144ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } while (++i < n); 2145ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// } 2146ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2147ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// and then it's possible for subsequent optimization to obscure the if 2148ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// test in such a way that indvars can't find it. 2149ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2150ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// When indvars can't find the if test in loops like this, it creates a 21512781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// max expression, which allows it to give the loop a canonical 2152ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// induction variable: 2153ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2154ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// i = 0; 21552781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// max = n < 1 ? 1 : n; 2156ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// do { 2157ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// p[i] = 0.0; 21582781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman/// } while (++i != max); 2159ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2160ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// Canonical induction variables are necessary because the loop passes 2161ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// are designed around them. The most obvious example of this is the 2162ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// LoopInfo analysis, which doesn't remember trip count values. It 2163ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// expects to be able to rediscover the trip count each time it is 2164ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// needed, and it does this using a simple analyis that only succeeds if 2165ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// the loop has a canonical induction variable. 2166ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2167ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// However, when it comes time to generate code, the maximum operation 2168ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// can be quite costly, especially if it's inside of an outer loop. 2169ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 2170ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// This function solves this problem by detecting this type of loop and 2171ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting 2172ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// the instructions for the maximum computation. 2173ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman/// 21742781f30eac8647552638246c28ab07dd0fc2c560Dan GohmanICmpInst *LoopStrengthReduce::OptimizeMax(Loop *L, ICmpInst *Cond, 21752781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman IVStrideUse* &CondUse) { 2176ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check that the loop matches the pattern we're looking for. 2177ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (Cond->getPredicate() != CmpInst::ICMP_EQ && 2178ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->getPredicate() != CmpInst::ICMP_NE) 2179ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 2180ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2181ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); 2182ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!Sel || !Sel->hasOneUse()) return Cond; 2183ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 21840bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *BackedgeTakenCount = SE->getBackedgeTakenCount(L); 218546bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 2186ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 21870bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *One = SE->getIntegerSCEV(1, BackedgeTakenCount->getType()); 2188ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 218946bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman // Add one to the backedge-taken count to get the trip count. 21900bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *IterationCount = SE->getAddExpr(BackedgeTakenCount, One); 2191ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2192ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check for a max calculation that matches the pattern. 21932781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman if (!isa<SCEVSMaxExpr>(IterationCount) && !isa<SCEVUMaxExpr>(IterationCount)) 21942781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman return Cond; 21952781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman const SCEVNAryExpr *Max = cast<SCEVNAryExpr>(IterationCount); 21962781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman if (Max != SE->getSCEV(Sel)) return Cond; 2197ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2198224a19c490da17c6ed75245a4a7ef49cd25f6cdfDan Gohman // To handle a max with more than two operands, this optimization would 21994d1c1efd800727165c12c2d186a5cb0b4f5834abDan Gohman // require additional checking and setup. 22002781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman if (Max->getNumOperands() != 2) 22014d1c1efd800727165c12c2d186a5cb0b4f5834abDan Gohman return Cond; 22024d1c1efd800727165c12c2d186a5cb0b4f5834abDan Gohman 22030bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *MaxLHS = Max->getOperand(0); 22040bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *MaxRHS = Max->getOperand(1); 22052781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman if (!MaxLHS || MaxLHS != One) return Cond; 2206ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2207ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check the relevant induction variable for conformance to 2208ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // the pattern. 22090bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *IV = SE->getSCEV(Cond->getOperand(0)); 2210890f92b744fb074465bc2b7006ee753a181f62a4Dan Gohman const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 2211ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!AR || !AR->isAffine() || 2212ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman AR->getStart() != One || 2213ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman AR->getStepRecurrence(*SE) != One) 2214ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return Cond; 2215ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2216bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman assert(AR->getLoop() == L && 2217bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman "Loop condition operand is an addrec in a different loop!"); 2218bc10b8c6c7e85e44d8231dfb2fb41a60300857e3Dan Gohman 2219ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Check the right operand of the select, and remember it, as it will 2220ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // be used in the new comparison instruction. 2221ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Value *NewRHS = 0; 22222781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman if (SE->getSCEV(Sel->getOperand(1)) == MaxRHS) 2223ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman NewRHS = Sel->getOperand(1); 22242781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman else if (SE->getSCEV(Sel->getOperand(2)) == MaxRHS) 2225ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman NewRHS = Sel->getOperand(2); 2226ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman if (!NewRHS) return Cond; 2227ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 22282781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman // Determine the new comparison opcode. It may be signed or unsigned, 22292781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman // and the original comparison may be either equality or inequality. 22302781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman CmpInst::Predicate Pred = 22312781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman isa<SCEVSMaxExpr>(Max) ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT; 22322781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman if (Cond->getPredicate() == CmpInst::ICMP_EQ) 22332781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman Pred = CmpInst::getInversePredicate(Pred); 22342781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman 2235ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Ok, everything looks ok to change the condition into an SLT or SGE and 2236ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // delete the max calculation. 2237ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman ICmpInst *NewCond = 2238333c40096561218bc3597cf153c0a3895274414cOwen Anderson new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp"); 2239ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2240ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // Delete the max calculation instructions. 2241ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Cond->replaceAllUsesWith(NewCond); 224281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman CondUse->setUser(NewCond); 2243ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); 224481db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman Cond->eraseFromParent(); 2245586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman Sel->eraseFromParent(); 224635738ac150afafe2359268d4b2169498c6c98c5fDan Gohman if (Cmp->use_empty()) 2247586b7b75479990475ed5a03fae72d44e0b5ca8e7Dan Gohman Cmp->eraseFromParent(); 2248ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman return NewCond; 2249ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman} 2250ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2251a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel/// OptimizeShadowIV - If IV is used in a int-to-float cast 2252a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel/// inside the loop then try to eliminate the cast opeation. 2253a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patelvoid LoopStrengthReduce::OptimizeShadowIV(Loop *L) { 2254a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 22550bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman const SCEV *BackedgeTakenCount = SE->getBackedgeTakenCount(L); 225646bdfb0e6bb9de86b19562fc52fddefd7014cf73Dan Gohman if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) 2257a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel return; 2258e922c0201916e0b980ab3cfe91e1413e68d55647Owen Anderson 225981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (unsigned Stride = 0, e = IU->StrideOrder.size(); Stride != e; 2260a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ++Stride) { 22610bba49cebc50c7bd4662a4807bcb3ee7f42cb470Dan Gohman std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI = 226281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IU->IVUsesByStride.find(IU->StrideOrder[Stride]); 226381db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman assert(SI != IU->IVUsesByStride.end() && "Stride doesn't exist!"); 2264a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!isa<SCEVConstant>(SI->first)) 2265a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2266a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 226781db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman for (ilist<IVStrideUse>::iterator UI = SI->second->Users.begin(), 226881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman E = SI->second->Users.end(); UI != E; /* empty */) { 226981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman ilist<IVStrideUse>::iterator CandidateUI = UI; 2270541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel ++UI; 227181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman Instruction *ShadowUse = CandidateUI->getUser(); 2272a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const Type *DestTy = NULL; 2273a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2274a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* If shadow use is a int->float cast then insert a second IV 2275541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel to eliminate this cast. 2276a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2277a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned i = 0; i < n; ++i) 2278a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel foo((double)i); 2279a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2280541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel is transformed into 2281a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2282a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel double d = 0.0; 2283a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel for (unsigned i = 0; i < n; ++i, ++d) 2284a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel foo(d); 2285a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel */ 228681db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) 2287a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel DestTy = UCast->getDestTy(); 228881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) 2289a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel DestTy = SCast->getDestTy(); 229018bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (!DestTy) continue; 229118bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel 229218bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (TLI) { 22935792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // If target does not support DestTy natively then do not apply 22945792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // this transformation. 2295e50ed30282bb5b4a9ed952580523f2dda16215acOwen Anderson EVT DVT = TLI->getValueType(DestTy); 229618bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel if (!TLI->isTypeLegal(DVT)) continue; 229718bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel } 229818bb2788a0edc0ec1c373465429743892c8d5fbeDevang Patel 2299a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); 2300a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!PH) continue; 2301a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (PH->getNumIncomingValues() != 2) continue; 2302a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2303a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel const Type *SrcTy = PH->getType(); 2304a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel int Mantissa = DestTy->getFPMantissaWidth(); 2305a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Mantissa == -1) continue; 2306af79fb5f47b0088c6a8973a7fdbaea96973a429dDan Gohman if ((int)SE->getTypeSizeInBits(SrcTy) > Mantissa) 2307a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2308a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2309a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel unsigned Entry, Latch; 2310a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { 2311a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Entry = 0; 2312a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Latch = 1; 2313a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } else { 2314a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Entry = 1; 2315a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel Latch = 0; 2316a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2317a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2318a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); 2319a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!Init) continue; 23206f83c9c6ef0e7f79825a0a8f22941815e4b684c7Owen Anderson Constant *NewInit = ConstantFP::get(DestTy, Init->getZExtValue()); 2321a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2322a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator *Incr = 2323a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); 2324a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!Incr) continue; 2325a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Incr->getOpcode() != Instruction::Add 2326a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel && Incr->getOpcode() != Instruction::Sub) 2327a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2328a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2329a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Initialize new IV, double d = 0.0 in above example. */ 2330a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ConstantInt *C = NULL; 2331a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (Incr->getOperand(0) == PH) 2332a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel C = dyn_cast<ConstantInt>(Incr->getOperand(1)); 2333a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel else if (Incr->getOperand(1) == PH) 2334a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel C = dyn_cast<ConstantInt>(Incr->getOperand(0)); 2335a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel else 2336a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel continue; 2337a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2338a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel if (!C) continue; 2339a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2340a8225083ba7b2368cec0ced232e43dc77d5f6f0aDan Gohman // Ignore negative constants, as the code below doesn't handle them 2341a8225083ba7b2368cec0ced232e43dc77d5f6f0aDan Gohman // correctly. TODO: Remove this restriction. 2342a8225083ba7b2368cec0ced232e43dc77d5f6f0aDan Gohman if (!C->getValue().isStrictlyPositive()) continue; 2343a8225083ba7b2368cec0ced232e43dc77d5f6f0aDan Gohman 2344a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Add new PHINode. */ 2345a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel PHINode *NewPH = PHINode::Create(DestTy, "IV.S.", PH); 2346a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2347541532724e29203e28c2fe0136cf6eabd49d4532Devang Patel /* create new increment. '++d' in above example. */ 23486f83c9c6ef0e7f79825a0a8f22941815e4b684c7Owen Anderson Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue()); 2349a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel BinaryOperator *NewIncr = 2350ae3a0be92e33bc716722aa600983fc1535acb122Dan Gohman BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ? 2351ae3a0be92e33bc716722aa600983fc1535acb122Dan Gohman Instruction::FAdd : Instruction::FSub, 2352a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH, CFP, "IV.S.next.", Incr); 2353a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2354a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); 2355a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); 2356a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2357a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel /* Remove cast operation */ 2358a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ShadowUse->replaceAllUsesWith(NewPH); 2359a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel ShadowUse->eraseFromParent(); 2360a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel NumShadow++; 2361a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel break; 2362a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2363a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel } 2364a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel} 2365a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel 2366e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// OptimizeIndvars - Now that IVUsesByStride is set up with all of the indvar 2367e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// uses in the loop, look to see if we can eliminate some, in favor of using 2368e3a6165423503f550b4f60e06469ad6420a585c9Dan Gohman/// common indvars for the different uses. 2369010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattnervoid LoopStrengthReduce::OptimizeIndvars(Loop *L) { 2370010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // TODO: implement optzns here. 2371010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2372a0b3909d432e9f2c79aee8ec3133f9b9ec71dc1aDevang Patel OptimizeShadowIV(L); 23732d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng} 23742d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng 2375586f69a11881d828c056ce017b3fb432341d9657Evan Chengbool LoopStrengthReduce::StrideMightBeShared(const SCEV* Stride, Loop *L, 2376586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool CheckPreInc) { 2377586f69a11881d828c056ce017b3fb432341d9657Evan Cheng int64_t SInt = cast<SCEVConstant>(Stride)->getValue()->getSExtValue(); 2378586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (unsigned i = 0, e = IU->StrideOrder.size(); i != e; ++i) { 2379586f69a11881d828c056ce017b3fb432341d9657Evan Cheng std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI = 2380586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride.find(IU->StrideOrder[i]); 2381586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *Share = SI->first; 2382586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!isa<SCEVConstant>(SI->first) || Share == Stride) 2383586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2384586f69a11881d828c056ce017b3fb432341d9657Evan Cheng int64_t SSInt = cast<SCEVConstant>(Share)->getValue()->getSExtValue(); 2385586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (SSInt == SInt) 2386586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return true; // This can definitely be reused. 2387586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (unsigned(abs64(SSInt)) < SInt || (SSInt % SInt) != 0) 2388586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2389586f69a11881d828c056ce017b3fb432341d9657Evan Cheng int64_t Scale = SSInt / SInt; 2390586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool AllUsesAreAddresses = true; 2391586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool AllUsesAreOutsideLoop = true; 2392586f69a11881d828c056ce017b3fb432341d9657Evan Cheng std::vector<BasedUser> UsersToProcess; 2393586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *CommonExprs = CollectIVUsers(SI->first, *SI->second, L, 2394586f69a11881d828c056ce017b3fb432341d9657Evan Cheng AllUsesAreAddresses, 2395586f69a11881d828c056ce017b3fb432341d9657Evan Cheng AllUsesAreOutsideLoop, 2396586f69a11881d828c056ce017b3fb432341d9657Evan Cheng UsersToProcess); 2397586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (AllUsesAreAddresses && 2398586f69a11881d828c056ce017b3fb432341d9657Evan Cheng ValidScale(!CommonExprs->isZero(), Scale, UsersToProcess)) { 2399586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!CheckPreInc) 2400586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return true; 2401586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Any pre-inc iv use? 2402586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVUsersOfOneStride &StrideUses = *IU->IVUsesByStride[Share]; 2403586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (ilist<IVStrideUse>::iterator I = StrideUses.Users.begin(), 2404586f69a11881d828c056ce017b3fb432341d9657Evan Cheng E = StrideUses.Users.end(); I != E; ++I) { 2405586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!I->isUseOfPostIncrementedValue()) 2406586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return true; 2407586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2408586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2409586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2410586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2411586f69a11881d828c056ce017b3fb432341d9657Evan Cheng} 2412586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2413586f69a11881d828c056ce017b3fb432341d9657Evan Cheng/// isUsedByExitBranch - Return true if icmp is used by a loop terminating 2414586f69a11881d828c056ce017b3fb432341d9657Evan Cheng/// conditional branch or it's and / or with other conditions before being used 2415586f69a11881d828c056ce017b3fb432341d9657Evan Cheng/// as the condition. 2416586f69a11881d828c056ce017b3fb432341d9657Evan Chengstatic bool isUsedByExitBranch(ICmpInst *Cond, Loop *L) { 2417586f69a11881d828c056ce017b3fb432341d9657Evan Cheng BasicBlock *CondBB = Cond->getParent(); 2418586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!L->isLoopExiting(CondBB)) 2419586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2420586f69a11881d828c056ce017b3fb432341d9657Evan Cheng BranchInst *TermBr = dyn_cast<BranchInst>(CondBB->getTerminator()); 2421586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!TermBr || !TermBr->isConditional()) 2422586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2423586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2424586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Value *User = *Cond->use_begin(); 2425586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Instruction *UserInst = dyn_cast<Instruction>(User); 2426586f69a11881d828c056ce017b3fb432341d9657Evan Cheng while (UserInst && 2427586f69a11881d828c056ce017b3fb432341d9657Evan Cheng (UserInst->getOpcode() == Instruction::And || 2428586f69a11881d828c056ce017b3fb432341d9657Evan Cheng UserInst->getOpcode() == Instruction::Or)) { 2429586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!UserInst->hasOneUse() || UserInst->getParent() != CondBB) 2430586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2431586f69a11881d828c056ce017b3fb432341d9657Evan Cheng User = *User->use_begin(); 2432586f69a11881d828c056ce017b3fb432341d9657Evan Cheng UserInst = dyn_cast<Instruction>(User); 2433586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2434586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return User == TermBr; 2435586f69a11881d828c056ce017b3fb432341d9657Evan Cheng} 2436586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2437586f69a11881d828c056ce017b3fb432341d9657Evan Chengstatic bool ShouldCountToZero(ICmpInst *Cond, IVStrideUse* &CondUse, 2438586f69a11881d828c056ce017b3fb432341d9657Evan Cheng ScalarEvolution *SE, Loop *L, 2439586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const TargetLowering *TLI = 0) { 2440586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!L->contains(Cond->getParent())) 2441586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2442586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2443586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!isa<SCEVConstant>(CondUse->getOffset())) 2444586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2445586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2446586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Handle only tests for equality for the moment. 2447586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!Cond->isEquality() || !Cond->hasOneUse()) 2448586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2449586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!isUsedByExitBranch(Cond, L)) 2450586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2451586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2452586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Value *CondOp0 = Cond->getOperand(0); 2453586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *IV = SE->getSCEV(CondOp0); 2454586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); 2455586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!AR || !AR->isAffine()) 2456586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2457586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2458586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEVConstant *SC = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*SE)); 2459586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!SC || SC->getValue()->getSExtValue() < 0) 2460586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If it's already counting down, don't do anything. 2461586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2462586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2463586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If the RHS of the comparison is not an loop invariant, the rewrite 2464586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // cannot be done. Also bail out if it's already comparing against a zero. 2465586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If we are checking this before cmp stride optimization, check if it's 2466586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // comparing against a already legal immediate. 2467586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Value *RHS = Cond->getOperand(1); 2468586f69a11881d828c056ce017b3fb432341d9657Evan Cheng ConstantInt *RHSC = dyn_cast<ConstantInt>(RHS); 2469586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!L->isLoopInvariant(RHS) || 2470586f69a11881d828c056ce017b3fb432341d9657Evan Cheng (RHSC && RHSC->isZero()) || 2471586f69a11881d828c056ce017b3fb432341d9657Evan Cheng (RHSC && TLI && TLI->isLegalICmpImmediate(RHSC->getSExtValue()))) 2472586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2473586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2474586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Make sure the IV is only used for counting. Value may be preinc or 2475586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // postinc; 2 uses in either case. 2476586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!CondOp0->hasNUses(2)) 2477586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2478586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2479586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return true; 2480586f69a11881d828c056ce017b3fb432341d9657Evan Cheng} 2481586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 24822d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng/// OptimizeLoopTermCond - Change loop terminating condition to use the 24832d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Cheng/// postinc iv when possible. 24842d85052f2bae5a5a31c03017c48ca8a9eba1453cEvan Chengvoid LoopStrengthReduce::OptimizeLoopTermCond(Loop *L) { 2485010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Finally, get the terminating condition for the loop if possible. If we 2486010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // can, we want to change it to use a post-incremented version of its 248798d9811db263ee040d9a6a69ef9e1c4fdc8c219dChris Lattner // induction variable, to allow coalescing the live ranges for the IV into 2488010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // one register value. 24895792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng BasicBlock *LatchBlock = L->getLoopLatch(); 24905392dce8815bc8975076a6b4f0079359878e2b7aDan Gohman BasicBlock *ExitingBlock = L->getExitingBlock(); 2491e922c0201916e0b980ab3cfe91e1413e68d55647Owen Anderson 24925392dce8815bc8975076a6b4f0079359878e2b7aDan Gohman if (!ExitingBlock) 24935792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // Multiple exits, just look at the exit in the latch block if there is one. 24945392dce8815bc8975076a6b4f0079359878e2b7aDan Gohman ExitingBlock = LatchBlock; 24955392dce8815bc8975076a6b4f0079359878e2b7aDan Gohman BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); 24965792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (!TermBr) 24975792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng return; 24985792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition())) 2499010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner return; 2500010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2501010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Search IVUsesByStride to find Cond's IVUse if there is one. 2502010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner IVStrideUse *CondUse = 0; 2503586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *CondStride = 0; 25045792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); 2505c677de2713646ab6d8200cd71613f6b4ae9885fbDevang Patel if (!FindIVUserForCond(Cond, CondUse, CondStride)) 2506586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return; 2507cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng 2508586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool UsePostInc = true; 25095392dce8815bc8975076a6b4f0079359878e2b7aDan Gohman if (ExitingBlock != LatchBlock) { 2510586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (Cond->hasOneUse()) { 25115792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // See below, we don't want the condition to be cloned. 25125792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 2513586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If exiting block is the latch block, we know it's safe and profitable 2514586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // to transform the icmp to use post-inc iv. Otherwise do so only if it 2515586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // would not reuse another iv and its iv would be reused by other uses. 2516586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // We are optimizing for the case where the icmp is the only use of the 2517586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // iv. 2518586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVUsersOfOneStride &StrideUses = *IU->IVUsesByStride[CondStride]; 2519586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (ilist<IVStrideUse>::iterator I = StrideUses.Users.begin(), 2520586f69a11881d828c056ce017b3fb432341d9657Evan Cheng E = StrideUses.Users.end(); I != E; ++I) { 2521586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (I->getUser() == Cond) 25225792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng continue; 2523586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!I->isUseOfPostIncrementedValue()) { 2524586f69a11881d828c056ce017b3fb432341d9657Evan Cheng UsePostInc = false; 2525586f69a11881d828c056ce017b3fb432341d9657Evan Cheng break; 2526586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 25275792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng } 25285792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng } 25295792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 2530586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If iv for the stride might be shared and any of the users use pre-inc iv 2531586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // might be used, then it's not safe to use post-inc iv. 2532586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (UsePostInc && 2533586f69a11881d828c056ce017b3fb432341d9657Evan Cheng isa<SCEVConstant>(CondStride) && 2534586f69a11881d828c056ce017b3fb432341d9657Evan Cheng StrideMightBeShared(CondStride, L, true)) 2535586f69a11881d828c056ce017b3fb432341d9657Evan Cheng UsePostInc = false; 25365792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng } 25375792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 25382781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman // If the trip count is computed in terms of a max (due to ScalarEvolution 2539ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman // being unable to find a sufficient guard, for example), change the loop 25402781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman // comparison to use SLT or ULT instead of NE. 25412781f30eac8647552638246c28ab07dd0fc2c560Dan Gohman Cond = OptimizeMax(L, Cond, CondUse); 2542ad7321f58a485ae80fe3da1f400aa695e250b1a8Dan Gohman 2543cdf43b1fadac74ecf1cc858e72bde877a10ceca1Evan Cheng // If possible, change stride and operands of the compare instruction to 2544586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // eliminate one stride. However, avoid rewriting the compare instruction with 2545586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // an iv of new stride if it's likely the new stride uses will be rewritten 2546586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // using the stride of the compare instruction. 2547586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (ExitingBlock == LatchBlock && isa<SCEVConstant>(CondStride)) { 2548586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If the condition stride is a constant and it's the only use, we might 2549586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // want to optimize it first by turning it to count toward zero. 2550586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!StrideMightBeShared(CondStride, L, false) && 2551586f69a11881d828c056ce017b3fb432341d9657Evan Cheng !ShouldCountToZero(Cond, CondUse, SE, L, TLI)) 2552586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Cond = ChangeCompareStride(L, Cond, CondUse, CondStride); 2553586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2554586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2555586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!UsePostInc) 2556586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return; 2557010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2558010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // It's possible for the setcc instruction to be anywhere in the loop, and 2559010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // possible for it to have multiple users. If it is not immediately before 2560010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // the latch block branch, move it. 2561010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (&*++BasicBlock::iterator(Cond) != (Instruction*)TermBr) { 2562010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner if (Cond->hasOneUse()) { // Condition has a single use, just move it. 2563010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner Cond->moveBefore(TermBr); 2564010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } else { 2565010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Otherwise, clone the terminating condition and insert into the loopend. 25666776064d190701c5bae4d5403939eed2e480d1cdNick Lewycky Cond = cast<ICmpInst>(Cond->clone()); 2567010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner Cond->setName(L->getHeader()->getName() + ".termcond"); 2568010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner LatchBlock->getInstList().insert(TermBr, Cond); 2569010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2570010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // Clone the IVUse, as the old use still exists! 2571586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride[CondStride]->addUser(CondUse->getOffset(), Cond, 25724e8a98519ebf75ed47456ea42706aaa57ecd2c27Dan Gohman CondUse->getOperandValToReplace()); 2573586f69a11881d828c056ce017b3fb432341d9657Evan Cheng CondUse = &IU->IVUsesByStride[CondStride]->Users.back(); 2574010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2575010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner } 2576010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 2577010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // If we get to here, we know that we can transform the setcc instruction to 257898d9811db263ee040d9a6a69ef9e1c4fdc8c219dChris Lattner // use the post-incremented version of the IV, allowing us to coalesce the 2579010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner // live ranges for the IV correctly. 2580586f69a11881d828c056ce017b3fb432341d9657Evan Cheng CondUse->setOffset(SE->getMinusSCEV(CondUse->getOffset(), CondStride)); 258181db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman CondUse->setIsUseOfPostIncrementedValue(true); 25821ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng Changed = true; 25835792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 25845792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng ++NumLoopCond; 2585010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner} 2586169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 2587586f69a11881d828c056ce017b3fb432341d9657Evan Chengbool LoopStrengthReduce::OptimizeLoopCountIVOfStride(const SCEV* &Stride, 2588586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVStrideUse* &CondUse, 2589586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Loop *L) { 259081ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // If the only use is an icmp of an loop exiting conditional branch, then 259181ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // attempts the optimization. 2592586f69a11881d828c056ce017b3fb432341d9657Evan Cheng BasedUser User = BasedUser(*CondUse, SE); 2593586f69a11881d828c056ce017b3fb432341d9657Evan Cheng assert(isa<ICmpInst>(User.Inst) && "Expecting an ICMPInst!"); 2594586f69a11881d828c056ce017b3fb432341d9657Evan Cheng ICmpInst *Cond = cast<ICmpInst>(User.Inst); 2595c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2596586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Less strict check now that compare stride optimization is done. 2597586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!ShouldCountToZero(Cond, CondUse, SE, L)) 2598586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2599c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2600586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Value *CondOp0 = Cond->getOperand(0); 2601586f69a11881d828c056ce017b3fb432341d9657Evan Cheng PHINode *PHIExpr = dyn_cast<PHINode>(CondOp0); 260281ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Instruction *Incr; 2603586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!PHIExpr) { 260481ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // Value tested is postinc. Find the phi node. 260581ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Incr = dyn_cast<BinaryOperator>(CondOp0); 2606586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // FIXME: Just use User.OperandValToReplace here? 260781ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng if (!Incr || Incr->getOpcode() != Instruction::Add) 2608586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2609c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2610586f69a11881d828c056ce017b3fb432341d9657Evan Cheng PHIExpr = dyn_cast<PHINode>(Incr->getOperand(0)); 261181ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng if (!PHIExpr) 2612586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2613c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // 1 use for preinc value, the increment. 2614586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!PHIExpr->hasOneUse()) 2615586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 261681ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng } else { 261781ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng assert(isa<PHINode>(CondOp0) && 261881ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng "Unexpected loop exiting counting instruction sequence!"); 261981ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng PHIExpr = cast<PHINode>(CondOp0); 262081ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // Value tested is preinc. Find the increment. 262181ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // A CmpInst is not a BinaryOperator; we depend on this. 262281ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Instruction::use_iterator UI = PHIExpr->use_begin(); 262381ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Incr = dyn_cast<BinaryOperator>(UI); 262481ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng if (!Incr) 262581ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Incr = dyn_cast<BinaryOperator>(++UI); 262681ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // One use for postinc value, the phi. Unnecessarily conservative? 262781ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng if (!Incr || !Incr->hasOneUse() || Incr->getOpcode() != Instruction::Add) 2628586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return false; 2629c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen } 2630c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2631c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // Replace the increment with a decrement. 2632586f69a11881d828c056ce017b3fb432341d9657Evan Cheng DEBUG(errs() << "LSR: Examining use "); 263381ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng DEBUG(WriteAsOperand(errs(), CondOp0, /*PrintType=*/false)); 2634586f69a11881d828c056ce017b3fb432341d9657Evan Cheng DEBUG(errs() << " in Inst: " << *Cond << '\n'); 263581ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng BinaryOperator *Decr = BinaryOperator::Create(Instruction::Sub, 263681ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Incr->getOperand(0), Incr->getOperand(1), "tmp", Incr); 263781ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Incr->replaceAllUsesWith(Decr); 263881ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Incr->eraseFromParent(); 2639c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2640c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // Substitute endval-startval for the original startval, and 0 for the 2641c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // original endval. Since we're only testing for equality this is OK even 2642c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen // if the computation wraps around. 2643c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen BasicBlock *Preheader = L->getLoopPreheader(); 2644c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen Instruction *PreInsertPt = Preheader->getTerminator(); 264581ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng unsigned InBlock = L->contains(PHIExpr->getIncomingBlock(0)) ? 1 : 0; 264681ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Value *StartVal = PHIExpr->getIncomingValue(InBlock); 264781ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng Value *EndVal = Cond->getOperand(1); 264881ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng DEBUG(errs() << " Optimize loop counting iv to count down [" 264981ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng << *EndVal << " .. " << *StartVal << "]\n"); 265081ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng 265181ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // FIXME: check for case where both are constant. 2652eed707b1e6097aac2bb6b3d47271f6300ace7f2eOwen Anderson Constant* Zero = ConstantInt::get(Cond->getOperand(1)->getType(), 0); 265381ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng BinaryOperator *NewStartVal = BinaryOperator::Create(Instruction::Sub, 265481ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng EndVal, StartVal, "tmp", PreInsertPt); 265581ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng PHIExpr->setIncomingValue(InBlock, NewStartVal); 2656c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen Cond->setOperand(1, Zero); 265781ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng DEBUG(errs() << " New icmp: " << *Cond << "\n"); 2658c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2659586f69a11881d828c056ce017b3fb432341d9657Evan Cheng int64_t SInt = cast<SCEVConstant>(Stride)->getValue()->getSExtValue(); 2660586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *NewStride = 0; 2661586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool Found = false; 2662586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (unsigned i = 0, e = IU->StrideOrder.size(); i != e; ++i) { 2663586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *OldStride = IU->StrideOrder[i]; 2664586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(OldStride)) 2665586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (SC->getValue()->getSExtValue() == -SInt) { 2666586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Found = true; 2667586f69a11881d828c056ce017b3fb432341d9657Evan Cheng NewStride = OldStride; 2668586f69a11881d828c056ce017b3fb432341d9657Evan Cheng break; 2669586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2670586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2671586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2672586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!Found) 2673586f69a11881d828c056ce017b3fb432341d9657Evan Cheng NewStride = SE->getIntegerSCEV(-SInt, Stride->getType()); 2674586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->AddUser(NewStride, CondUse->getOffset(), Cond, Cond->getOperand(0)); 2675586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride[Stride]->removeUser(CondUse); 2676586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2677586f69a11881d828c056ce017b3fb432341d9657Evan Cheng CondUse = &IU->IVUsesByStride[NewStride]->Users.back(); 2678586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Stride = NewStride; 2679586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 268081ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng ++NumCountZero; 2681586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2682586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return true; 2683586f69a11881d828c056ce017b3fb432341d9657Evan Cheng} 2684586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2685586f69a11881d828c056ce017b3fb432341d9657Evan Cheng/// OptimizeLoopCountIV - If, after all sharing of IVs, the IV used for deciding 2686586f69a11881d828c056ce017b3fb432341d9657Evan Cheng/// when to exit the loop is used only for that purpose, try to rearrange things 2687586f69a11881d828c056ce017b3fb432341d9657Evan Cheng/// so it counts down to a test against zero. 2688586f69a11881d828c056ce017b3fb432341d9657Evan Chengbool LoopStrengthReduce::OptimizeLoopCountIV(Loop *L) { 2689586f69a11881d828c056ce017b3fb432341d9657Evan Cheng bool ThisChanged = false; 2690586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (unsigned i = 0, e = IU->StrideOrder.size(); i != e; ++i) { 2691586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *Stride = IU->StrideOrder[i]; 2692586f69a11881d828c056ce017b3fb432341d9657Evan Cheng std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI = 2693586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride.find(Stride); 2694586f69a11881d828c056ce017b3fb432341d9657Evan Cheng assert(SI != IU->IVUsesByStride.end() && "Stride doesn't exist!"); 2695586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // FIXME: Generalize to non-affine IV's. 2696586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!SI->first->isLoopInvariant(L)) 2697586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2698586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // If stride is a constant and it has an icmpinst use, check if we can 2699586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // optimize the loop to count down. 2700586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (isa<SCEVConstant>(Stride) && SI->second->Users.size() == 1) { 2701586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Instruction *User = SI->second->Users.begin()->getUser(); 2702586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!isa<ICmpInst>(User)) 2703586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2704586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *CondStride = Stride; 2705586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IVStrideUse *Use = &*SI->second->Users.begin(); 2706586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!OptimizeLoopCountIVOfStride(CondStride, Use, L)) 2707586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2708586f69a11881d828c056ce017b3fb432341d9657Evan Cheng ThisChanged = true; 2709586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2710586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // Now check if it's possible to reuse this iv for other stride uses. 2711586f69a11881d828c056ce017b3fb432341d9657Evan Cheng for (unsigned j = 0, ee = IU->StrideOrder.size(); j != ee; ++j) { 2712586f69a11881d828c056ce017b3fb432341d9657Evan Cheng const SCEV *SStride = IU->StrideOrder[j]; 2713586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (SStride == CondStride) 2714586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2715586f69a11881d828c056ce017b3fb432341d9657Evan Cheng std::map<const SCEV *, IVUsersOfOneStride *>::iterator SII = 2716586f69a11881d828c056ce017b3fb432341d9657Evan Cheng IU->IVUsesByStride.find(SStride); 2717586f69a11881d828c056ce017b3fb432341d9657Evan Cheng assert(SII != IU->IVUsesByStride.end() && "Stride doesn't exist!"); 2718586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // FIXME: Generalize to non-affine IV's. 2719586f69a11881d828c056ce017b3fb432341d9657Evan Cheng if (!SII->first->isLoopInvariant(L)) 2720586f69a11881d828c056ce017b3fb432341d9657Evan Cheng continue; 2721586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // FIXME: Rewrite other stride using CondStride. 2722586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2723586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2724586f69a11881d828c056ce017b3fb432341d9657Evan Cheng } 2725586f69a11881d828c056ce017b3fb432341d9657Evan Cheng 2726586f69a11881d828c056ce017b3fb432341d9657Evan Cheng Changed |= ThisChanged; 2727586f69a11881d828c056ce017b3fb432341d9657Evan Cheng return ThisChanged; 2728c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen} 2729c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 27300f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patelbool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager &LPM) { 2731eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 273281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman IU = &getAnalysis<IVUsers>(); 27330f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel LI = &getAnalysis<LoopInfo>(); 2734b7d9dfc7ba4ae1ae9482eee62b1912b40dc64f42Devang Patel DT = &getAnalysis<DominatorTree>(); 27350f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel SE = &getAnalysis<ScalarEvolution>(); 27363fea643fb427e72907b7a40e940d7949d1d57e76Dan Gohman Changed = false; 27370f54dcbf07c69e41ecaa6b4fbf0d94956d8e9ff5Devang Patel 273803e896bd6073efc4523d8bcd0239d6ed62126db7Dan Gohman // If LoopSimplify form is not available, stay out of trouble. 273903e896bd6073efc4523d8bcd0239d6ed62126db7Dan Gohman if (!L->getLoopPreheader() || !L->getLoopLatch()) 274003e896bd6073efc4523d8bcd0239d6ed62126db7Dan Gohman return false; 274103e896bd6073efc4523d8bcd0239d6ed62126db7Dan Gohman 274281db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman if (!IU->IVUsesByStride.empty()) { 2743460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar DEBUG(errs() << "\nLSR on \"" << L->getHeader()->getParent()->getName() 2744460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar << "\" "; 2745460f656475738d1a95a6be95346908ce1597df25Daniel Dunbar L->dump()); 274680b0f8c0628d848d36f04440687b61e3b2d3dff1Dan Gohman 2747f7912df4cbdb44aeac9ac9907c192dfc1e22646dDan Gohman // Sort the StrideOrder so we process larger strides first. 274881db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman std::stable_sort(IU->StrideOrder.begin(), IU->StrideOrder.end(), 274981db61a2e6d3c95a2738c3559a108e05e9d7a05aDan Gohman StrideCompare(SE)); 2750f7912df4cbdb44aeac9ac9907c192dfc1e22646dDan Gohman 27511ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Optimize induction variables. Some indvar uses can be transformed to use 27521ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // strides that will be needed for other purposes. A common example of this 27531ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // is the exit test for the loop, which can often be rewritten to use the 27541ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // computation of some other indvar to decide when to terminate the loop. 27551ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng OptimizeIndvars(L); 2756010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 27575792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // Change loop terminating condition to use the postinc iv when possible 27585792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng // and optimize loop terminating compare. FIXME: Move this after 2759586f69a11881d828c056ce017b3fb432341d9657Evan Cheng // StrengthReduceIVUsersOfStride? 27605792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng OptimizeLoopTermCond(L); 27615792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng 2762ce174f8e15774546e9bd276f17cb4b7c74274a93Dan Gohman // FIXME: We can shrink overlarge IV's here. e.g. if the code has 27634221ae84ef45c2099cbb7a61625e8260f37df275Dan Gohman // computation in i64 values and the target doesn't support i64, demote 27644221ae84ef45c2099cbb7a61625e8260f37df275Dan Gohman // the computation to 32-bit if safe. 2765010de25f42dabbb7e0a2fe8b42aecc4285362e0cChris Lattner 27661ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // FIXME: Attempt to reuse values across multiple IV's. In particular, we 27671ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // could have something like "for(i) { foo(i*8); bar(i*16) }", which should 27681ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // be codegened as "for (j = 0;; j+=8) { foo(j); bar(j+j); }" on X86/PPC. 27691ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // Need to be careful that IV's are all the same type. Only works for 27701ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // intptr_t indvars. 2771169974856781a1ce27af9ce6220c390b20c9e6ddNate Begeman 27721ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng // IVsByStride keeps IVs for one particular loop. 27731ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng assert(IVsByStride.empty() && "Stale entries in IVsByStride?"); 27741ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng 2775586f69a11881d828c056ce017b3fb432341d9657Evan Cheng StrengthReduceIVUsers(L); 2776eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 277781ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // After all sharing is done, see if we can adjust the loop to test against 277881ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng // zero instead of counting up to a maximum. This is usually faster. 2779586f69a11881d828c056ce017b3fb432341d9657Evan Cheng OptimizeLoopCountIV(L); 278081ebdcf7dd3c182f5fca352b59b336f79d5da23eEvan Cheng } 2781c1acc3f764804d25f70d88f937ef9c460143e0f1Dale Johannesen 2782010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman // We're done analyzing this loop; release all the state we built up for it. 2783010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman IVsByStride.clear(); 27845792f51e12d9c8685399e9857799365854ab5bf6Evan Cheng StrideNoReuse.clear(); 2785010ee2d95516fe13a574bce5d682a8f8997ab60bDan Gohman 2786eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman // Clean up after ourselves 2787afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman if (!DeadInsts.empty()) 2788a68d4ca73e9cd0b19b2a48a2943e16cc0f89da27Chris Lattner DeleteTriviallyDeadInstructions(); 2789eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman 2790afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman // At this point, it is worth checking to see if any recurrence PHIs are also 279135738ac150afafe2359268d4b2169498c6c98c5fDan Gohman // dead, so that we can remove them as well. 279235738ac150afafe2359268d4b2169498c6c98c5fDan Gohman DeleteDeadPHIs(L->getHeader()); 2793afc36a9520971832dfbebc0333593bf5d3098296Dan Gohman 27941ce75dcbbcb6a67904a23b4ec701d1e994767c7eEvan Cheng return Changed; 2795eaa13851a7fe604363577350c5cf65c257c4d41aNate Begeman} 2796