ScalarEvolution.cpp revision 4d3bba5be47ec159ad3489793248acc812d75575
1//===- ScalarEvolution.cpp - Scalar Evolution Analysis ----------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file contains the implementation of the scalar evolution analysis 11// engine, which is used primarily to analyze expressions involving induction 12// variables in loops. 13// 14// There are several aspects to this library. First is the representation of 15// scalar expressions, which are represented as subclasses of the SCEV class. 16// These classes are used to represent certain types of subexpressions that we 17// can handle. We only create one SCEV of a particular shape, so 18// pointer-comparisons for equality are legal. 19// 20// One important aspect of the SCEV objects is that they are never cyclic, even 21// if there is a cycle in the dataflow for an expression (ie, a PHI node). If 22// the PHI node is one of the idioms that we can represent (e.g., a polynomial 23// recurrence) then we represent it directly as a recurrence node, otherwise we 24// represent it as a SCEVUnknown node. 25// 26// In addition to being able to represent expressions of various types, we also 27// have folders that are used to build the *canonical* representation for a 28// particular expression. These folders are capable of using a variety of 29// rewrite rules to simplify the expressions. 30// 31// Once the folders are defined, we can implement the more interesting 32// higher-level code, such as the code that recognizes PHI nodes of various 33// types, computes the execution count of a loop, etc. 34// 35// TODO: We should use these routines and value representations to implement 36// dependence analysis! 37// 38//===----------------------------------------------------------------------===// 39// 40// There are several good references for the techniques used in this analysis. 41// 42// Chains of recurrences -- a method to expedite the evaluation 43// of closed-form functions 44// Olaf Bachmann, Paul S. Wang, Eugene V. Zima 45// 46// On computational properties of chains of recurrences 47// Eugene V. Zima 48// 49// Symbolic Evaluation of Chains of Recurrences for Loop Optimization 50// Robert A. van Engelen 51// 52// Efficient Symbolic Analysis for Optimizing Compilers 53// Robert A. van Engelen 54// 55// Using the chains of recurrences algebra for data dependence testing and 56// induction variable substitution 57// MS Thesis, Johnie Birch 58// 59//===----------------------------------------------------------------------===// 60 61#define DEBUG_TYPE "scalar-evolution" 62#include "llvm/Analysis/ScalarEvolutionExpressions.h" 63#include "llvm/Constants.h" 64#include "llvm/DerivedTypes.h" 65#include "llvm/GlobalVariable.h" 66#include "llvm/GlobalAlias.h" 67#include "llvm/Instructions.h" 68#include "llvm/LLVMContext.h" 69#include "llvm/Operator.h" 70#include "llvm/Analysis/ConstantFolding.h" 71#include "llvm/Analysis/Dominators.h" 72#include "llvm/Analysis/InstructionSimplify.h" 73#include "llvm/Analysis/LoopInfo.h" 74#include "llvm/Analysis/ValueTracking.h" 75#include "llvm/Assembly/Writer.h" 76#include "llvm/Target/TargetData.h" 77#include "llvm/Target/TargetLibraryInfo.h" 78#include "llvm/Support/CommandLine.h" 79#include "llvm/Support/ConstantRange.h" 80#include "llvm/Support/Debug.h" 81#include "llvm/Support/ErrorHandling.h" 82#include "llvm/Support/GetElementPtrTypeIterator.h" 83#include "llvm/Support/InstIterator.h" 84#include "llvm/Support/MathExtras.h" 85#include "llvm/Support/raw_ostream.h" 86#include "llvm/ADT/Statistic.h" 87#include "llvm/ADT/STLExtras.h" 88#include "llvm/ADT/SmallPtrSet.h" 89#include <algorithm> 90using namespace llvm; 91 92STATISTIC(NumArrayLenItCounts, 93 "Number of trip counts computed with array length"); 94STATISTIC(NumTripCountsComputed, 95 "Number of loops with predictable loop counts"); 96STATISTIC(NumTripCountsNotComputed, 97 "Number of loops without predictable loop counts"); 98STATISTIC(NumBruteForceTripCountsComputed, 99 "Number of loops with trip counts computed by force"); 100 101static cl::opt<unsigned> 102MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, 103 cl::desc("Maximum number of iterations SCEV will " 104 "symbolically execute a constant " 105 "derived loop"), 106 cl::init(100)); 107 108INITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution", 109 "Scalar Evolution Analysis", false, true) 110INITIALIZE_PASS_DEPENDENCY(LoopInfo) 111INITIALIZE_PASS_DEPENDENCY(DominatorTree) 112INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo) 113INITIALIZE_PASS_END(ScalarEvolution, "scalar-evolution", 114 "Scalar Evolution Analysis", false, true) 115char ScalarEvolution::ID = 0; 116 117//===----------------------------------------------------------------------===// 118// SCEV class definitions 119//===----------------------------------------------------------------------===// 120 121//===----------------------------------------------------------------------===// 122// Implementation of the SCEV class. 123// 124 125void SCEV::dump() const { 126 print(dbgs()); 127 dbgs() << '\n'; 128} 129 130void SCEV::print(raw_ostream &OS) const { 131 switch (getSCEVType()) { 132 case scConstant: 133 WriteAsOperand(OS, cast<SCEVConstant>(this)->getValue(), false); 134 return; 135 case scTruncate: { 136 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this); 137 const SCEV *Op = Trunc->getOperand(); 138 OS << "(trunc " << *Op->getType() << " " << *Op << " to " 139 << *Trunc->getType() << ")"; 140 return; 141 } 142 case scZeroExtend: { 143 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this); 144 const SCEV *Op = ZExt->getOperand(); 145 OS << "(zext " << *Op->getType() << " " << *Op << " to " 146 << *ZExt->getType() << ")"; 147 return; 148 } 149 case scSignExtend: { 150 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this); 151 const SCEV *Op = SExt->getOperand(); 152 OS << "(sext " << *Op->getType() << " " << *Op << " to " 153 << *SExt->getType() << ")"; 154 return; 155 } 156 case scAddRecExpr: { 157 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this); 158 OS << "{" << *AR->getOperand(0); 159 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i) 160 OS << ",+," << *AR->getOperand(i); 161 OS << "}<"; 162 if (AR->getNoWrapFlags(FlagNUW)) 163 OS << "nuw><"; 164 if (AR->getNoWrapFlags(FlagNSW)) 165 OS << "nsw><"; 166 if (AR->getNoWrapFlags(FlagNW) && 167 !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW))) 168 OS << "nw><"; 169 WriteAsOperand(OS, AR->getLoop()->getHeader(), /*PrintType=*/false); 170 OS << ">"; 171 return; 172 } 173 case scAddExpr: 174 case scMulExpr: 175 case scUMaxExpr: 176 case scSMaxExpr: { 177 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this); 178 const char *OpStr = 0; 179 switch (NAry->getSCEVType()) { 180 case scAddExpr: OpStr = " + "; break; 181 case scMulExpr: OpStr = " * "; break; 182 case scUMaxExpr: OpStr = " umax "; break; 183 case scSMaxExpr: OpStr = " smax "; break; 184 } 185 OS << "("; 186 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 187 I != E; ++I) { 188 OS << **I; 189 if (llvm::next(I) != E) 190 OS << OpStr; 191 } 192 OS << ")"; 193 switch (NAry->getSCEVType()) { 194 case scAddExpr: 195 case scMulExpr: 196 if (NAry->getNoWrapFlags(FlagNUW)) 197 OS << "<nuw>"; 198 if (NAry->getNoWrapFlags(FlagNSW)) 199 OS << "<nsw>"; 200 } 201 return; 202 } 203 case scUDivExpr: { 204 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this); 205 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")"; 206 return; 207 } 208 case scUnknown: { 209 const SCEVUnknown *U = cast<SCEVUnknown>(this); 210 Type *AllocTy; 211 if (U->isSizeOf(AllocTy)) { 212 OS << "sizeof(" << *AllocTy << ")"; 213 return; 214 } 215 if (U->isAlignOf(AllocTy)) { 216 OS << "alignof(" << *AllocTy << ")"; 217 return; 218 } 219 220 Type *CTy; 221 Constant *FieldNo; 222 if (U->isOffsetOf(CTy, FieldNo)) { 223 OS << "offsetof(" << *CTy << ", "; 224 WriteAsOperand(OS, FieldNo, false); 225 OS << ")"; 226 return; 227 } 228 229 // Otherwise just print it normally. 230 WriteAsOperand(OS, U->getValue(), false); 231 return; 232 } 233 case scCouldNotCompute: 234 OS << "***COULDNOTCOMPUTE***"; 235 return; 236 default: break; 237 } 238 llvm_unreachable("Unknown SCEV kind!"); 239} 240 241Type *SCEV::getType() const { 242 switch (getSCEVType()) { 243 case scConstant: 244 return cast<SCEVConstant>(this)->getType(); 245 case scTruncate: 246 case scZeroExtend: 247 case scSignExtend: 248 return cast<SCEVCastExpr>(this)->getType(); 249 case scAddRecExpr: 250 case scMulExpr: 251 case scUMaxExpr: 252 case scSMaxExpr: 253 return cast<SCEVNAryExpr>(this)->getType(); 254 case scAddExpr: 255 return cast<SCEVAddExpr>(this)->getType(); 256 case scUDivExpr: 257 return cast<SCEVUDivExpr>(this)->getType(); 258 case scUnknown: 259 return cast<SCEVUnknown>(this)->getType(); 260 case scCouldNotCompute: 261 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 262 default: 263 llvm_unreachable("Unknown SCEV kind!"); 264 } 265} 266 267bool SCEV::isZero() const { 268 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 269 return SC->getValue()->isZero(); 270 return false; 271} 272 273bool SCEV::isOne() const { 274 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 275 return SC->getValue()->isOne(); 276 return false; 277} 278 279bool SCEV::isAllOnesValue() const { 280 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 281 return SC->getValue()->isAllOnesValue(); 282 return false; 283} 284 285/// isNonConstantNegative - Return true if the specified scev is negated, but 286/// not a constant. 287bool SCEV::isNonConstantNegative() const { 288 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this); 289 if (!Mul) return false; 290 291 // If there is a constant factor, it will be first. 292 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 293 if (!SC) return false; 294 295 // Return true if the value is negative, this matches things like (-42 * V). 296 return SC->getValue()->getValue().isNegative(); 297} 298 299SCEVCouldNotCompute::SCEVCouldNotCompute() : 300 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {} 301 302bool SCEVCouldNotCompute::classof(const SCEV *S) { 303 return S->getSCEVType() == scCouldNotCompute; 304} 305 306const SCEV *ScalarEvolution::getConstant(ConstantInt *V) { 307 FoldingSetNodeID ID; 308 ID.AddInteger(scConstant); 309 ID.AddPointer(V); 310 void *IP = 0; 311 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 312 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V); 313 UniqueSCEVs.InsertNode(S, IP); 314 return S; 315} 316 317const SCEV *ScalarEvolution::getConstant(const APInt& Val) { 318 return getConstant(ConstantInt::get(getContext(), Val)); 319} 320 321const SCEV * 322ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) { 323 IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty)); 324 return getConstant(ConstantInt::get(ITy, V, isSigned)); 325} 326 327SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, 328 unsigned SCEVTy, const SCEV *op, Type *ty) 329 : SCEV(ID, SCEVTy), Op(op), Ty(ty) {} 330 331SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, 332 const SCEV *op, Type *ty) 333 : SCEVCastExpr(ID, scTruncate, op, ty) { 334 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 335 (Ty->isIntegerTy() || Ty->isPointerTy()) && 336 "Cannot truncate non-integer value!"); 337} 338 339SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, 340 const SCEV *op, Type *ty) 341 : SCEVCastExpr(ID, scZeroExtend, op, ty) { 342 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 343 (Ty->isIntegerTy() || Ty->isPointerTy()) && 344 "Cannot zero extend non-integer value!"); 345} 346 347SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, 348 const SCEV *op, Type *ty) 349 : SCEVCastExpr(ID, scSignExtend, op, ty) { 350 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 351 (Ty->isIntegerTy() || Ty->isPointerTy()) && 352 "Cannot sign extend non-integer value!"); 353} 354 355void SCEVUnknown::deleted() { 356 // Clear this SCEVUnknown from various maps. 357 SE->forgetMemoizedResults(this); 358 359 // Remove this SCEVUnknown from the uniquing map. 360 SE->UniqueSCEVs.RemoveNode(this); 361 362 // Release the value. 363 setValPtr(0); 364} 365 366void SCEVUnknown::allUsesReplacedWith(Value *New) { 367 // Clear this SCEVUnknown from various maps. 368 SE->forgetMemoizedResults(this); 369 370 // Remove this SCEVUnknown from the uniquing map. 371 SE->UniqueSCEVs.RemoveNode(this); 372 373 // Update this SCEVUnknown to point to the new value. This is needed 374 // because there may still be outstanding SCEVs which still point to 375 // this SCEVUnknown. 376 setValPtr(New); 377} 378 379bool SCEVUnknown::isSizeOf(Type *&AllocTy) const { 380 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 381 if (VCE->getOpcode() == Instruction::PtrToInt) 382 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 383 if (CE->getOpcode() == Instruction::GetElementPtr && 384 CE->getOperand(0)->isNullValue() && 385 CE->getNumOperands() == 2) 386 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1))) 387 if (CI->isOne()) { 388 AllocTy = cast<PointerType>(CE->getOperand(0)->getType()) 389 ->getElementType(); 390 return true; 391 } 392 393 return false; 394} 395 396bool SCEVUnknown::isAlignOf(Type *&AllocTy) const { 397 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 398 if (VCE->getOpcode() == Instruction::PtrToInt) 399 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 400 if (CE->getOpcode() == Instruction::GetElementPtr && 401 CE->getOperand(0)->isNullValue()) { 402 Type *Ty = 403 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 404 if (StructType *STy = dyn_cast<StructType>(Ty)) 405 if (!STy->isPacked() && 406 CE->getNumOperands() == 3 && 407 CE->getOperand(1)->isNullValue()) { 408 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2))) 409 if (CI->isOne() && 410 STy->getNumElements() == 2 && 411 STy->getElementType(0)->isIntegerTy(1)) { 412 AllocTy = STy->getElementType(1); 413 return true; 414 } 415 } 416 } 417 418 return false; 419} 420 421bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const { 422 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 423 if (VCE->getOpcode() == Instruction::PtrToInt) 424 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 425 if (CE->getOpcode() == Instruction::GetElementPtr && 426 CE->getNumOperands() == 3 && 427 CE->getOperand(0)->isNullValue() && 428 CE->getOperand(1)->isNullValue()) { 429 Type *Ty = 430 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 431 // Ignore vector types here so that ScalarEvolutionExpander doesn't 432 // emit getelementptrs that index into vectors. 433 if (Ty->isStructTy() || Ty->isArrayTy()) { 434 CTy = Ty; 435 FieldNo = CE->getOperand(2); 436 return true; 437 } 438 } 439 440 return false; 441} 442 443//===----------------------------------------------------------------------===// 444// SCEV Utilities 445//===----------------------------------------------------------------------===// 446 447namespace { 448 /// SCEVComplexityCompare - Return true if the complexity of the LHS is less 449 /// than the complexity of the RHS. This comparator is used to canonicalize 450 /// expressions. 451 class SCEVComplexityCompare { 452 const LoopInfo *const LI; 453 public: 454 explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {} 455 456 // Return true or false if LHS is less than, or at least RHS, respectively. 457 bool operator()(const SCEV *LHS, const SCEV *RHS) const { 458 return compare(LHS, RHS) < 0; 459 } 460 461 // Return negative, zero, or positive, if LHS is less than, equal to, or 462 // greater than RHS, respectively. A three-way result allows recursive 463 // comparisons to be more efficient. 464 int compare(const SCEV *LHS, const SCEV *RHS) const { 465 // Fast-path: SCEVs are uniqued so we can do a quick equality check. 466 if (LHS == RHS) 467 return 0; 468 469 // Primarily, sort the SCEVs by their getSCEVType(). 470 unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType(); 471 if (LType != RType) 472 return (int)LType - (int)RType; 473 474 // Aside from the getSCEVType() ordering, the particular ordering 475 // isn't very important except that it's beneficial to be consistent, 476 // so that (a + b) and (b + a) don't end up as different expressions. 477 switch (LType) { 478 case scUnknown: { 479 const SCEVUnknown *LU = cast<SCEVUnknown>(LHS); 480 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS); 481 482 // Sort SCEVUnknown values with some loose heuristics. TODO: This is 483 // not as complete as it could be. 484 const Value *LV = LU->getValue(), *RV = RU->getValue(); 485 486 // Order pointer values after integer values. This helps SCEVExpander 487 // form GEPs. 488 bool LIsPointer = LV->getType()->isPointerTy(), 489 RIsPointer = RV->getType()->isPointerTy(); 490 if (LIsPointer != RIsPointer) 491 return (int)LIsPointer - (int)RIsPointer; 492 493 // Compare getValueID values. 494 unsigned LID = LV->getValueID(), 495 RID = RV->getValueID(); 496 if (LID != RID) 497 return (int)LID - (int)RID; 498 499 // Sort arguments by their position. 500 if (const Argument *LA = dyn_cast<Argument>(LV)) { 501 const Argument *RA = cast<Argument>(RV); 502 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo(); 503 return (int)LArgNo - (int)RArgNo; 504 } 505 506 // For instructions, compare their loop depth, and their operand 507 // count. This is pretty loose. 508 if (const Instruction *LInst = dyn_cast<Instruction>(LV)) { 509 const Instruction *RInst = cast<Instruction>(RV); 510 511 // Compare loop depths. 512 const BasicBlock *LParent = LInst->getParent(), 513 *RParent = RInst->getParent(); 514 if (LParent != RParent) { 515 unsigned LDepth = LI->getLoopDepth(LParent), 516 RDepth = LI->getLoopDepth(RParent); 517 if (LDepth != RDepth) 518 return (int)LDepth - (int)RDepth; 519 } 520 521 // Compare the number of operands. 522 unsigned LNumOps = LInst->getNumOperands(), 523 RNumOps = RInst->getNumOperands(); 524 return (int)LNumOps - (int)RNumOps; 525 } 526 527 return 0; 528 } 529 530 case scConstant: { 531 const SCEVConstant *LC = cast<SCEVConstant>(LHS); 532 const SCEVConstant *RC = cast<SCEVConstant>(RHS); 533 534 // Compare constant values. 535 const APInt &LA = LC->getValue()->getValue(); 536 const APInt &RA = RC->getValue()->getValue(); 537 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth(); 538 if (LBitWidth != RBitWidth) 539 return (int)LBitWidth - (int)RBitWidth; 540 return LA.ult(RA) ? -1 : 1; 541 } 542 543 case scAddRecExpr: { 544 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS); 545 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS); 546 547 // Compare addrec loop depths. 548 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop(); 549 if (LLoop != RLoop) { 550 unsigned LDepth = LLoop->getLoopDepth(), 551 RDepth = RLoop->getLoopDepth(); 552 if (LDepth != RDepth) 553 return (int)LDepth - (int)RDepth; 554 } 555 556 // Addrec complexity grows with operand count. 557 unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands(); 558 if (LNumOps != RNumOps) 559 return (int)LNumOps - (int)RNumOps; 560 561 // Lexicographically compare. 562 for (unsigned i = 0; i != LNumOps; ++i) { 563 long X = compare(LA->getOperand(i), RA->getOperand(i)); 564 if (X != 0) 565 return X; 566 } 567 568 return 0; 569 } 570 571 case scAddExpr: 572 case scMulExpr: 573 case scSMaxExpr: 574 case scUMaxExpr: { 575 const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS); 576 const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS); 577 578 // Lexicographically compare n-ary expressions. 579 unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands(); 580 for (unsigned i = 0; i != LNumOps; ++i) { 581 if (i >= RNumOps) 582 return 1; 583 long X = compare(LC->getOperand(i), RC->getOperand(i)); 584 if (X != 0) 585 return X; 586 } 587 return (int)LNumOps - (int)RNumOps; 588 } 589 590 case scUDivExpr: { 591 const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS); 592 const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS); 593 594 // Lexicographically compare udiv expressions. 595 long X = compare(LC->getLHS(), RC->getLHS()); 596 if (X != 0) 597 return X; 598 return compare(LC->getRHS(), RC->getRHS()); 599 } 600 601 case scTruncate: 602 case scZeroExtend: 603 case scSignExtend: { 604 const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS); 605 const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS); 606 607 // Compare cast expressions by operand. 608 return compare(LC->getOperand(), RC->getOperand()); 609 } 610 611 default: 612 llvm_unreachable("Unknown SCEV kind!"); 613 } 614 } 615 }; 616} 617 618/// GroupByComplexity - Given a list of SCEV objects, order them by their 619/// complexity, and group objects of the same complexity together by value. 620/// When this routine is finished, we know that any duplicates in the vector are 621/// consecutive and that complexity is monotonically increasing. 622/// 623/// Note that we go take special precautions to ensure that we get deterministic 624/// results from this routine. In other words, we don't want the results of 625/// this to depend on where the addresses of various SCEV objects happened to 626/// land in memory. 627/// 628static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops, 629 LoopInfo *LI) { 630 if (Ops.size() < 2) return; // Noop 631 if (Ops.size() == 2) { 632 // This is the common case, which also happens to be trivially simple. 633 // Special case it. 634 const SCEV *&LHS = Ops[0], *&RHS = Ops[1]; 635 if (SCEVComplexityCompare(LI)(RHS, LHS)) 636 std::swap(LHS, RHS); 637 return; 638 } 639 640 // Do the rough sort by complexity. 641 std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI)); 642 643 // Now that we are sorted by complexity, group elements of the same 644 // complexity. Note that this is, at worst, N^2, but the vector is likely to 645 // be extremely short in practice. Note that we take this approach because we 646 // do not want to depend on the addresses of the objects we are grouping. 647 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) { 648 const SCEV *S = Ops[i]; 649 unsigned Complexity = S->getSCEVType(); 650 651 // If there are any objects of the same complexity and same value as this 652 // one, group them. 653 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) { 654 if (Ops[j] == S) { // Found a duplicate. 655 // Move it to immediately after i'th element. 656 std::swap(Ops[i+1], Ops[j]); 657 ++i; // no need to rescan it. 658 if (i == e-2) return; // Done! 659 } 660 } 661 } 662} 663 664 665 666//===----------------------------------------------------------------------===// 667// Simple SCEV method implementations 668//===----------------------------------------------------------------------===// 669 670/// BinomialCoefficient - Compute BC(It, K). The result has width W. 671/// Assume, K > 0. 672static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K, 673 ScalarEvolution &SE, 674 Type *ResultTy) { 675 // Handle the simplest case efficiently. 676 if (K == 1) 677 return SE.getTruncateOrZeroExtend(It, ResultTy); 678 679 // We are using the following formula for BC(It, K): 680 // 681 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K! 682 // 683 // Suppose, W is the bitwidth of the return value. We must be prepared for 684 // overflow. Hence, we must assure that the result of our computation is 685 // equal to the accurate one modulo 2^W. Unfortunately, division isn't 686 // safe in modular arithmetic. 687 // 688 // However, this code doesn't use exactly that formula; the formula it uses 689 // is something like the following, where T is the number of factors of 2 in 690 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is 691 // exponentiation: 692 // 693 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T) 694 // 695 // This formula is trivially equivalent to the previous formula. However, 696 // this formula can be implemented much more efficiently. The trick is that 697 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular 698 // arithmetic. To do exact division in modular arithmetic, all we have 699 // to do is multiply by the inverse. Therefore, this step can be done at 700 // width W. 701 // 702 // The next issue is how to safely do the division by 2^T. The way this 703 // is done is by doing the multiplication step at a width of at least W + T 704 // bits. This way, the bottom W+T bits of the product are accurate. Then, 705 // when we perform the division by 2^T (which is equivalent to a right shift 706 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get 707 // truncated out after the division by 2^T. 708 // 709 // In comparison to just directly using the first formula, this technique 710 // is much more efficient; using the first formula requires W * K bits, 711 // but this formula less than W + K bits. Also, the first formula requires 712 // a division step, whereas this formula only requires multiplies and shifts. 713 // 714 // It doesn't matter whether the subtraction step is done in the calculation 715 // width or the input iteration count's width; if the subtraction overflows, 716 // the result must be zero anyway. We prefer here to do it in the width of 717 // the induction variable because it helps a lot for certain cases; CodeGen 718 // isn't smart enough to ignore the overflow, which leads to much less 719 // efficient code if the width of the subtraction is wider than the native 720 // register width. 721 // 722 // (It's possible to not widen at all by pulling out factors of 2 before 723 // the multiplication; for example, K=2 can be calculated as 724 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires 725 // extra arithmetic, so it's not an obvious win, and it gets 726 // much more complicated for K > 3.) 727 728 // Protection from insane SCEVs; this bound is conservative, 729 // but it probably doesn't matter. 730 if (K > 1000) 731 return SE.getCouldNotCompute(); 732 733 unsigned W = SE.getTypeSizeInBits(ResultTy); 734 735 // Calculate K! / 2^T and T; we divide out the factors of two before 736 // multiplying for calculating K! / 2^T to avoid overflow. 737 // Other overflow doesn't matter because we only care about the bottom 738 // W bits of the result. 739 APInt OddFactorial(W, 1); 740 unsigned T = 1; 741 for (unsigned i = 3; i <= K; ++i) { 742 APInt Mult(W, i); 743 unsigned TwoFactors = Mult.countTrailingZeros(); 744 T += TwoFactors; 745 Mult = Mult.lshr(TwoFactors); 746 OddFactorial *= Mult; 747 } 748 749 // We need at least W + T bits for the multiplication step 750 unsigned CalculationBits = W + T; 751 752 // Calculate 2^T, at width T+W. 753 APInt DivFactor = APInt(CalculationBits, 1).shl(T); 754 755 // Calculate the multiplicative inverse of K! / 2^T; 756 // this multiplication factor will perform the exact division by 757 // K! / 2^T. 758 APInt Mod = APInt::getSignedMinValue(W+1); 759 APInt MultiplyFactor = OddFactorial.zext(W+1); 760 MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod); 761 MultiplyFactor = MultiplyFactor.trunc(W); 762 763 // Calculate the product, at width T+W 764 IntegerType *CalculationTy = IntegerType::get(SE.getContext(), 765 CalculationBits); 766 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy); 767 for (unsigned i = 1; i != K; ++i) { 768 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i)); 769 Dividend = SE.getMulExpr(Dividend, 770 SE.getTruncateOrZeroExtend(S, CalculationTy)); 771 } 772 773 // Divide by 2^T 774 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor)); 775 776 // Truncate the result, and divide by K! / 2^T. 777 778 return SE.getMulExpr(SE.getConstant(MultiplyFactor), 779 SE.getTruncateOrZeroExtend(DivResult, ResultTy)); 780} 781 782/// evaluateAtIteration - Return the value of this chain of recurrences at 783/// the specified iteration number. We can evaluate this recurrence by 784/// multiplying each element in the chain by the binomial coefficient 785/// corresponding to it. In other words, we can evaluate {A,+,B,+,C,+,D} as: 786/// 787/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3) 788/// 789/// where BC(It, k) stands for binomial coefficient. 790/// 791const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It, 792 ScalarEvolution &SE) const { 793 const SCEV *Result = getStart(); 794 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) { 795 // The computation is correct in the face of overflow provided that the 796 // multiplication is performed _after_ the evaluation of the binomial 797 // coefficient. 798 const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType()); 799 if (isa<SCEVCouldNotCompute>(Coeff)) 800 return Coeff; 801 802 Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff)); 803 } 804 return Result; 805} 806 807//===----------------------------------------------------------------------===// 808// SCEV Expression folder implementations 809//===----------------------------------------------------------------------===// 810 811const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, 812 Type *Ty) { 813 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && 814 "This is not a truncating conversion!"); 815 assert(isSCEVable(Ty) && 816 "This is not a conversion to a SCEVable type!"); 817 Ty = getEffectiveSCEVType(Ty); 818 819 FoldingSetNodeID ID; 820 ID.AddInteger(scTruncate); 821 ID.AddPointer(Op); 822 ID.AddPointer(Ty); 823 void *IP = 0; 824 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 825 826 // Fold if the operand is constant. 827 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 828 return getConstant( 829 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty))); 830 831 // trunc(trunc(x)) --> trunc(x) 832 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) 833 return getTruncateExpr(ST->getOperand(), Ty); 834 835 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing 836 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 837 return getTruncateOrSignExtend(SS->getOperand(), Ty); 838 839 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing 840 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 841 return getTruncateOrZeroExtend(SZ->getOperand(), Ty); 842 843 // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can 844 // eliminate all the truncates. 845 if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) { 846 SmallVector<const SCEV *, 4> Operands; 847 bool hasTrunc = false; 848 for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) { 849 const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty); 850 hasTrunc = isa<SCEVTruncateExpr>(S); 851 Operands.push_back(S); 852 } 853 if (!hasTrunc) 854 return getAddExpr(Operands); 855 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 856 } 857 858 // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can 859 // eliminate all the truncates. 860 if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) { 861 SmallVector<const SCEV *, 4> Operands; 862 bool hasTrunc = false; 863 for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) { 864 const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty); 865 hasTrunc = isa<SCEVTruncateExpr>(S); 866 Operands.push_back(S); 867 } 868 if (!hasTrunc) 869 return getMulExpr(Operands); 870 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 871 } 872 873 // If the input value is a chrec scev, truncate the chrec's operands. 874 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { 875 SmallVector<const SCEV *, 4> Operands; 876 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 877 Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty)); 878 return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap); 879 } 880 881 // As a special case, fold trunc(undef) to undef. We don't want to 882 // know too much about SCEVUnknowns, but this special case is handy 883 // and harmless. 884 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Op)) 885 if (isa<UndefValue>(U->getValue())) 886 return getSCEV(UndefValue::get(Ty)); 887 888 // The cast wasn't folded; create an explicit cast node. We can reuse 889 // the existing insert position since if we get here, we won't have 890 // made any changes which would invalidate it. 891 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), 892 Op, Ty); 893 UniqueSCEVs.InsertNode(S, IP); 894 return S; 895} 896 897const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, 898 Type *Ty) { 899 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 900 "This is not an extending conversion!"); 901 assert(isSCEVable(Ty) && 902 "This is not a conversion to a SCEVable type!"); 903 Ty = getEffectiveSCEVType(Ty); 904 905 // Fold if the operand is constant. 906 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 907 return getConstant( 908 cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty))); 909 910 // zext(zext(x)) --> zext(x) 911 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 912 return getZeroExtendExpr(SZ->getOperand(), Ty); 913 914 // Before doing any expensive analysis, check to see if we've already 915 // computed a SCEV for this Op and Ty. 916 FoldingSetNodeID ID; 917 ID.AddInteger(scZeroExtend); 918 ID.AddPointer(Op); 919 ID.AddPointer(Ty); 920 void *IP = 0; 921 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 922 923 // zext(trunc(x)) --> zext(x) or x or trunc(x) 924 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 925 // It's possible the bits taken off by the truncate were all zero bits. If 926 // so, we should be able to simplify this further. 927 const SCEV *X = ST->getOperand(); 928 ConstantRange CR = getUnsignedRange(X); 929 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 930 unsigned NewBits = getTypeSizeInBits(Ty); 931 if (CR.truncate(TruncBits).zeroExtend(NewBits).contains( 932 CR.zextOrTrunc(NewBits))) 933 return getTruncateOrZeroExtend(X, Ty); 934 } 935 936 // If the input value is a chrec scev, and we can prove that the value 937 // did not overflow the old, smaller, value, we can zero extend all of the 938 // operands (often constants). This allows analysis of something like 939 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; } 940 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 941 if (AR->isAffine()) { 942 const SCEV *Start = AR->getStart(); 943 const SCEV *Step = AR->getStepRecurrence(*this); 944 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 945 const Loop *L = AR->getLoop(); 946 947 // If we have special knowledge that this addrec won't overflow, 948 // we don't need to do any further analysis. 949 if (AR->getNoWrapFlags(SCEV::FlagNUW)) 950 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 951 getZeroExtendExpr(Step, Ty), 952 L, AR->getNoWrapFlags()); 953 954 // Check whether the backedge-taken count is SCEVCouldNotCompute. 955 // Note that this serves two purposes: It filters out loops that are 956 // simply not analyzable, and it covers the case where this code is 957 // being called from within backedge-taken count analysis, such that 958 // attempting to ask for the backedge-taken count would likely result 959 // in infinite recursion. In the later case, the analysis code will 960 // cope with a conservative value, and it will take care to purge 961 // that value once it has finished. 962 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 963 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 964 // Manually compute the final value for AR, checking for 965 // overflow. 966 967 // Check whether the backedge-taken count can be losslessly casted to 968 // the addrec's type. The count is always unsigned. 969 const SCEV *CastedMaxBECount = 970 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 971 const SCEV *RecastedMaxBECount = 972 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 973 if (MaxBECount == RecastedMaxBECount) { 974 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 975 // Check whether Start+Step*MaxBECount has no unsigned overflow. 976 const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step); 977 const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy); 978 const SCEV *WideStart = getZeroExtendExpr(Start, WideTy); 979 const SCEV *WideMaxBECount = 980 getZeroExtendExpr(CastedMaxBECount, WideTy); 981 const SCEV *OperandExtendedAdd = 982 getAddExpr(WideStart, 983 getMulExpr(WideMaxBECount, 984 getZeroExtendExpr(Step, WideTy))); 985 if (ZAdd == OperandExtendedAdd) { 986 // Cache knowledge of AR NUW, which is propagated to this AddRec. 987 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 988 // Return the expression with the addrec on the outside. 989 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 990 getZeroExtendExpr(Step, Ty), 991 L, AR->getNoWrapFlags()); 992 } 993 // Similar to above, only this time treat the step value as signed. 994 // This covers loops that count down. 995 OperandExtendedAdd = 996 getAddExpr(WideStart, 997 getMulExpr(WideMaxBECount, 998 getSignExtendExpr(Step, WideTy))); 999 if (ZAdd == OperandExtendedAdd) { 1000 // Cache knowledge of AR NW, which is propagated to this AddRec. 1001 // Negative step causes unsigned wrap, but it still can't self-wrap. 1002 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1003 // Return the expression with the addrec on the outside. 1004 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1005 getSignExtendExpr(Step, Ty), 1006 L, AR->getNoWrapFlags()); 1007 } 1008 } 1009 1010 // If the backedge is guarded by a comparison with the pre-inc value 1011 // the addrec is safe. Also, if the entry is guarded by a comparison 1012 // with the start value and the backedge is guarded by a comparison 1013 // with the post-inc value, the addrec is safe. 1014 if (isKnownPositive(Step)) { 1015 const SCEV *N = getConstant(APInt::getMinValue(BitWidth) - 1016 getUnsignedRange(Step).getUnsignedMax()); 1017 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) || 1018 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) && 1019 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, 1020 AR->getPostIncExpr(*this), N))) { 1021 // Cache knowledge of AR NUW, which is propagated to this AddRec. 1022 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1023 // Return the expression with the addrec on the outside. 1024 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1025 getZeroExtendExpr(Step, Ty), 1026 L, AR->getNoWrapFlags()); 1027 } 1028 } else if (isKnownNegative(Step)) { 1029 const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) - 1030 getSignedRange(Step).getSignedMin()); 1031 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) || 1032 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) && 1033 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, 1034 AR->getPostIncExpr(*this), N))) { 1035 // Cache knowledge of AR NW, which is propagated to this AddRec. 1036 // Negative step causes unsigned wrap, but it still can't self-wrap. 1037 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1038 // Return the expression with the addrec on the outside. 1039 return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1040 getSignExtendExpr(Step, Ty), 1041 L, AR->getNoWrapFlags()); 1042 } 1043 } 1044 } 1045 } 1046 1047 // The cast wasn't folded; create an explicit cast node. 1048 // Recompute the insert position, as it may have been invalidated. 1049 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1050 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), 1051 Op, Ty); 1052 UniqueSCEVs.InsertNode(S, IP); 1053 return S; 1054} 1055 1056// Get the limit of a recurrence such that incrementing by Step cannot cause 1057// signed overflow as long as the value of the recurrence within the loop does 1058// not exceed this limit before incrementing. 1059static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1060 ICmpInst::Predicate *Pred, 1061 ScalarEvolution *SE) { 1062 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType()); 1063 if (SE->isKnownPositive(Step)) { 1064 *Pred = ICmpInst::ICMP_SLT; 1065 return SE->getConstant(APInt::getSignedMinValue(BitWidth) - 1066 SE->getSignedRange(Step).getSignedMax()); 1067 } 1068 if (SE->isKnownNegative(Step)) { 1069 *Pred = ICmpInst::ICMP_SGT; 1070 return SE->getConstant(APInt::getSignedMaxValue(BitWidth) - 1071 SE->getSignedRange(Step).getSignedMin()); 1072 } 1073 return 0; 1074} 1075 1076// The recurrence AR has been shown to have no signed wrap. Typically, if we can 1077// prove NSW for AR, then we can just as easily prove NSW for its preincrement 1078// or postincrement sibling. This allows normalizing a sign extended AddRec as 1079// such: {sext(Step + Start),+,Step} => {(Step + sext(Start),+,Step} As a 1080// result, the expression "Step + sext(PreIncAR)" is congruent with 1081// "sext(PostIncAR)" 1082static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR, 1083 Type *Ty, 1084 ScalarEvolution *SE) { 1085 const Loop *L = AR->getLoop(); 1086 const SCEV *Start = AR->getStart(); 1087 const SCEV *Step = AR->getStepRecurrence(*SE); 1088 1089 // Check for a simple looking step prior to loop entry. 1090 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start); 1091 if (!SA) 1092 return 0; 1093 1094 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV 1095 // subtraction is expensive. For this purpose, perform a quick and dirty 1096 // difference, by checking for Step in the operand list. 1097 SmallVector<const SCEV *, 4> DiffOps; 1098 for (SCEVAddExpr::op_iterator I = SA->op_begin(), E = SA->op_end(); 1099 I != E; ++I) { 1100 if (*I != Step) 1101 DiffOps.push_back(*I); 1102 } 1103 if (DiffOps.size() == SA->getNumOperands()) 1104 return 0; 1105 1106 // This is a postinc AR. Check for overflow on the preinc recurrence using the 1107 // same three conditions that getSignExtendedExpr checks. 1108 1109 // 1. NSW flags on the step increment. 1110 const SCEV *PreStart = SE->getAddExpr(DiffOps, SA->getNoWrapFlags()); 1111 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>( 1112 SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap)); 1113 1114 if (PreAR && PreAR->getNoWrapFlags(SCEV::FlagNSW)) 1115 return PreStart; 1116 1117 // 2. Direct overflow check on the step operation's expression. 1118 unsigned BitWidth = SE->getTypeSizeInBits(AR->getType()); 1119 Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2); 1120 const SCEV *OperandExtendedStart = 1121 SE->getAddExpr(SE->getSignExtendExpr(PreStart, WideTy), 1122 SE->getSignExtendExpr(Step, WideTy)); 1123 if (SE->getSignExtendExpr(Start, WideTy) == OperandExtendedStart) { 1124 // Cache knowledge of PreAR NSW. 1125 if (PreAR) 1126 const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(SCEV::FlagNSW); 1127 // FIXME: this optimization needs a unit test 1128 DEBUG(dbgs() << "SCEV: untested prestart overflow check\n"); 1129 return PreStart; 1130 } 1131 1132 // 3. Loop precondition. 1133 ICmpInst::Predicate Pred; 1134 const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, SE); 1135 1136 if (OverflowLimit && 1137 SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) { 1138 return PreStart; 1139 } 1140 return 0; 1141} 1142 1143// Get the normalized sign-extended expression for this AddRec's Start. 1144static const SCEV *getSignExtendAddRecStart(const SCEVAddRecExpr *AR, 1145 Type *Ty, 1146 ScalarEvolution *SE) { 1147 const SCEV *PreStart = getPreStartForSignExtend(AR, Ty, SE); 1148 if (!PreStart) 1149 return SE->getSignExtendExpr(AR->getStart(), Ty); 1150 1151 return SE->getAddExpr(SE->getSignExtendExpr(AR->getStepRecurrence(*SE), Ty), 1152 SE->getSignExtendExpr(PreStart, Ty)); 1153} 1154 1155const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, 1156 Type *Ty) { 1157 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1158 "This is not an extending conversion!"); 1159 assert(isSCEVable(Ty) && 1160 "This is not a conversion to a SCEVable type!"); 1161 Ty = getEffectiveSCEVType(Ty); 1162 1163 // Fold if the operand is constant. 1164 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1165 return getConstant( 1166 cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty))); 1167 1168 // sext(sext(x)) --> sext(x) 1169 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1170 return getSignExtendExpr(SS->getOperand(), Ty); 1171 1172 // sext(zext(x)) --> zext(x) 1173 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1174 return getZeroExtendExpr(SZ->getOperand(), Ty); 1175 1176 // Before doing any expensive analysis, check to see if we've already 1177 // computed a SCEV for this Op and Ty. 1178 FoldingSetNodeID ID; 1179 ID.AddInteger(scSignExtend); 1180 ID.AddPointer(Op); 1181 ID.AddPointer(Ty); 1182 void *IP = 0; 1183 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1184 1185 // If the input value is provably positive, build a zext instead. 1186 if (isKnownNonNegative(Op)) 1187 return getZeroExtendExpr(Op, Ty); 1188 1189 // sext(trunc(x)) --> sext(x) or x or trunc(x) 1190 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1191 // It's possible the bits taken off by the truncate were all sign bits. If 1192 // so, we should be able to simplify this further. 1193 const SCEV *X = ST->getOperand(); 1194 ConstantRange CR = getSignedRange(X); 1195 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1196 unsigned NewBits = getTypeSizeInBits(Ty); 1197 if (CR.truncate(TruncBits).signExtend(NewBits).contains( 1198 CR.sextOrTrunc(NewBits))) 1199 return getTruncateOrSignExtend(X, Ty); 1200 } 1201 1202 // If the input value is a chrec scev, and we can prove that the value 1203 // did not overflow the old, smaller, value, we can sign extend all of the 1204 // operands (often constants). This allows analysis of something like 1205 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; } 1206 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1207 if (AR->isAffine()) { 1208 const SCEV *Start = AR->getStart(); 1209 const SCEV *Step = AR->getStepRecurrence(*this); 1210 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1211 const Loop *L = AR->getLoop(); 1212 1213 // If we have special knowledge that this addrec won't overflow, 1214 // we don't need to do any further analysis. 1215 if (AR->getNoWrapFlags(SCEV::FlagNSW)) 1216 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1217 getSignExtendExpr(Step, Ty), 1218 L, SCEV::FlagNSW); 1219 1220 // Check whether the backedge-taken count is SCEVCouldNotCompute. 1221 // Note that this serves two purposes: It filters out loops that are 1222 // simply not analyzable, and it covers the case where this code is 1223 // being called from within backedge-taken count analysis, such that 1224 // attempting to ask for the backedge-taken count would likely result 1225 // in infinite recursion. In the later case, the analysis code will 1226 // cope with a conservative value, and it will take care to purge 1227 // that value once it has finished. 1228 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1229 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1230 // Manually compute the final value for AR, checking for 1231 // overflow. 1232 1233 // Check whether the backedge-taken count can be losslessly casted to 1234 // the addrec's type. The count is always unsigned. 1235 const SCEV *CastedMaxBECount = 1236 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1237 const SCEV *RecastedMaxBECount = 1238 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1239 if (MaxBECount == RecastedMaxBECount) { 1240 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1241 // Check whether Start+Step*MaxBECount has no signed overflow. 1242 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step); 1243 const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy); 1244 const SCEV *WideStart = getSignExtendExpr(Start, WideTy); 1245 const SCEV *WideMaxBECount = 1246 getZeroExtendExpr(CastedMaxBECount, WideTy); 1247 const SCEV *OperandExtendedAdd = 1248 getAddExpr(WideStart, 1249 getMulExpr(WideMaxBECount, 1250 getSignExtendExpr(Step, WideTy))); 1251 if (SAdd == OperandExtendedAdd) { 1252 // Cache knowledge of AR NSW, which is propagated to this AddRec. 1253 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1254 // Return the expression with the addrec on the outside. 1255 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1256 getSignExtendExpr(Step, Ty), 1257 L, AR->getNoWrapFlags()); 1258 } 1259 // Similar to above, only this time treat the step value as unsigned. 1260 // This covers loops that count up with an unsigned step. 1261 OperandExtendedAdd = 1262 getAddExpr(WideStart, 1263 getMulExpr(WideMaxBECount, 1264 getZeroExtendExpr(Step, WideTy))); 1265 if (SAdd == OperandExtendedAdd) { 1266 // Cache knowledge of AR NSW, which is propagated to this AddRec. 1267 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1268 // Return the expression with the addrec on the outside. 1269 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1270 getZeroExtendExpr(Step, Ty), 1271 L, AR->getNoWrapFlags()); 1272 } 1273 } 1274 1275 // If the backedge is guarded by a comparison with the pre-inc value 1276 // the addrec is safe. Also, if the entry is guarded by a comparison 1277 // with the start value and the backedge is guarded by a comparison 1278 // with the post-inc value, the addrec is safe. 1279 ICmpInst::Predicate Pred; 1280 const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, this); 1281 if (OverflowLimit && 1282 (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) || 1283 (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) && 1284 isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this), 1285 OverflowLimit)))) { 1286 // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec. 1287 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1288 return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1289 getSignExtendExpr(Step, Ty), 1290 L, AR->getNoWrapFlags()); 1291 } 1292 } 1293 } 1294 1295 // The cast wasn't folded; create an explicit cast node. 1296 // Recompute the insert position, as it may have been invalidated. 1297 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1298 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), 1299 Op, Ty); 1300 UniqueSCEVs.InsertNode(S, IP); 1301 return S; 1302} 1303 1304/// getAnyExtendExpr - Return a SCEV for the given operand extended with 1305/// unspecified bits out to the given type. 1306/// 1307const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op, 1308 Type *Ty) { 1309 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1310 "This is not an extending conversion!"); 1311 assert(isSCEVable(Ty) && 1312 "This is not a conversion to a SCEVable type!"); 1313 Ty = getEffectiveSCEVType(Ty); 1314 1315 // Sign-extend negative constants. 1316 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1317 if (SC->getValue()->getValue().isNegative()) 1318 return getSignExtendExpr(Op, Ty); 1319 1320 // Peel off a truncate cast. 1321 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) { 1322 const SCEV *NewOp = T->getOperand(); 1323 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty)) 1324 return getAnyExtendExpr(NewOp, Ty); 1325 return getTruncateOrNoop(NewOp, Ty); 1326 } 1327 1328 // Next try a zext cast. If the cast is folded, use it. 1329 const SCEV *ZExt = getZeroExtendExpr(Op, Ty); 1330 if (!isa<SCEVZeroExtendExpr>(ZExt)) 1331 return ZExt; 1332 1333 // Next try a sext cast. If the cast is folded, use it. 1334 const SCEV *SExt = getSignExtendExpr(Op, Ty); 1335 if (!isa<SCEVSignExtendExpr>(SExt)) 1336 return SExt; 1337 1338 // Force the cast to be folded into the operands of an addrec. 1339 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) { 1340 SmallVector<const SCEV *, 4> Ops; 1341 for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end(); 1342 I != E; ++I) 1343 Ops.push_back(getAnyExtendExpr(*I, Ty)); 1344 return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW); 1345 } 1346 1347 // As a special case, fold anyext(undef) to undef. We don't want to 1348 // know too much about SCEVUnknowns, but this special case is handy 1349 // and harmless. 1350 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Op)) 1351 if (isa<UndefValue>(U->getValue())) 1352 return getSCEV(UndefValue::get(Ty)); 1353 1354 // If the expression is obviously signed, use the sext cast value. 1355 if (isa<SCEVSMaxExpr>(Op)) 1356 return SExt; 1357 1358 // Absent any other information, use the zext cast value. 1359 return ZExt; 1360} 1361 1362/// CollectAddOperandsWithScales - Process the given Ops list, which is 1363/// a list of operands to be added under the given scale, update the given 1364/// map. This is a helper function for getAddRecExpr. As an example of 1365/// what it does, given a sequence of operands that would form an add 1366/// expression like this: 1367/// 1368/// m + n + 13 + (A * (o + p + (B * q + m + 29))) + r + (-1 * r) 1369/// 1370/// where A and B are constants, update the map with these values: 1371/// 1372/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0) 1373/// 1374/// and add 13 + A*B*29 to AccumulatedConstant. 1375/// This will allow getAddRecExpr to produce this: 1376/// 1377/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B) 1378/// 1379/// This form often exposes folding opportunities that are hidden in 1380/// the original operand list. 1381/// 1382/// Return true iff it appears that any interesting folding opportunities 1383/// may be exposed. This helps getAddRecExpr short-circuit extra work in 1384/// the common case where no interesting opportunities are present, and 1385/// is also used as a check to avoid infinite recursion. 1386/// 1387static bool 1388CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M, 1389 SmallVector<const SCEV *, 8> &NewOps, 1390 APInt &AccumulatedConstant, 1391 const SCEV *const *Ops, size_t NumOperands, 1392 const APInt &Scale, 1393 ScalarEvolution &SE) { 1394 bool Interesting = false; 1395 1396 // Iterate over the add operands. They are sorted, with constants first. 1397 unsigned i = 0; 1398 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1399 ++i; 1400 // Pull a buried constant out to the outside. 1401 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero()) 1402 Interesting = true; 1403 AccumulatedConstant += Scale * C->getValue()->getValue(); 1404 } 1405 1406 // Next comes everything else. We're especially interested in multiplies 1407 // here, but they're in the middle, so just visit the rest with one loop. 1408 for (; i != NumOperands; ++i) { 1409 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]); 1410 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) { 1411 APInt NewScale = 1412 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getValue()->getValue(); 1413 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) { 1414 // A multiplication of a constant with another add; recurse. 1415 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1)); 1416 Interesting |= 1417 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1418 Add->op_begin(), Add->getNumOperands(), 1419 NewScale, SE); 1420 } else { 1421 // A multiplication of a constant with some other value. Update 1422 // the map. 1423 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end()); 1424 const SCEV *Key = SE.getMulExpr(MulOps); 1425 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1426 M.insert(std::make_pair(Key, NewScale)); 1427 if (Pair.second) { 1428 NewOps.push_back(Pair.first->first); 1429 } else { 1430 Pair.first->second += NewScale; 1431 // The map already had an entry for this value, which may indicate 1432 // a folding opportunity. 1433 Interesting = true; 1434 } 1435 } 1436 } else { 1437 // An ordinary operand. Update the map. 1438 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1439 M.insert(std::make_pair(Ops[i], Scale)); 1440 if (Pair.second) { 1441 NewOps.push_back(Pair.first->first); 1442 } else { 1443 Pair.first->second += Scale; 1444 // The map already had an entry for this value, which may indicate 1445 // a folding opportunity. 1446 Interesting = true; 1447 } 1448 } 1449 } 1450 1451 return Interesting; 1452} 1453 1454namespace { 1455 struct APIntCompare { 1456 bool operator()(const APInt &LHS, const APInt &RHS) const { 1457 return LHS.ult(RHS); 1458 } 1459 }; 1460} 1461 1462/// getAddExpr - Get a canonical add expression, or something simpler if 1463/// possible. 1464const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, 1465 SCEV::NoWrapFlags Flags) { 1466 assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && 1467 "only nuw or nsw allowed"); 1468 assert(!Ops.empty() && "Cannot get empty add!"); 1469 if (Ops.size() == 1) return Ops[0]; 1470#ifndef NDEBUG 1471 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 1472 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 1473 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 1474 "SCEVAddExpr operand types don't match!"); 1475#endif 1476 1477 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 1478 // And vice-versa. 1479 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 1480 SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 1481 if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 1482 bool All = true; 1483 for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(), 1484 E = Ops.end(); I != E; ++I) 1485 if (!isKnownNonNegative(*I)) { 1486 All = false; 1487 break; 1488 } 1489 if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 1490 } 1491 1492 // Sort by complexity, this groups all similar expression types together. 1493 GroupByComplexity(Ops, LI); 1494 1495 // If there are any constants, fold them together. 1496 unsigned Idx = 0; 1497 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 1498 ++Idx; 1499 assert(Idx < Ops.size()); 1500 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 1501 // We found two constants, fold them together! 1502 Ops[0] = getConstant(LHSC->getValue()->getValue() + 1503 RHSC->getValue()->getValue()); 1504 if (Ops.size() == 2) return Ops[0]; 1505 Ops.erase(Ops.begin()+1); // Erase the folded element 1506 LHSC = cast<SCEVConstant>(Ops[0]); 1507 } 1508 1509 // If we are left with a constant zero being added, strip it off. 1510 if (LHSC->getValue()->isZero()) { 1511 Ops.erase(Ops.begin()); 1512 --Idx; 1513 } 1514 1515 if (Ops.size() == 1) return Ops[0]; 1516 } 1517 1518 // Okay, check to see if the same value occurs in the operand list more than 1519 // once. If so, merge them together into an multiply expression. Since we 1520 // sorted the list, these values are required to be adjacent. 1521 Type *Ty = Ops[0]->getType(); 1522 bool FoundMatch = false; 1523 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i) 1524 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2 1525 // Scan ahead to count how many equal operands there are. 1526 unsigned Count = 2; 1527 while (i+Count != e && Ops[i+Count] == Ops[i]) 1528 ++Count; 1529 // Merge the values into a multiply. 1530 const SCEV *Scale = getConstant(Ty, Count); 1531 const SCEV *Mul = getMulExpr(Scale, Ops[i]); 1532 if (Ops.size() == Count) 1533 return Mul; 1534 Ops[i] = Mul; 1535 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count); 1536 --i; e -= Count - 1; 1537 FoundMatch = true; 1538 } 1539 if (FoundMatch) 1540 return getAddExpr(Ops, Flags); 1541 1542 // Check for truncates. If all the operands are truncated from the same 1543 // type, see if factoring out the truncate would permit the result to be 1544 // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n) 1545 // if the contents of the resulting outer trunc fold to something simple. 1546 for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) { 1547 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]); 1548 Type *DstType = Trunc->getType(); 1549 Type *SrcType = Trunc->getOperand()->getType(); 1550 SmallVector<const SCEV *, 8> LargeOps; 1551 bool Ok = true; 1552 // Check all the operands to see if they can be represented in the 1553 // source type of the truncate. 1554 for (unsigned i = 0, e = Ops.size(); i != e; ++i) { 1555 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) { 1556 if (T->getOperand()->getType() != SrcType) { 1557 Ok = false; 1558 break; 1559 } 1560 LargeOps.push_back(T->getOperand()); 1561 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1562 LargeOps.push_back(getAnyExtendExpr(C, SrcType)); 1563 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) { 1564 SmallVector<const SCEV *, 8> LargeMulOps; 1565 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) { 1566 if (const SCEVTruncateExpr *T = 1567 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) { 1568 if (T->getOperand()->getType() != SrcType) { 1569 Ok = false; 1570 break; 1571 } 1572 LargeMulOps.push_back(T->getOperand()); 1573 } else if (const SCEVConstant *C = 1574 dyn_cast<SCEVConstant>(M->getOperand(j))) { 1575 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType)); 1576 } else { 1577 Ok = false; 1578 break; 1579 } 1580 } 1581 if (Ok) 1582 LargeOps.push_back(getMulExpr(LargeMulOps)); 1583 } else { 1584 Ok = false; 1585 break; 1586 } 1587 } 1588 if (Ok) { 1589 // Evaluate the expression in the larger type. 1590 const SCEV *Fold = getAddExpr(LargeOps, Flags); 1591 // If it folds to something simple, use it. Otherwise, don't. 1592 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold)) 1593 return getTruncateExpr(Fold, DstType); 1594 } 1595 } 1596 1597 // Skip past any other cast SCEVs. 1598 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr) 1599 ++Idx; 1600 1601 // If there are add operands they would be next. 1602 if (Idx < Ops.size()) { 1603 bool DeletedAdd = false; 1604 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) { 1605 // If we have an add, expand the add operands onto the end of the operands 1606 // list. 1607 Ops.erase(Ops.begin()+Idx); 1608 Ops.append(Add->op_begin(), Add->op_end()); 1609 DeletedAdd = true; 1610 } 1611 1612 // If we deleted at least one add, we added operands to the end of the list, 1613 // and they are not necessarily sorted. Recurse to resort and resimplify 1614 // any operands we just acquired. 1615 if (DeletedAdd) 1616 return getAddExpr(Ops); 1617 } 1618 1619 // Skip over the add expression until we get to a multiply. 1620 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 1621 ++Idx; 1622 1623 // Check to see if there are any folding opportunities present with 1624 // operands multiplied by constant values. 1625 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) { 1626 uint64_t BitWidth = getTypeSizeInBits(Ty); 1627 DenseMap<const SCEV *, APInt> M; 1628 SmallVector<const SCEV *, 8> NewOps; 1629 APInt AccumulatedConstant(BitWidth, 0); 1630 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1631 Ops.data(), Ops.size(), 1632 APInt(BitWidth, 1), *this)) { 1633 // Some interesting folding opportunity is present, so its worthwhile to 1634 // re-generate the operands list. Group the operands by constant scale, 1635 // to avoid multiplying by the same constant scale multiple times. 1636 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists; 1637 for (SmallVector<const SCEV *, 8>::const_iterator I = NewOps.begin(), 1638 E = NewOps.end(); I != E; ++I) 1639 MulOpLists[M.find(*I)->second].push_back(*I); 1640 // Re-generate the operands list. 1641 Ops.clear(); 1642 if (AccumulatedConstant != 0) 1643 Ops.push_back(getConstant(AccumulatedConstant)); 1644 for (std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare>::iterator 1645 I = MulOpLists.begin(), E = MulOpLists.end(); I != E; ++I) 1646 if (I->first != 0) 1647 Ops.push_back(getMulExpr(getConstant(I->first), 1648 getAddExpr(I->second))); 1649 if (Ops.empty()) 1650 return getConstant(Ty, 0); 1651 if (Ops.size() == 1) 1652 return Ops[0]; 1653 return getAddExpr(Ops); 1654 } 1655 } 1656 1657 // If we are adding something to a multiply expression, make sure the 1658 // something is not already an operand of the multiply. If so, merge it into 1659 // the multiply. 1660 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) { 1661 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]); 1662 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) { 1663 const SCEV *MulOpSCEV = Mul->getOperand(MulOp); 1664 if (isa<SCEVConstant>(MulOpSCEV)) 1665 continue; 1666 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) 1667 if (MulOpSCEV == Ops[AddOp]) { 1668 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1)) 1669 const SCEV *InnerMul = Mul->getOperand(MulOp == 0); 1670 if (Mul->getNumOperands() != 2) { 1671 // If the multiply has more than two operands, we must get the 1672 // Y*Z term. 1673 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 1674 Mul->op_begin()+MulOp); 1675 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 1676 InnerMul = getMulExpr(MulOps); 1677 } 1678 const SCEV *One = getConstant(Ty, 1); 1679 const SCEV *AddOne = getAddExpr(One, InnerMul); 1680 const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV); 1681 if (Ops.size() == 2) return OuterMul; 1682 if (AddOp < Idx) { 1683 Ops.erase(Ops.begin()+AddOp); 1684 Ops.erase(Ops.begin()+Idx-1); 1685 } else { 1686 Ops.erase(Ops.begin()+Idx); 1687 Ops.erase(Ops.begin()+AddOp-1); 1688 } 1689 Ops.push_back(OuterMul); 1690 return getAddExpr(Ops); 1691 } 1692 1693 // Check this multiply against other multiplies being added together. 1694 for (unsigned OtherMulIdx = Idx+1; 1695 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]); 1696 ++OtherMulIdx) { 1697 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]); 1698 // If MulOp occurs in OtherMul, we can fold the two multiplies 1699 // together. 1700 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands(); 1701 OMulOp != e; ++OMulOp) 1702 if (OtherMul->getOperand(OMulOp) == MulOpSCEV) { 1703 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E)) 1704 const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0); 1705 if (Mul->getNumOperands() != 2) { 1706 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 1707 Mul->op_begin()+MulOp); 1708 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 1709 InnerMul1 = getMulExpr(MulOps); 1710 } 1711 const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0); 1712 if (OtherMul->getNumOperands() != 2) { 1713 SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(), 1714 OtherMul->op_begin()+OMulOp); 1715 MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end()); 1716 InnerMul2 = getMulExpr(MulOps); 1717 } 1718 const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2); 1719 const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum); 1720 if (Ops.size() == 2) return OuterMul; 1721 Ops.erase(Ops.begin()+Idx); 1722 Ops.erase(Ops.begin()+OtherMulIdx-1); 1723 Ops.push_back(OuterMul); 1724 return getAddExpr(Ops); 1725 } 1726 } 1727 } 1728 } 1729 1730 // If there are any add recurrences in the operands list, see if any other 1731 // added values are loop invariant. If so, we can fold them into the 1732 // recurrence. 1733 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 1734 ++Idx; 1735 1736 // Scan over all recurrences, trying to fold loop invariants into them. 1737 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 1738 // Scan all of the other operands to this add and add them to the vector if 1739 // they are loop invariant w.r.t. the recurrence. 1740 SmallVector<const SCEV *, 8> LIOps; 1741 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 1742 const Loop *AddRecLoop = AddRec->getLoop(); 1743 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 1744 if (isLoopInvariant(Ops[i], AddRecLoop)) { 1745 LIOps.push_back(Ops[i]); 1746 Ops.erase(Ops.begin()+i); 1747 --i; --e; 1748 } 1749 1750 // If we found some loop invariants, fold them into the recurrence. 1751 if (!LIOps.empty()) { 1752 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step} 1753 LIOps.push_back(AddRec->getStart()); 1754 1755 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 1756 AddRec->op_end()); 1757 AddRecOps[0] = getAddExpr(LIOps); 1758 1759 // Build the new addrec. Propagate the NUW and NSW flags if both the 1760 // outer add and the inner addrec are guaranteed to have no overflow. 1761 // Always propagate NW. 1762 Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW)); 1763 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags); 1764 1765 // If all of the other operands were loop invariant, we are done. 1766 if (Ops.size() == 1) return NewRec; 1767 1768 // Otherwise, add the folded AddRec by the non-invariant parts. 1769 for (unsigned i = 0;; ++i) 1770 if (Ops[i] == AddRec) { 1771 Ops[i] = NewRec; 1772 break; 1773 } 1774 return getAddExpr(Ops); 1775 } 1776 1777 // Okay, if there weren't any loop invariants to be folded, check to see if 1778 // there are multiple AddRec's with the same loop induction variable being 1779 // added together. If so, we can fold them. 1780 for (unsigned OtherIdx = Idx+1; 1781 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 1782 ++OtherIdx) 1783 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) { 1784 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L> 1785 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 1786 AddRec->op_end()); 1787 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 1788 ++OtherIdx) 1789 if (const SCEVAddRecExpr *OtherAddRec = 1790 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx])) 1791 if (OtherAddRec->getLoop() == AddRecLoop) { 1792 for (unsigned i = 0, e = OtherAddRec->getNumOperands(); 1793 i != e; ++i) { 1794 if (i >= AddRecOps.size()) { 1795 AddRecOps.append(OtherAddRec->op_begin()+i, 1796 OtherAddRec->op_end()); 1797 break; 1798 } 1799 AddRecOps[i] = getAddExpr(AddRecOps[i], 1800 OtherAddRec->getOperand(i)); 1801 } 1802 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 1803 } 1804 // Step size has changed, so we cannot guarantee no self-wraparound. 1805 Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap); 1806 return getAddExpr(Ops); 1807 } 1808 1809 // Otherwise couldn't fold anything into this recurrence. Move onto the 1810 // next one. 1811 } 1812 1813 // Okay, it looks like we really DO need an add expr. Check to see if we 1814 // already have one, otherwise create a new one. 1815 FoldingSetNodeID ID; 1816 ID.AddInteger(scAddExpr); 1817 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 1818 ID.AddPointer(Ops[i]); 1819 void *IP = 0; 1820 SCEVAddExpr *S = 1821 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 1822 if (!S) { 1823 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 1824 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 1825 S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator), 1826 O, Ops.size()); 1827 UniqueSCEVs.InsertNode(S, IP); 1828 } 1829 S->setNoWrapFlags(Flags); 1830 return S; 1831} 1832 1833static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) { 1834 uint64_t k = i*j; 1835 if (j > 1 && k / j != i) Overflow = true; 1836 return k; 1837} 1838 1839/// Compute the result of "n choose k", the binomial coefficient. If an 1840/// intermediate computation overflows, Overflow will be set and the return will 1841/// be garbage. Overflow is not cleared on absence of overflow. 1842static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) { 1843 // We use the multiplicative formula: 1844 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 . 1845 // At each iteration, we take the n-th term of the numeral and divide by the 1846 // (k-n)th term of the denominator. This division will always produce an 1847 // integral result, and helps reduce the chance of overflow in the 1848 // intermediate computations. However, we can still overflow even when the 1849 // final result would fit. 1850 1851 if (n == 0 || n == k) return 1; 1852 if (k > n) return 0; 1853 1854 if (k > n/2) 1855 k = n-k; 1856 1857 uint64_t r = 1; 1858 for (uint64_t i = 1; i <= k; ++i) { 1859 r = umul_ov(r, n-(i-1), Overflow); 1860 r /= i; 1861 } 1862 return r; 1863} 1864 1865/// getMulExpr - Get a canonical multiply expression, or something simpler if 1866/// possible. 1867const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops, 1868 SCEV::NoWrapFlags Flags) { 1869 assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) && 1870 "only nuw or nsw allowed"); 1871 assert(!Ops.empty() && "Cannot get empty mul!"); 1872 if (Ops.size() == 1) return Ops[0]; 1873#ifndef NDEBUG 1874 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 1875 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 1876 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 1877 "SCEVMulExpr operand types don't match!"); 1878#endif 1879 1880 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 1881 // And vice-versa. 1882 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 1883 SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 1884 if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 1885 bool All = true; 1886 for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(), 1887 E = Ops.end(); I != E; ++I) 1888 if (!isKnownNonNegative(*I)) { 1889 All = false; 1890 break; 1891 } 1892 if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 1893 } 1894 1895 // Sort by complexity, this groups all similar expression types together. 1896 GroupByComplexity(Ops, LI); 1897 1898 // If there are any constants, fold them together. 1899 unsigned Idx = 0; 1900 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 1901 1902 // C1*(C2+V) -> C1*C2 + C1*V 1903 if (Ops.size() == 2) 1904 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) 1905 if (Add->getNumOperands() == 2 && 1906 isa<SCEVConstant>(Add->getOperand(0))) 1907 return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)), 1908 getMulExpr(LHSC, Add->getOperand(1))); 1909 1910 ++Idx; 1911 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 1912 // We found two constants, fold them together! 1913 ConstantInt *Fold = ConstantInt::get(getContext(), 1914 LHSC->getValue()->getValue() * 1915 RHSC->getValue()->getValue()); 1916 Ops[0] = getConstant(Fold); 1917 Ops.erase(Ops.begin()+1); // Erase the folded element 1918 if (Ops.size() == 1) return Ops[0]; 1919 LHSC = cast<SCEVConstant>(Ops[0]); 1920 } 1921 1922 // If we are left with a constant one being multiplied, strip it off. 1923 if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) { 1924 Ops.erase(Ops.begin()); 1925 --Idx; 1926 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) { 1927 // If we have a multiply of zero, it will always be zero. 1928 return Ops[0]; 1929 } else if (Ops[0]->isAllOnesValue()) { 1930 // If we have a mul by -1 of an add, try distributing the -1 among the 1931 // add operands. 1932 if (Ops.size() == 2) { 1933 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) { 1934 SmallVector<const SCEV *, 4> NewOps; 1935 bool AnyFolded = false; 1936 for (SCEVAddRecExpr::op_iterator I = Add->op_begin(), 1937 E = Add->op_end(); I != E; ++I) { 1938 const SCEV *Mul = getMulExpr(Ops[0], *I); 1939 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true; 1940 NewOps.push_back(Mul); 1941 } 1942 if (AnyFolded) 1943 return getAddExpr(NewOps); 1944 } 1945 else if (const SCEVAddRecExpr * 1946 AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) { 1947 // Negation preserves a recurrence's no self-wrap property. 1948 SmallVector<const SCEV *, 4> Operands; 1949 for (SCEVAddRecExpr::op_iterator I = AddRec->op_begin(), 1950 E = AddRec->op_end(); I != E; ++I) { 1951 Operands.push_back(getMulExpr(Ops[0], *I)); 1952 } 1953 return getAddRecExpr(Operands, AddRec->getLoop(), 1954 AddRec->getNoWrapFlags(SCEV::FlagNW)); 1955 } 1956 } 1957 } 1958 1959 if (Ops.size() == 1) 1960 return Ops[0]; 1961 } 1962 1963 // Skip over the add expression until we get to a multiply. 1964 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 1965 ++Idx; 1966 1967 // If there are mul operands inline them all into this expression. 1968 if (Idx < Ops.size()) { 1969 bool DeletedMul = false; 1970 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) { 1971 // If we have an mul, expand the mul operands onto the end of the operands 1972 // list. 1973 Ops.erase(Ops.begin()+Idx); 1974 Ops.append(Mul->op_begin(), Mul->op_end()); 1975 DeletedMul = true; 1976 } 1977 1978 // If we deleted at least one mul, we added operands to the end of the list, 1979 // and they are not necessarily sorted. Recurse to resort and resimplify 1980 // any operands we just acquired. 1981 if (DeletedMul) 1982 return getMulExpr(Ops); 1983 } 1984 1985 // If there are any add recurrences in the operands list, see if any other 1986 // added values are loop invariant. If so, we can fold them into the 1987 // recurrence. 1988 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 1989 ++Idx; 1990 1991 // Scan over all recurrences, trying to fold loop invariants into them. 1992 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 1993 // Scan all of the other operands to this mul and add them to the vector if 1994 // they are loop invariant w.r.t. the recurrence. 1995 SmallVector<const SCEV *, 8> LIOps; 1996 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 1997 const Loop *AddRecLoop = AddRec->getLoop(); 1998 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 1999 if (isLoopInvariant(Ops[i], AddRecLoop)) { 2000 LIOps.push_back(Ops[i]); 2001 Ops.erase(Ops.begin()+i); 2002 --i; --e; 2003 } 2004 2005 // If we found some loop invariants, fold them into the recurrence. 2006 if (!LIOps.empty()) { 2007 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step} 2008 SmallVector<const SCEV *, 4> NewOps; 2009 NewOps.reserve(AddRec->getNumOperands()); 2010 const SCEV *Scale = getMulExpr(LIOps); 2011 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 2012 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i))); 2013 2014 // Build the new addrec. Propagate the NUW and NSW flags if both the 2015 // outer mul and the inner addrec are guaranteed to have no overflow. 2016 // 2017 // No self-wrap cannot be guaranteed after changing the step size, but 2018 // will be inferred if either NUW or NSW is true. 2019 Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW)); 2020 const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags); 2021 2022 // If all of the other operands were loop invariant, we are done. 2023 if (Ops.size() == 1) return NewRec; 2024 2025 // Otherwise, multiply the folded AddRec by the non-invariant parts. 2026 for (unsigned i = 0;; ++i) 2027 if (Ops[i] == AddRec) { 2028 Ops[i] = NewRec; 2029 break; 2030 } 2031 return getMulExpr(Ops); 2032 } 2033 2034 // Okay, if there weren't any loop invariants to be folded, check to see if 2035 // there are multiple AddRec's with the same loop induction variable being 2036 // multiplied together. If so, we can fold them. 2037 for (unsigned OtherIdx = Idx+1; 2038 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2039 ++OtherIdx) { 2040 if (AddRecLoop != cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) 2041 continue; 2042 2043 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L> 2044 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [ 2045 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z 2046 // ]]],+,...up to x=2n}. 2047 // Note that the arguments to choose() are always integers with values 2048 // known at compile time, never SCEV objects. 2049 // 2050 // The implementation avoids pointless extra computations when the two 2051 // addrec's are of different length (mathematically, it's equivalent to 2052 // an infinite stream of zeros on the right). 2053 bool OpsModified = false; 2054 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2055 ++OtherIdx) { 2056 const SCEVAddRecExpr *OtherAddRec = 2057 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]); 2058 if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop) 2059 continue; 2060 2061 bool Overflow = false; 2062 Type *Ty = AddRec->getType(); 2063 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64; 2064 SmallVector<const SCEV*, 7> AddRecOps; 2065 for (int x = 0, xe = AddRec->getNumOperands() + 2066 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) { 2067 const SCEV *Term = getConstant(Ty, 0); 2068 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) { 2069 uint64_t Coeff1 = Choose(x, 2*x - y, Overflow); 2070 for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1), 2071 ze = std::min(x+1, (int)OtherAddRec->getNumOperands()); 2072 z < ze && !Overflow; ++z) { 2073 uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow); 2074 uint64_t Coeff; 2075 if (LargerThan64Bits) 2076 Coeff = umul_ov(Coeff1, Coeff2, Overflow); 2077 else 2078 Coeff = Coeff1*Coeff2; 2079 const SCEV *CoeffTerm = getConstant(Ty, Coeff); 2080 const SCEV *Term1 = AddRec->getOperand(y-z); 2081 const SCEV *Term2 = OtherAddRec->getOperand(z); 2082 Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2)); 2083 } 2084 } 2085 AddRecOps.push_back(Term); 2086 } 2087 if (!Overflow) { 2088 const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(), 2089 SCEV::FlagAnyWrap); 2090 if (Ops.size() == 2) return NewAddRec; 2091 Ops[Idx] = NewAddRec; 2092 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2093 OpsModified = true; 2094 AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec); 2095 if (!AddRec) 2096 break; 2097 } 2098 } 2099 if (OpsModified) 2100 return getMulExpr(Ops); 2101 } 2102 2103 // Otherwise couldn't fold anything into this recurrence. Move onto the 2104 // next one. 2105 } 2106 2107 // Okay, it looks like we really DO need an mul expr. Check to see if we 2108 // already have one, otherwise create a new one. 2109 FoldingSetNodeID ID; 2110 ID.AddInteger(scMulExpr); 2111 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2112 ID.AddPointer(Ops[i]); 2113 void *IP = 0; 2114 SCEVMulExpr *S = 2115 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2116 if (!S) { 2117 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2118 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2119 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator), 2120 O, Ops.size()); 2121 UniqueSCEVs.InsertNode(S, IP); 2122 } 2123 S->setNoWrapFlags(Flags); 2124 return S; 2125} 2126 2127/// getUDivExpr - Get a canonical unsigned division expression, or something 2128/// simpler if possible. 2129const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, 2130 const SCEV *RHS) { 2131 assert(getEffectiveSCEVType(LHS->getType()) == 2132 getEffectiveSCEVType(RHS->getType()) && 2133 "SCEVUDivExpr operand types don't match!"); 2134 2135 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 2136 if (RHSC->getValue()->equalsInt(1)) 2137 return LHS; // X udiv 1 --> x 2138 // If the denominator is zero, the result of the udiv is undefined. Don't 2139 // try to analyze it, because the resolution chosen here may differ from 2140 // the resolution chosen in other parts of the compiler. 2141 if (!RHSC->getValue()->isZero()) { 2142 // Determine if the division can be folded into the operands of 2143 // its operands. 2144 // TODO: Generalize this to non-constants by using known-bits information. 2145 Type *Ty = LHS->getType(); 2146 unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros(); 2147 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1; 2148 // For non-power-of-two values, effectively round the value up to the 2149 // nearest power of two. 2150 if (!RHSC->getValue()->getValue().isPowerOf2()) 2151 ++MaxShiftAmt; 2152 IntegerType *ExtTy = 2153 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt); 2154 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) 2155 if (const SCEVConstant *Step = 2156 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) { 2157 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded. 2158 const APInt &StepInt = Step->getValue()->getValue(); 2159 const APInt &DivInt = RHSC->getValue()->getValue(); 2160 if (!StepInt.urem(DivInt) && 2161 getZeroExtendExpr(AR, ExtTy) == 2162 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2163 getZeroExtendExpr(Step, ExtTy), 2164 AR->getLoop(), SCEV::FlagAnyWrap)) { 2165 SmallVector<const SCEV *, 4> Operands; 2166 for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i) 2167 Operands.push_back(getUDivExpr(AR->getOperand(i), RHS)); 2168 return getAddRecExpr(Operands, AR->getLoop(), 2169 SCEV::FlagNW); 2170 } 2171 /// Get a canonical UDivExpr for a recurrence. 2172 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0. 2173 // We can currently only fold X%N if X is constant. 2174 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart()); 2175 if (StartC && !DivInt.urem(StepInt) && 2176 getZeroExtendExpr(AR, ExtTy) == 2177 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2178 getZeroExtendExpr(Step, ExtTy), 2179 AR->getLoop(), SCEV::FlagAnyWrap)) { 2180 const APInt &StartInt = StartC->getValue()->getValue(); 2181 const APInt &StartRem = StartInt.urem(StepInt); 2182 if (StartRem != 0) 2183 LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step, 2184 AR->getLoop(), SCEV::FlagNW); 2185 } 2186 } 2187 // (A*B)/C --> A*(B/C) if safe and B/C can be folded. 2188 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) { 2189 SmallVector<const SCEV *, 4> Operands; 2190 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) 2191 Operands.push_back(getZeroExtendExpr(M->getOperand(i), ExtTy)); 2192 if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands)) 2193 // Find an operand that's safely divisible. 2194 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 2195 const SCEV *Op = M->getOperand(i); 2196 const SCEV *Div = getUDivExpr(Op, RHSC); 2197 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) { 2198 Operands = SmallVector<const SCEV *, 4>(M->op_begin(), 2199 M->op_end()); 2200 Operands[i] = Div; 2201 return getMulExpr(Operands); 2202 } 2203 } 2204 } 2205 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded. 2206 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) { 2207 SmallVector<const SCEV *, 4> Operands; 2208 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) 2209 Operands.push_back(getZeroExtendExpr(A->getOperand(i), ExtTy)); 2210 if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) { 2211 Operands.clear(); 2212 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) { 2213 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS); 2214 if (isa<SCEVUDivExpr>(Op) || 2215 getMulExpr(Op, RHS) != A->getOperand(i)) 2216 break; 2217 Operands.push_back(Op); 2218 } 2219 if (Operands.size() == A->getNumOperands()) 2220 return getAddExpr(Operands); 2221 } 2222 } 2223 2224 // Fold if both operands are constant. 2225 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 2226 Constant *LHSCV = LHSC->getValue(); 2227 Constant *RHSCV = RHSC->getValue(); 2228 return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV, 2229 RHSCV))); 2230 } 2231 } 2232 } 2233 2234 FoldingSetNodeID ID; 2235 ID.AddInteger(scUDivExpr); 2236 ID.AddPointer(LHS); 2237 ID.AddPointer(RHS); 2238 void *IP = 0; 2239 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2240 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), 2241 LHS, RHS); 2242 UniqueSCEVs.InsertNode(S, IP); 2243 return S; 2244} 2245 2246 2247/// getAddRecExpr - Get an add recurrence expression for the specified loop. 2248/// Simplify the expression as much as possible. 2249const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step, 2250 const Loop *L, 2251 SCEV::NoWrapFlags Flags) { 2252 SmallVector<const SCEV *, 4> Operands; 2253 Operands.push_back(Start); 2254 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step)) 2255 if (StepChrec->getLoop() == L) { 2256 Operands.append(StepChrec->op_begin(), StepChrec->op_end()); 2257 return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW)); 2258 } 2259 2260 Operands.push_back(Step); 2261 return getAddRecExpr(Operands, L, Flags); 2262} 2263 2264/// getAddRecExpr - Get an add recurrence expression for the specified loop. 2265/// Simplify the expression as much as possible. 2266const SCEV * 2267ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands, 2268 const Loop *L, SCEV::NoWrapFlags Flags) { 2269 if (Operands.size() == 1) return Operands[0]; 2270#ifndef NDEBUG 2271 Type *ETy = getEffectiveSCEVType(Operands[0]->getType()); 2272 for (unsigned i = 1, e = Operands.size(); i != e; ++i) 2273 assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy && 2274 "SCEVAddRecExpr operand types don't match!"); 2275 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2276 assert(isLoopInvariant(Operands[i], L) && 2277 "SCEVAddRecExpr operand is not loop-invariant!"); 2278#endif 2279 2280 if (Operands.back()->isZero()) { 2281 Operands.pop_back(); 2282 return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X 2283 } 2284 2285 // It's tempting to want to call getMaxBackedgeTakenCount count here and 2286 // use that information to infer NUW and NSW flags. However, computing a 2287 // BE count requires calling getAddRecExpr, so we may not yet have a 2288 // meaningful BE count at this point (and if we don't, we'd be stuck 2289 // with a SCEVCouldNotCompute as the cached BE count). 2290 2291 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 2292 // And vice-versa. 2293 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 2294 SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 2295 if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 2296 bool All = true; 2297 for (SmallVectorImpl<const SCEV *>::const_iterator I = Operands.begin(), 2298 E = Operands.end(); I != E; ++I) 2299 if (!isKnownNonNegative(*I)) { 2300 All = false; 2301 break; 2302 } 2303 if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 2304 } 2305 2306 // Canonicalize nested AddRecs in by nesting them in order of loop depth. 2307 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) { 2308 const Loop *NestedLoop = NestedAR->getLoop(); 2309 if (L->contains(NestedLoop) ? 2310 (L->getLoopDepth() < NestedLoop->getLoopDepth()) : 2311 (!NestedLoop->contains(L) && 2312 DT->dominates(L->getHeader(), NestedLoop->getHeader()))) { 2313 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(), 2314 NestedAR->op_end()); 2315 Operands[0] = NestedAR->getStart(); 2316 // AddRecs require their operands be loop-invariant with respect to their 2317 // loops. Don't perform this transformation if it would break this 2318 // requirement. 2319 bool AllInvariant = true; 2320 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2321 if (!isLoopInvariant(Operands[i], L)) { 2322 AllInvariant = false; 2323 break; 2324 } 2325 if (AllInvariant) { 2326 // Create a recurrence for the outer loop with the same step size. 2327 // 2328 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the 2329 // inner recurrence has the same property. 2330 SCEV::NoWrapFlags OuterFlags = 2331 maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags()); 2332 2333 NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags); 2334 AllInvariant = true; 2335 for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i) 2336 if (!isLoopInvariant(NestedOperands[i], NestedLoop)) { 2337 AllInvariant = false; 2338 break; 2339 } 2340 if (AllInvariant) { 2341 // Ok, both add recurrences are valid after the transformation. 2342 // 2343 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if 2344 // the outer recurrence has the same property. 2345 SCEV::NoWrapFlags InnerFlags = 2346 maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags); 2347 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags); 2348 } 2349 } 2350 // Reset Operands to its original state. 2351 Operands[0] = NestedAR; 2352 } 2353 } 2354 2355 // Okay, it looks like we really DO need an addrec expr. Check to see if we 2356 // already have one, otherwise create a new one. 2357 FoldingSetNodeID ID; 2358 ID.AddInteger(scAddRecExpr); 2359 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2360 ID.AddPointer(Operands[i]); 2361 ID.AddPointer(L); 2362 void *IP = 0; 2363 SCEVAddRecExpr *S = 2364 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2365 if (!S) { 2366 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size()); 2367 std::uninitialized_copy(Operands.begin(), Operands.end(), O); 2368 S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator), 2369 O, Operands.size(), L); 2370 UniqueSCEVs.InsertNode(S, IP); 2371 } 2372 S->setNoWrapFlags(Flags); 2373 return S; 2374} 2375 2376const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, 2377 const SCEV *RHS) { 2378 SmallVector<const SCEV *, 2> Ops; 2379 Ops.push_back(LHS); 2380 Ops.push_back(RHS); 2381 return getSMaxExpr(Ops); 2382} 2383 2384const SCEV * 2385ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 2386 assert(!Ops.empty() && "Cannot get empty smax!"); 2387 if (Ops.size() == 1) return Ops[0]; 2388#ifndef NDEBUG 2389 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2390 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2391 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2392 "SCEVSMaxExpr operand types don't match!"); 2393#endif 2394 2395 // Sort by complexity, this groups all similar expression types together. 2396 GroupByComplexity(Ops, LI); 2397 2398 // If there are any constants, fold them together. 2399 unsigned Idx = 0; 2400 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2401 ++Idx; 2402 assert(Idx < Ops.size()); 2403 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2404 // We found two constants, fold them together! 2405 ConstantInt *Fold = ConstantInt::get(getContext(), 2406 APIntOps::smax(LHSC->getValue()->getValue(), 2407 RHSC->getValue()->getValue())); 2408 Ops[0] = getConstant(Fold); 2409 Ops.erase(Ops.begin()+1); // Erase the folded element 2410 if (Ops.size() == 1) return Ops[0]; 2411 LHSC = cast<SCEVConstant>(Ops[0]); 2412 } 2413 2414 // If we are left with a constant minimum-int, strip it off. 2415 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) { 2416 Ops.erase(Ops.begin()); 2417 --Idx; 2418 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) { 2419 // If we have an smax with a constant maximum-int, it will always be 2420 // maximum-int. 2421 return Ops[0]; 2422 } 2423 2424 if (Ops.size() == 1) return Ops[0]; 2425 } 2426 2427 // Find the first SMax 2428 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr) 2429 ++Idx; 2430 2431 // Check to see if one of the operands is an SMax. If so, expand its operands 2432 // onto our operand list, and recurse to simplify. 2433 if (Idx < Ops.size()) { 2434 bool DeletedSMax = false; 2435 while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) { 2436 Ops.erase(Ops.begin()+Idx); 2437 Ops.append(SMax->op_begin(), SMax->op_end()); 2438 DeletedSMax = true; 2439 } 2440 2441 if (DeletedSMax) 2442 return getSMaxExpr(Ops); 2443 } 2444 2445 // Okay, check to see if the same value occurs in the operand list twice. If 2446 // so, delete one. Since we sorted the list, these values are required to 2447 // be adjacent. 2448 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 2449 // X smax Y smax Y --> X smax Y 2450 // X smax Y --> X, if X is always greater than Y 2451 if (Ops[i] == Ops[i+1] || 2452 isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) { 2453 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 2454 --i; --e; 2455 } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) { 2456 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 2457 --i; --e; 2458 } 2459 2460 if (Ops.size() == 1) return Ops[0]; 2461 2462 assert(!Ops.empty() && "Reduced smax down to nothing!"); 2463 2464 // Okay, it looks like we really DO need an smax expr. Check to see if we 2465 // already have one, otherwise create a new one. 2466 FoldingSetNodeID ID; 2467 ID.AddInteger(scSMaxExpr); 2468 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2469 ID.AddPointer(Ops[i]); 2470 void *IP = 0; 2471 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2472 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2473 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2474 SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator), 2475 O, Ops.size()); 2476 UniqueSCEVs.InsertNode(S, IP); 2477 return S; 2478} 2479 2480const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, 2481 const SCEV *RHS) { 2482 SmallVector<const SCEV *, 2> Ops; 2483 Ops.push_back(LHS); 2484 Ops.push_back(RHS); 2485 return getUMaxExpr(Ops); 2486} 2487 2488const SCEV * 2489ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 2490 assert(!Ops.empty() && "Cannot get empty umax!"); 2491 if (Ops.size() == 1) return Ops[0]; 2492#ifndef NDEBUG 2493 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2494 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2495 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2496 "SCEVUMaxExpr operand types don't match!"); 2497#endif 2498 2499 // Sort by complexity, this groups all similar expression types together. 2500 GroupByComplexity(Ops, LI); 2501 2502 // If there are any constants, fold them together. 2503 unsigned Idx = 0; 2504 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2505 ++Idx; 2506 assert(Idx < Ops.size()); 2507 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2508 // We found two constants, fold them together! 2509 ConstantInt *Fold = ConstantInt::get(getContext(), 2510 APIntOps::umax(LHSC->getValue()->getValue(), 2511 RHSC->getValue()->getValue())); 2512 Ops[0] = getConstant(Fold); 2513 Ops.erase(Ops.begin()+1); // Erase the folded element 2514 if (Ops.size() == 1) return Ops[0]; 2515 LHSC = cast<SCEVConstant>(Ops[0]); 2516 } 2517 2518 // If we are left with a constant minimum-int, strip it off. 2519 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) { 2520 Ops.erase(Ops.begin()); 2521 --Idx; 2522 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) { 2523 // If we have an umax with a constant maximum-int, it will always be 2524 // maximum-int. 2525 return Ops[0]; 2526 } 2527 2528 if (Ops.size() == 1) return Ops[0]; 2529 } 2530 2531 // Find the first UMax 2532 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr) 2533 ++Idx; 2534 2535 // Check to see if one of the operands is a UMax. If so, expand its operands 2536 // onto our operand list, and recurse to simplify. 2537 if (Idx < Ops.size()) { 2538 bool DeletedUMax = false; 2539 while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) { 2540 Ops.erase(Ops.begin()+Idx); 2541 Ops.append(UMax->op_begin(), UMax->op_end()); 2542 DeletedUMax = true; 2543 } 2544 2545 if (DeletedUMax) 2546 return getUMaxExpr(Ops); 2547 } 2548 2549 // Okay, check to see if the same value occurs in the operand list twice. If 2550 // so, delete one. Since we sorted the list, these values are required to 2551 // be adjacent. 2552 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 2553 // X umax Y umax Y --> X umax Y 2554 // X umax Y --> X, if X is always greater than Y 2555 if (Ops[i] == Ops[i+1] || 2556 isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) { 2557 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 2558 --i; --e; 2559 } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) { 2560 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 2561 --i; --e; 2562 } 2563 2564 if (Ops.size() == 1) return Ops[0]; 2565 2566 assert(!Ops.empty() && "Reduced umax down to nothing!"); 2567 2568 // Okay, it looks like we really DO need a umax expr. Check to see if we 2569 // already have one, otherwise create a new one. 2570 FoldingSetNodeID ID; 2571 ID.AddInteger(scUMaxExpr); 2572 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2573 ID.AddPointer(Ops[i]); 2574 void *IP = 0; 2575 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2576 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2577 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2578 SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator), 2579 O, Ops.size()); 2580 UniqueSCEVs.InsertNode(S, IP); 2581 return S; 2582} 2583 2584const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS, 2585 const SCEV *RHS) { 2586 // ~smax(~x, ~y) == smin(x, y). 2587 return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 2588} 2589 2590const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, 2591 const SCEV *RHS) { 2592 // ~umax(~x, ~y) == umin(x, y) 2593 return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 2594} 2595 2596const SCEV *ScalarEvolution::getSizeOfExpr(Type *AllocTy) { 2597 // If we have TargetData, we can bypass creating a target-independent 2598 // constant expression and then folding it back into a ConstantInt. 2599 // This is just a compile-time optimization. 2600 if (TD) 2601 return getConstant(TD->getIntPtrType(getContext()), 2602 TD->getTypeAllocSize(AllocTy)); 2603 2604 Constant *C = ConstantExpr::getSizeOf(AllocTy); 2605 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2606 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2607 C = Folded; 2608 Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy)); 2609 return getTruncateOrZeroExtend(getSCEV(C), Ty); 2610} 2611 2612const SCEV *ScalarEvolution::getAlignOfExpr(Type *AllocTy) { 2613 Constant *C = ConstantExpr::getAlignOf(AllocTy); 2614 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2615 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2616 C = Folded; 2617 Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy)); 2618 return getTruncateOrZeroExtend(getSCEV(C), Ty); 2619} 2620 2621const SCEV *ScalarEvolution::getOffsetOfExpr(StructType *STy, 2622 unsigned FieldNo) { 2623 // If we have TargetData, we can bypass creating a target-independent 2624 // constant expression and then folding it back into a ConstantInt. 2625 // This is just a compile-time optimization. 2626 if (TD) 2627 return getConstant(TD->getIntPtrType(getContext()), 2628 TD->getStructLayout(STy)->getElementOffset(FieldNo)); 2629 2630 Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo); 2631 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2632 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2633 C = Folded; 2634 Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy)); 2635 return getTruncateOrZeroExtend(getSCEV(C), Ty); 2636} 2637 2638const SCEV *ScalarEvolution::getOffsetOfExpr(Type *CTy, 2639 Constant *FieldNo) { 2640 Constant *C = ConstantExpr::getOffsetOf(CTy, FieldNo); 2641 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2642 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2643 C = Folded; 2644 Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(CTy)); 2645 return getTruncateOrZeroExtend(getSCEV(C), Ty); 2646} 2647 2648const SCEV *ScalarEvolution::getUnknown(Value *V) { 2649 // Don't attempt to do anything other than create a SCEVUnknown object 2650 // here. createSCEV only calls getUnknown after checking for all other 2651 // interesting possibilities, and any other code that calls getUnknown 2652 // is doing so in order to hide a value from SCEV canonicalization. 2653 2654 FoldingSetNodeID ID; 2655 ID.AddInteger(scUnknown); 2656 ID.AddPointer(V); 2657 void *IP = 0; 2658 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) { 2659 assert(cast<SCEVUnknown>(S)->getValue() == V && 2660 "Stale SCEVUnknown in uniquing map!"); 2661 return S; 2662 } 2663 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this, 2664 FirstUnknown); 2665 FirstUnknown = cast<SCEVUnknown>(S); 2666 UniqueSCEVs.InsertNode(S, IP); 2667 return S; 2668} 2669 2670//===----------------------------------------------------------------------===// 2671// Basic SCEV Analysis and PHI Idiom Recognition Code 2672// 2673 2674/// isSCEVable - Test if values of the given type are analyzable within 2675/// the SCEV framework. This primarily includes integer types, and it 2676/// can optionally include pointer types if the ScalarEvolution class 2677/// has access to target-specific information. 2678bool ScalarEvolution::isSCEVable(Type *Ty) const { 2679 // Integers and pointers are always SCEVable. 2680 return Ty->isIntegerTy() || Ty->isPointerTy(); 2681} 2682 2683/// getTypeSizeInBits - Return the size in bits of the specified type, 2684/// for which isSCEVable must return true. 2685uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const { 2686 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 2687 2688 // If we have a TargetData, use it! 2689 if (TD) 2690 return TD->getTypeSizeInBits(Ty); 2691 2692 // Integer types have fixed sizes. 2693 if (Ty->isIntegerTy()) 2694 return Ty->getPrimitiveSizeInBits(); 2695 2696 // The only other support type is pointer. Without TargetData, conservatively 2697 // assume pointers are 64-bit. 2698 assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!"); 2699 return 64; 2700} 2701 2702/// getEffectiveSCEVType - Return a type with the same bitwidth as 2703/// the given type and which represents how SCEV will treat the given 2704/// type, for which isSCEVable must return true. For pointer types, 2705/// this is the pointer-sized integer type. 2706Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const { 2707 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 2708 2709 if (Ty->isIntegerTy()) 2710 return Ty; 2711 2712 // The only other support type is pointer. 2713 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!"); 2714 if (TD) return TD->getIntPtrType(getContext()); 2715 2716 // Without TargetData, conservatively assume pointers are 64-bit. 2717 return Type::getInt64Ty(getContext()); 2718} 2719 2720const SCEV *ScalarEvolution::getCouldNotCompute() { 2721 return &CouldNotCompute; 2722} 2723 2724/// getSCEV - Return an existing SCEV if it exists, otherwise analyze the 2725/// expression and create a new one. 2726const SCEV *ScalarEvolution::getSCEV(Value *V) { 2727 assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); 2728 2729 ValueExprMapType::const_iterator I = ValueExprMap.find(V); 2730 if (I != ValueExprMap.end()) return I->second; 2731 const SCEV *S = createSCEV(V); 2732 2733 // The process of creating a SCEV for V may have caused other SCEVs 2734 // to have been created, so it's necessary to insert the new entry 2735 // from scratch, rather than trying to remember the insert position 2736 // above. 2737 ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S)); 2738 return S; 2739} 2740 2741/// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V 2742/// 2743const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) { 2744 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 2745 return getConstant( 2746 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue()))); 2747 2748 Type *Ty = V->getType(); 2749 Ty = getEffectiveSCEVType(Ty); 2750 return getMulExpr(V, 2751 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)))); 2752} 2753 2754/// getNotSCEV - Return a SCEV corresponding to ~V = -1-V 2755const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { 2756 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 2757 return getConstant( 2758 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue()))); 2759 2760 Type *Ty = V->getType(); 2761 Ty = getEffectiveSCEVType(Ty); 2762 const SCEV *AllOnes = 2763 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))); 2764 return getMinusSCEV(AllOnes, V); 2765} 2766 2767/// getMinusSCEV - Return LHS-RHS. Minus is represented in SCEV as A+B*-1. 2768const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, 2769 SCEV::NoWrapFlags Flags) { 2770 assert(!maskFlags(Flags, SCEV::FlagNUW) && "subtraction does not have NUW"); 2771 2772 // Fast path: X - X --> 0. 2773 if (LHS == RHS) 2774 return getConstant(LHS->getType(), 0); 2775 2776 // X - Y --> X + -Y 2777 return getAddExpr(LHS, getNegativeSCEV(RHS), Flags); 2778} 2779 2780/// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the 2781/// input value to the specified type. If the type must be extended, it is zero 2782/// extended. 2783const SCEV * 2784ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) { 2785 Type *SrcTy = V->getType(); 2786 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2787 (Ty->isIntegerTy() || Ty->isPointerTy()) && 2788 "Cannot truncate or zero extend with non-integer arguments!"); 2789 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2790 return V; // No conversion 2791 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 2792 return getTruncateExpr(V, Ty); 2793 return getZeroExtendExpr(V, Ty); 2794} 2795 2796/// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the 2797/// input value to the specified type. If the type must be extended, it is sign 2798/// extended. 2799const SCEV * 2800ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, 2801 Type *Ty) { 2802 Type *SrcTy = V->getType(); 2803 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2804 (Ty->isIntegerTy() || Ty->isPointerTy()) && 2805 "Cannot truncate or zero extend with non-integer arguments!"); 2806 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2807 return V; // No conversion 2808 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 2809 return getTruncateExpr(V, Ty); 2810 return getSignExtendExpr(V, Ty); 2811} 2812 2813/// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the 2814/// input value to the specified type. If the type must be extended, it is zero 2815/// extended. The conversion must not be narrowing. 2816const SCEV * 2817ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) { 2818 Type *SrcTy = V->getType(); 2819 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2820 (Ty->isIntegerTy() || Ty->isPointerTy()) && 2821 "Cannot noop or zero extend with non-integer arguments!"); 2822 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 2823 "getNoopOrZeroExtend cannot truncate!"); 2824 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2825 return V; // No conversion 2826 return getZeroExtendExpr(V, Ty); 2827} 2828 2829/// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the 2830/// input value to the specified type. If the type must be extended, it is sign 2831/// extended. The conversion must not be narrowing. 2832const SCEV * 2833ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) { 2834 Type *SrcTy = V->getType(); 2835 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2836 (Ty->isIntegerTy() || Ty->isPointerTy()) && 2837 "Cannot noop or sign extend with non-integer arguments!"); 2838 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 2839 "getNoopOrSignExtend cannot truncate!"); 2840 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2841 return V; // No conversion 2842 return getSignExtendExpr(V, Ty); 2843} 2844 2845/// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of 2846/// the input value to the specified type. If the type must be extended, 2847/// it is extended with unspecified bits. The conversion must not be 2848/// narrowing. 2849const SCEV * 2850ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) { 2851 Type *SrcTy = V->getType(); 2852 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2853 (Ty->isIntegerTy() || Ty->isPointerTy()) && 2854 "Cannot noop or any extend with non-integer arguments!"); 2855 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 2856 "getNoopOrAnyExtend cannot truncate!"); 2857 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2858 return V; // No conversion 2859 return getAnyExtendExpr(V, Ty); 2860} 2861 2862/// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the 2863/// input value to the specified type. The conversion must not be widening. 2864const SCEV * 2865ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) { 2866 Type *SrcTy = V->getType(); 2867 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2868 (Ty->isIntegerTy() || Ty->isPointerTy()) && 2869 "Cannot truncate or noop with non-integer arguments!"); 2870 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) && 2871 "getTruncateOrNoop cannot extend!"); 2872 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2873 return V; // No conversion 2874 return getTruncateExpr(V, Ty); 2875} 2876 2877/// getUMaxFromMismatchedTypes - Promote the operands to the wider of 2878/// the types using zero-extension, and then perform a umax operation 2879/// with them. 2880const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS, 2881 const SCEV *RHS) { 2882 const SCEV *PromotedLHS = LHS; 2883 const SCEV *PromotedRHS = RHS; 2884 2885 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 2886 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 2887 else 2888 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 2889 2890 return getUMaxExpr(PromotedLHS, PromotedRHS); 2891} 2892 2893/// getUMinFromMismatchedTypes - Promote the operands to the wider of 2894/// the types using zero-extension, and then perform a umin operation 2895/// with them. 2896const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS, 2897 const SCEV *RHS) { 2898 const SCEV *PromotedLHS = LHS; 2899 const SCEV *PromotedRHS = RHS; 2900 2901 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 2902 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 2903 else 2904 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 2905 2906 return getUMinExpr(PromotedLHS, PromotedRHS); 2907} 2908 2909/// getPointerBase - Transitively follow the chain of pointer-type operands 2910/// until reaching a SCEV that does not have a single pointer operand. This 2911/// returns a SCEVUnknown pointer for well-formed pointer-type expressions, 2912/// but corner cases do exist. 2913const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) { 2914 // A pointer operand may evaluate to a nonpointer expression, such as null. 2915 if (!V->getType()->isPointerTy()) 2916 return V; 2917 2918 if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) { 2919 return getPointerBase(Cast->getOperand()); 2920 } 2921 else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) { 2922 const SCEV *PtrOp = 0; 2923 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 2924 I != E; ++I) { 2925 if ((*I)->getType()->isPointerTy()) { 2926 // Cannot find the base of an expression with multiple pointer operands. 2927 if (PtrOp) 2928 return V; 2929 PtrOp = *I; 2930 } 2931 } 2932 if (!PtrOp) 2933 return V; 2934 return getPointerBase(PtrOp); 2935 } 2936 return V; 2937} 2938 2939/// PushDefUseChildren - Push users of the given Instruction 2940/// onto the given Worklist. 2941static void 2942PushDefUseChildren(Instruction *I, 2943 SmallVectorImpl<Instruction *> &Worklist) { 2944 // Push the def-use children onto the Worklist stack. 2945 for (Value::use_iterator UI = I->use_begin(), UE = I->use_end(); 2946 UI != UE; ++UI) 2947 Worklist.push_back(cast<Instruction>(*UI)); 2948} 2949 2950/// ForgetSymbolicValue - This looks up computed SCEV values for all 2951/// instructions that depend on the given instruction and removes them from 2952/// the ValueExprMapType map if they reference SymName. This is used during PHI 2953/// resolution. 2954void 2955ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) { 2956 SmallVector<Instruction *, 16> Worklist; 2957 PushDefUseChildren(PN, Worklist); 2958 2959 SmallPtrSet<Instruction *, 8> Visited; 2960 Visited.insert(PN); 2961 while (!Worklist.empty()) { 2962 Instruction *I = Worklist.pop_back_val(); 2963 if (!Visited.insert(I)) continue; 2964 2965 ValueExprMapType::iterator It = 2966 ValueExprMap.find(static_cast<Value *>(I)); 2967 if (It != ValueExprMap.end()) { 2968 const SCEV *Old = It->second; 2969 2970 // Short-circuit the def-use traversal if the symbolic name 2971 // ceases to appear in expressions. 2972 if (Old != SymName && !hasOperand(Old, SymName)) 2973 continue; 2974 2975 // SCEVUnknown for a PHI either means that it has an unrecognized 2976 // structure, it's a PHI that's in the progress of being computed 2977 // by createNodeForPHI, or it's a single-value PHI. In the first case, 2978 // additional loop trip count information isn't going to change anything. 2979 // In the second case, createNodeForPHI will perform the necessary 2980 // updates on its own when it gets to that point. In the third, we do 2981 // want to forget the SCEVUnknown. 2982 if (!isa<PHINode>(I) || 2983 !isa<SCEVUnknown>(Old) || 2984 (I != PN && Old == SymName)) { 2985 forgetMemoizedResults(Old); 2986 ValueExprMap.erase(It); 2987 } 2988 } 2989 2990 PushDefUseChildren(I, Worklist); 2991 } 2992} 2993 2994/// createNodeForPHI - PHI nodes have two cases. Either the PHI node exists in 2995/// a loop header, making it a potential recurrence, or it doesn't. 2996/// 2997const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { 2998 if (const Loop *L = LI->getLoopFor(PN->getParent())) 2999 if (L->getHeader() == PN->getParent()) { 3000 // The loop may have multiple entrances or multiple exits; we can analyze 3001 // this phi as an addrec if it has a unique entry value and a unique 3002 // backedge value. 3003 Value *BEValueV = 0, *StartValueV = 0; 3004 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 3005 Value *V = PN->getIncomingValue(i); 3006 if (L->contains(PN->getIncomingBlock(i))) { 3007 if (!BEValueV) { 3008 BEValueV = V; 3009 } else if (BEValueV != V) { 3010 BEValueV = 0; 3011 break; 3012 } 3013 } else if (!StartValueV) { 3014 StartValueV = V; 3015 } else if (StartValueV != V) { 3016 StartValueV = 0; 3017 break; 3018 } 3019 } 3020 if (BEValueV && StartValueV) { 3021 // While we are analyzing this PHI node, handle its value symbolically. 3022 const SCEV *SymbolicName = getUnknown(PN); 3023 assert(ValueExprMap.find(PN) == ValueExprMap.end() && 3024 "PHI node already processed?"); 3025 ValueExprMap.insert(std::make_pair(SCEVCallbackVH(PN, this), SymbolicName)); 3026 3027 // Using this symbolic name for the PHI, analyze the value coming around 3028 // the back-edge. 3029 const SCEV *BEValue = getSCEV(BEValueV); 3030 3031 // NOTE: If BEValue is loop invariant, we know that the PHI node just 3032 // has a special value for the first iteration of the loop. 3033 3034 // If the value coming around the backedge is an add with the symbolic 3035 // value we just inserted, then we found a simple induction variable! 3036 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) { 3037 // If there is a single occurrence of the symbolic value, replace it 3038 // with a recurrence. 3039 unsigned FoundIndex = Add->getNumOperands(); 3040 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3041 if (Add->getOperand(i) == SymbolicName) 3042 if (FoundIndex == e) { 3043 FoundIndex = i; 3044 break; 3045 } 3046 3047 if (FoundIndex != Add->getNumOperands()) { 3048 // Create an add with everything but the specified operand. 3049 SmallVector<const SCEV *, 8> Ops; 3050 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3051 if (i != FoundIndex) 3052 Ops.push_back(Add->getOperand(i)); 3053 const SCEV *Accum = getAddExpr(Ops); 3054 3055 // This is not a valid addrec if the step amount is varying each 3056 // loop iteration, but is not itself an addrec in this loop. 3057 if (isLoopInvariant(Accum, L) || 3058 (isa<SCEVAddRecExpr>(Accum) && 3059 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) { 3060 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 3061 3062 // If the increment doesn't overflow, then neither the addrec nor 3063 // the post-increment will overflow. 3064 if (const AddOperator *OBO = dyn_cast<AddOperator>(BEValueV)) { 3065 if (OBO->hasNoUnsignedWrap()) 3066 Flags = setFlags(Flags, SCEV::FlagNUW); 3067 if (OBO->hasNoSignedWrap()) 3068 Flags = setFlags(Flags, SCEV::FlagNSW); 3069 } else if (const GEPOperator *GEP = 3070 dyn_cast<GEPOperator>(BEValueV)) { 3071 // If the increment is an inbounds GEP, then we know the address 3072 // space cannot be wrapped around. We cannot make any guarantee 3073 // about signed or unsigned overflow because pointers are 3074 // unsigned but we may have a negative index from the base 3075 // pointer. 3076 if (GEP->isInBounds()) 3077 Flags = setFlags(Flags, SCEV::FlagNW); 3078 } 3079 3080 const SCEV *StartVal = getSCEV(StartValueV); 3081 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); 3082 3083 // Since the no-wrap flags are on the increment, they apply to the 3084 // post-incremented value as well. 3085 if (isLoopInvariant(Accum, L)) 3086 (void)getAddRecExpr(getAddExpr(StartVal, Accum), 3087 Accum, L, Flags); 3088 3089 // Okay, for the entire analysis of this edge we assumed the PHI 3090 // to be symbolic. We now need to go back and purge all of the 3091 // entries for the scalars that use the symbolic expression. 3092 ForgetSymbolicName(PN, SymbolicName); 3093 ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 3094 return PHISCEV; 3095 } 3096 } 3097 } else if (const SCEVAddRecExpr *AddRec = 3098 dyn_cast<SCEVAddRecExpr>(BEValue)) { 3099 // Otherwise, this could be a loop like this: 3100 // i = 0; for (j = 1; ..; ++j) { .... i = j; } 3101 // In this case, j = {1,+,1} and BEValue is j. 3102 // Because the other in-value of i (0) fits the evolution of BEValue 3103 // i really is an addrec evolution. 3104 if (AddRec->getLoop() == L && AddRec->isAffine()) { 3105 const SCEV *StartVal = getSCEV(StartValueV); 3106 3107 // If StartVal = j.start - j.stride, we can use StartVal as the 3108 // initial step of the addrec evolution. 3109 if (StartVal == getMinusSCEV(AddRec->getOperand(0), 3110 AddRec->getOperand(1))) { 3111 // FIXME: For constant StartVal, we should be able to infer 3112 // no-wrap flags. 3113 const SCEV *PHISCEV = 3114 getAddRecExpr(StartVal, AddRec->getOperand(1), L, 3115 SCEV::FlagAnyWrap); 3116 3117 // Okay, for the entire analysis of this edge we assumed the PHI 3118 // to be symbolic. We now need to go back and purge all of the 3119 // entries for the scalars that use the symbolic expression. 3120 ForgetSymbolicName(PN, SymbolicName); 3121 ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 3122 return PHISCEV; 3123 } 3124 } 3125 } 3126 } 3127 } 3128 3129 // If the PHI has a single incoming value, follow that value, unless the 3130 // PHI's incoming blocks are in a different loop, in which case doing so 3131 // risks breaking LCSSA form. Instcombine would normally zap these, but 3132 // it doesn't have DominatorTree information, so it may miss cases. 3133 if (Value *V = SimplifyInstruction(PN, TD, TLI, DT)) 3134 if (LI->replacementPreservesLCSSAForm(PN, V)) 3135 return getSCEV(V); 3136 3137 // If it's not a loop phi, we can't handle it yet. 3138 return getUnknown(PN); 3139} 3140 3141/// createNodeForGEP - Expand GEP instructions into add and multiply 3142/// operations. This allows them to be analyzed by regular SCEV code. 3143/// 3144const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { 3145 3146 // Don't blindly transfer the inbounds flag from the GEP instruction to the 3147 // Add expression, because the Instruction may be guarded by control flow 3148 // and the no-overflow bits may not be valid for the expression in any 3149 // context. 3150 bool isInBounds = GEP->isInBounds(); 3151 3152 Type *IntPtrTy = getEffectiveSCEVType(GEP->getType()); 3153 Value *Base = GEP->getOperand(0); 3154 // Don't attempt to analyze GEPs over unsized objects. 3155 if (!cast<PointerType>(Base->getType())->getElementType()->isSized()) 3156 return getUnknown(GEP); 3157 const SCEV *TotalOffset = getConstant(IntPtrTy, 0); 3158 gep_type_iterator GTI = gep_type_begin(GEP); 3159 for (GetElementPtrInst::op_iterator I = llvm::next(GEP->op_begin()), 3160 E = GEP->op_end(); 3161 I != E; ++I) { 3162 Value *Index = *I; 3163 // Compute the (potentially symbolic) offset in bytes for this index. 3164 if (StructType *STy = dyn_cast<StructType>(*GTI++)) { 3165 // For a struct, add the member offset. 3166 unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue(); 3167 const SCEV *FieldOffset = getOffsetOfExpr(STy, FieldNo); 3168 3169 // Add the field offset to the running total offset. 3170 TotalOffset = getAddExpr(TotalOffset, FieldOffset); 3171 } else { 3172 // For an array, add the element offset, explicitly scaled. 3173 const SCEV *ElementSize = getSizeOfExpr(*GTI); 3174 const SCEV *IndexS = getSCEV(Index); 3175 // Getelementptr indices are signed. 3176 IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy); 3177 3178 // Multiply the index by the element size to compute the element offset. 3179 const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, 3180 isInBounds ? SCEV::FlagNSW : 3181 SCEV::FlagAnyWrap); 3182 3183 // Add the element offset to the running total offset. 3184 TotalOffset = getAddExpr(TotalOffset, LocalOffset); 3185 } 3186 } 3187 3188 // Get the SCEV for the GEP base. 3189 const SCEV *BaseS = getSCEV(Base); 3190 3191 // Add the total offset from all the GEP indices to the base. 3192 return getAddExpr(BaseS, TotalOffset, 3193 isInBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap); 3194} 3195 3196/// GetMinTrailingZeros - Determine the minimum number of zero bits that S is 3197/// guaranteed to end in (at every loop iteration). It is, at the same time, 3198/// the minimum number of times S is divisible by 2. For example, given {4,+,8} 3199/// it returns 2. If S is guaranteed to be 0, it returns the bitwidth of S. 3200uint32_t 3201ScalarEvolution::GetMinTrailingZeros(const SCEV *S) { 3202 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3203 return C->getValue()->getValue().countTrailingZeros(); 3204 3205 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S)) 3206 return std::min(GetMinTrailingZeros(T->getOperand()), 3207 (uint32_t)getTypeSizeInBits(T->getType())); 3208 3209 if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) { 3210 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 3211 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 3212 getTypeSizeInBits(E->getType()) : OpRes; 3213 } 3214 3215 if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) { 3216 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 3217 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 3218 getTypeSizeInBits(E->getType()) : OpRes; 3219 } 3220 3221 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) { 3222 // The result is the min of all operands results. 3223 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 3224 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 3225 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 3226 return MinOpRes; 3227 } 3228 3229 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) { 3230 // The result is the sum of all operands results. 3231 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0)); 3232 uint32_t BitWidth = getTypeSizeInBits(M->getType()); 3233 for (unsigned i = 1, e = M->getNumOperands(); 3234 SumOpRes != BitWidth && i != e; ++i) 3235 SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)), 3236 BitWidth); 3237 return SumOpRes; 3238 } 3239 3240 if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) { 3241 // The result is the min of all operands results. 3242 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 3243 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 3244 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 3245 return MinOpRes; 3246 } 3247 3248 if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) { 3249 // The result is the min of all operands results. 3250 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 3251 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 3252 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 3253 return MinOpRes; 3254 } 3255 3256 if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) { 3257 // The result is the min of all operands results. 3258 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 3259 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 3260 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 3261 return MinOpRes; 3262 } 3263 3264 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3265 // For a SCEVUnknown, ask ValueTracking. 3266 unsigned BitWidth = getTypeSizeInBits(U->getType()); 3267 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 3268 ComputeMaskedBits(U->getValue(), Zeros, Ones); 3269 return Zeros.countTrailingOnes(); 3270 } 3271 3272 // SCEVUDivExpr 3273 return 0; 3274} 3275 3276/// getUnsignedRange - Determine the unsigned range for a particular SCEV. 3277/// 3278ConstantRange 3279ScalarEvolution::getUnsignedRange(const SCEV *S) { 3280 // See if we've computed this range already. 3281 DenseMap<const SCEV *, ConstantRange>::iterator I = UnsignedRanges.find(S); 3282 if (I != UnsignedRanges.end()) 3283 return I->second; 3284 3285 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3286 return setUnsignedRange(C, ConstantRange(C->getValue()->getValue())); 3287 3288 unsigned BitWidth = getTypeSizeInBits(S->getType()); 3289 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 3290 3291 // If the value has known zeros, the maximum unsigned value will have those 3292 // known zeros as well. 3293 uint32_t TZ = GetMinTrailingZeros(S); 3294 if (TZ != 0) 3295 ConservativeResult = 3296 ConstantRange(APInt::getMinValue(BitWidth), 3297 APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1); 3298 3299 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 3300 ConstantRange X = getUnsignedRange(Add->getOperand(0)); 3301 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 3302 X = X.add(getUnsignedRange(Add->getOperand(i))); 3303 return setUnsignedRange(Add, ConservativeResult.intersectWith(X)); 3304 } 3305 3306 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3307 ConstantRange X = getUnsignedRange(Mul->getOperand(0)); 3308 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 3309 X = X.multiply(getUnsignedRange(Mul->getOperand(i))); 3310 return setUnsignedRange(Mul, ConservativeResult.intersectWith(X)); 3311 } 3312 3313 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 3314 ConstantRange X = getUnsignedRange(SMax->getOperand(0)); 3315 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 3316 X = X.smax(getUnsignedRange(SMax->getOperand(i))); 3317 return setUnsignedRange(SMax, ConservativeResult.intersectWith(X)); 3318 } 3319 3320 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 3321 ConstantRange X = getUnsignedRange(UMax->getOperand(0)); 3322 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 3323 X = X.umax(getUnsignedRange(UMax->getOperand(i))); 3324 return setUnsignedRange(UMax, ConservativeResult.intersectWith(X)); 3325 } 3326 3327 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 3328 ConstantRange X = getUnsignedRange(UDiv->getLHS()); 3329 ConstantRange Y = getUnsignedRange(UDiv->getRHS()); 3330 return setUnsignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); 3331 } 3332 3333 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 3334 ConstantRange X = getUnsignedRange(ZExt->getOperand()); 3335 return setUnsignedRange(ZExt, 3336 ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 3337 } 3338 3339 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 3340 ConstantRange X = getUnsignedRange(SExt->getOperand()); 3341 return setUnsignedRange(SExt, 3342 ConservativeResult.intersectWith(X.signExtend(BitWidth))); 3343 } 3344 3345 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 3346 ConstantRange X = getUnsignedRange(Trunc->getOperand()); 3347 return setUnsignedRange(Trunc, 3348 ConservativeResult.intersectWith(X.truncate(BitWidth))); 3349 } 3350 3351 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 3352 // If there's no unsigned wrap, the value will never be less than its 3353 // initial value. 3354 if (AddRec->getNoWrapFlags(SCEV::FlagNUW)) 3355 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart())) 3356 if (!C->getValue()->isZero()) 3357 ConservativeResult = 3358 ConservativeResult.intersectWith( 3359 ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0))); 3360 3361 // TODO: non-affine addrec 3362 if (AddRec->isAffine()) { 3363 Type *Ty = AddRec->getType(); 3364 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 3365 if (!isa<SCEVCouldNotCompute>(MaxBECount) && 3366 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 3367 MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); 3368 3369 const SCEV *Start = AddRec->getStart(); 3370 const SCEV *Step = AddRec->getStepRecurrence(*this); 3371 3372 ConstantRange StartRange = getUnsignedRange(Start); 3373 ConstantRange StepRange = getSignedRange(Step); 3374 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 3375 ConstantRange EndRange = 3376 StartRange.add(MaxBECountRange.multiply(StepRange)); 3377 3378 // Check for overflow. This must be done with ConstantRange arithmetic 3379 // because we could be called from within the ScalarEvolution overflow 3380 // checking code. 3381 ConstantRange ExtStartRange = StartRange.zextOrTrunc(BitWidth*2+1); 3382 ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); 3383 ConstantRange ExtMaxBECountRange = 3384 MaxBECountRange.zextOrTrunc(BitWidth*2+1); 3385 ConstantRange ExtEndRange = EndRange.zextOrTrunc(BitWidth*2+1); 3386 if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != 3387 ExtEndRange) 3388 return setUnsignedRange(AddRec, ConservativeResult); 3389 3390 APInt Min = APIntOps::umin(StartRange.getUnsignedMin(), 3391 EndRange.getUnsignedMin()); 3392 APInt Max = APIntOps::umax(StartRange.getUnsignedMax(), 3393 EndRange.getUnsignedMax()); 3394 if (Min.isMinValue() && Max.isMaxValue()) 3395 return setUnsignedRange(AddRec, ConservativeResult); 3396 return setUnsignedRange(AddRec, 3397 ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); 3398 } 3399 } 3400 3401 return setUnsignedRange(AddRec, ConservativeResult); 3402 } 3403 3404 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3405 // For a SCEVUnknown, ask ValueTracking. 3406 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 3407 ComputeMaskedBits(U->getValue(), Zeros, Ones, TD); 3408 if (Ones == ~Zeros + 1) 3409 return setUnsignedRange(U, ConservativeResult); 3410 return setUnsignedRange(U, 3411 ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1))); 3412 } 3413 3414 return setUnsignedRange(S, ConservativeResult); 3415} 3416 3417/// getSignedRange - Determine the signed range for a particular SCEV. 3418/// 3419ConstantRange 3420ScalarEvolution::getSignedRange(const SCEV *S) { 3421 // See if we've computed this range already. 3422 DenseMap<const SCEV *, ConstantRange>::iterator I = SignedRanges.find(S); 3423 if (I != SignedRanges.end()) 3424 return I->second; 3425 3426 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3427 return setSignedRange(C, ConstantRange(C->getValue()->getValue())); 3428 3429 unsigned BitWidth = getTypeSizeInBits(S->getType()); 3430 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 3431 3432 // If the value has known zeros, the maximum signed value will have those 3433 // known zeros as well. 3434 uint32_t TZ = GetMinTrailingZeros(S); 3435 if (TZ != 0) 3436 ConservativeResult = 3437 ConstantRange(APInt::getSignedMinValue(BitWidth), 3438 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1); 3439 3440 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 3441 ConstantRange X = getSignedRange(Add->getOperand(0)); 3442 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 3443 X = X.add(getSignedRange(Add->getOperand(i))); 3444 return setSignedRange(Add, ConservativeResult.intersectWith(X)); 3445 } 3446 3447 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3448 ConstantRange X = getSignedRange(Mul->getOperand(0)); 3449 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 3450 X = X.multiply(getSignedRange(Mul->getOperand(i))); 3451 return setSignedRange(Mul, ConservativeResult.intersectWith(X)); 3452 } 3453 3454 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 3455 ConstantRange X = getSignedRange(SMax->getOperand(0)); 3456 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 3457 X = X.smax(getSignedRange(SMax->getOperand(i))); 3458 return setSignedRange(SMax, ConservativeResult.intersectWith(X)); 3459 } 3460 3461 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 3462 ConstantRange X = getSignedRange(UMax->getOperand(0)); 3463 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 3464 X = X.umax(getSignedRange(UMax->getOperand(i))); 3465 return setSignedRange(UMax, ConservativeResult.intersectWith(X)); 3466 } 3467 3468 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 3469 ConstantRange X = getSignedRange(UDiv->getLHS()); 3470 ConstantRange Y = getSignedRange(UDiv->getRHS()); 3471 return setSignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); 3472 } 3473 3474 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 3475 ConstantRange X = getSignedRange(ZExt->getOperand()); 3476 return setSignedRange(ZExt, 3477 ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 3478 } 3479 3480 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 3481 ConstantRange X = getSignedRange(SExt->getOperand()); 3482 return setSignedRange(SExt, 3483 ConservativeResult.intersectWith(X.signExtend(BitWidth))); 3484 } 3485 3486 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 3487 ConstantRange X = getSignedRange(Trunc->getOperand()); 3488 return setSignedRange(Trunc, 3489 ConservativeResult.intersectWith(X.truncate(BitWidth))); 3490 } 3491 3492 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 3493 // If there's no signed wrap, and all the operands have the same sign or 3494 // zero, the value won't ever change sign. 3495 if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) { 3496 bool AllNonNeg = true; 3497 bool AllNonPos = true; 3498 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 3499 if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false; 3500 if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false; 3501 } 3502 if (AllNonNeg) 3503 ConservativeResult = ConservativeResult.intersectWith( 3504 ConstantRange(APInt(BitWidth, 0), 3505 APInt::getSignedMinValue(BitWidth))); 3506 else if (AllNonPos) 3507 ConservativeResult = ConservativeResult.intersectWith( 3508 ConstantRange(APInt::getSignedMinValue(BitWidth), 3509 APInt(BitWidth, 1))); 3510 } 3511 3512 // TODO: non-affine addrec 3513 if (AddRec->isAffine()) { 3514 Type *Ty = AddRec->getType(); 3515 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 3516 if (!isa<SCEVCouldNotCompute>(MaxBECount) && 3517 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 3518 MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); 3519 3520 const SCEV *Start = AddRec->getStart(); 3521 const SCEV *Step = AddRec->getStepRecurrence(*this); 3522 3523 ConstantRange StartRange = getSignedRange(Start); 3524 ConstantRange StepRange = getSignedRange(Step); 3525 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 3526 ConstantRange EndRange = 3527 StartRange.add(MaxBECountRange.multiply(StepRange)); 3528 3529 // Check for overflow. This must be done with ConstantRange arithmetic 3530 // because we could be called from within the ScalarEvolution overflow 3531 // checking code. 3532 ConstantRange ExtStartRange = StartRange.sextOrTrunc(BitWidth*2+1); 3533 ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); 3534 ConstantRange ExtMaxBECountRange = 3535 MaxBECountRange.zextOrTrunc(BitWidth*2+1); 3536 ConstantRange ExtEndRange = EndRange.sextOrTrunc(BitWidth*2+1); 3537 if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != 3538 ExtEndRange) 3539 return setSignedRange(AddRec, ConservativeResult); 3540 3541 APInt Min = APIntOps::smin(StartRange.getSignedMin(), 3542 EndRange.getSignedMin()); 3543 APInt Max = APIntOps::smax(StartRange.getSignedMax(), 3544 EndRange.getSignedMax()); 3545 if (Min.isMinSignedValue() && Max.isMaxSignedValue()) 3546 return setSignedRange(AddRec, ConservativeResult); 3547 return setSignedRange(AddRec, 3548 ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); 3549 } 3550 } 3551 3552 return setSignedRange(AddRec, ConservativeResult); 3553 } 3554 3555 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3556 // For a SCEVUnknown, ask ValueTracking. 3557 if (!U->getValue()->getType()->isIntegerTy() && !TD) 3558 return setSignedRange(U, ConservativeResult); 3559 unsigned NS = ComputeNumSignBits(U->getValue(), TD); 3560 if (NS == 1) 3561 return setSignedRange(U, ConservativeResult); 3562 return setSignedRange(U, ConservativeResult.intersectWith( 3563 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), 3564 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1)+1))); 3565 } 3566 3567 return setSignedRange(S, ConservativeResult); 3568} 3569 3570/// createSCEV - We know that there is no SCEV for the specified value. 3571/// Analyze the expression. 3572/// 3573const SCEV *ScalarEvolution::createSCEV(Value *V) { 3574 if (!isSCEVable(V->getType())) 3575 return getUnknown(V); 3576 3577 unsigned Opcode = Instruction::UserOp1; 3578 if (Instruction *I = dyn_cast<Instruction>(V)) { 3579 Opcode = I->getOpcode(); 3580 3581 // Don't attempt to analyze instructions in blocks that aren't 3582 // reachable. Such instructions don't matter, and they aren't required 3583 // to obey basic rules for definitions dominating uses which this 3584 // analysis depends on. 3585 if (!DT->isReachableFromEntry(I->getParent())) 3586 return getUnknown(V); 3587 } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) 3588 Opcode = CE->getOpcode(); 3589 else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) 3590 return getConstant(CI); 3591 else if (isa<ConstantPointerNull>(V)) 3592 return getConstant(V->getType(), 0); 3593 else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 3594 return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee()); 3595 else 3596 return getUnknown(V); 3597 3598 Operator *U = cast<Operator>(V); 3599 switch (Opcode) { 3600 case Instruction::Add: { 3601 // The simple thing to do would be to just call getSCEV on both operands 3602 // and call getAddExpr with the result. However if we're looking at a 3603 // bunch of things all added together, this can be quite inefficient, 3604 // because it leads to N-1 getAddExpr calls for N ultimate operands. 3605 // Instead, gather up all the operands and make a single getAddExpr call. 3606 // LLVM IR canonical form means we need only traverse the left operands. 3607 // 3608 // Don't apply this instruction's NSW or NUW flags to the new 3609 // expression. The instruction may be guarded by control flow that the 3610 // no-wrap behavior depends on. Non-control-equivalent instructions can be 3611 // mapped to the same SCEV expression, and it would be incorrect to transfer 3612 // NSW/NUW semantics to those operations. 3613 SmallVector<const SCEV *, 4> AddOps; 3614 AddOps.push_back(getSCEV(U->getOperand(1))); 3615 for (Value *Op = U->getOperand(0); ; Op = U->getOperand(0)) { 3616 unsigned Opcode = Op->getValueID() - Value::InstructionVal; 3617 if (Opcode != Instruction::Add && Opcode != Instruction::Sub) 3618 break; 3619 U = cast<Operator>(Op); 3620 const SCEV *Op1 = getSCEV(U->getOperand(1)); 3621 if (Opcode == Instruction::Sub) 3622 AddOps.push_back(getNegativeSCEV(Op1)); 3623 else 3624 AddOps.push_back(Op1); 3625 } 3626 AddOps.push_back(getSCEV(U->getOperand(0))); 3627 return getAddExpr(AddOps); 3628 } 3629 case Instruction::Mul: { 3630 // Don't transfer NSW/NUW for the same reason as AddExpr. 3631 SmallVector<const SCEV *, 4> MulOps; 3632 MulOps.push_back(getSCEV(U->getOperand(1))); 3633 for (Value *Op = U->getOperand(0); 3634 Op->getValueID() == Instruction::Mul + Value::InstructionVal; 3635 Op = U->getOperand(0)) { 3636 U = cast<Operator>(Op); 3637 MulOps.push_back(getSCEV(U->getOperand(1))); 3638 } 3639 MulOps.push_back(getSCEV(U->getOperand(0))); 3640 return getMulExpr(MulOps); 3641 } 3642 case Instruction::UDiv: 3643 return getUDivExpr(getSCEV(U->getOperand(0)), 3644 getSCEV(U->getOperand(1))); 3645 case Instruction::Sub: 3646 return getMinusSCEV(getSCEV(U->getOperand(0)), 3647 getSCEV(U->getOperand(1))); 3648 case Instruction::And: 3649 // For an expression like x&255 that merely masks off the high bits, 3650 // use zext(trunc(x)) as the SCEV expression. 3651 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 3652 if (CI->isNullValue()) 3653 return getSCEV(U->getOperand(1)); 3654 if (CI->isAllOnesValue()) 3655 return getSCEV(U->getOperand(0)); 3656 const APInt &A = CI->getValue(); 3657 3658 // Instcombine's ShrinkDemandedConstant may strip bits out of 3659 // constants, obscuring what would otherwise be a low-bits mask. 3660 // Use ComputeMaskedBits to compute what ShrinkDemandedConstant 3661 // knew about to reconstruct a low-bits mask value. 3662 unsigned LZ = A.countLeadingZeros(); 3663 unsigned BitWidth = A.getBitWidth(); 3664 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0); 3665 ComputeMaskedBits(U->getOperand(0), KnownZero, KnownOne, TD); 3666 3667 APInt EffectiveMask = APInt::getLowBitsSet(BitWidth, BitWidth - LZ); 3668 3669 if (LZ != 0 && !((~A & ~KnownZero) & EffectiveMask)) 3670 return 3671 getZeroExtendExpr(getTruncateExpr(getSCEV(U->getOperand(0)), 3672 IntegerType::get(getContext(), BitWidth - LZ)), 3673 U->getType()); 3674 } 3675 break; 3676 3677 case Instruction::Or: 3678 // If the RHS of the Or is a constant, we may have something like: 3679 // X*4+1 which got turned into X*4|1. Handle this as an Add so loop 3680 // optimizations will transparently handle this case. 3681 // 3682 // In order for this transformation to be safe, the LHS must be of the 3683 // form X*(2^n) and the Or constant must be less than 2^n. 3684 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 3685 const SCEV *LHS = getSCEV(U->getOperand(0)); 3686 const APInt &CIVal = CI->getValue(); 3687 if (GetMinTrailingZeros(LHS) >= 3688 (CIVal.getBitWidth() - CIVal.countLeadingZeros())) { 3689 // Build a plain add SCEV. 3690 const SCEV *S = getAddExpr(LHS, getSCEV(CI)); 3691 // If the LHS of the add was an addrec and it has no-wrap flags, 3692 // transfer the no-wrap flags, since an or won't introduce a wrap. 3693 if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) { 3694 const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS); 3695 const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags( 3696 OldAR->getNoWrapFlags()); 3697 } 3698 return S; 3699 } 3700 } 3701 break; 3702 case Instruction::Xor: 3703 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 3704 // If the RHS of the xor is a signbit, then this is just an add. 3705 // Instcombine turns add of signbit into xor as a strength reduction step. 3706 if (CI->getValue().isSignBit()) 3707 return getAddExpr(getSCEV(U->getOperand(0)), 3708 getSCEV(U->getOperand(1))); 3709 3710 // If the RHS of xor is -1, then this is a not operation. 3711 if (CI->isAllOnesValue()) 3712 return getNotSCEV(getSCEV(U->getOperand(0))); 3713 3714 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask. 3715 // This is a variant of the check for xor with -1, and it handles 3716 // the case where instcombine has trimmed non-demanded bits out 3717 // of an xor with -1. 3718 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getOperand(0))) 3719 if (ConstantInt *LCI = dyn_cast<ConstantInt>(BO->getOperand(1))) 3720 if (BO->getOpcode() == Instruction::And && 3721 LCI->getValue() == CI->getValue()) 3722 if (const SCEVZeroExtendExpr *Z = 3723 dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) { 3724 Type *UTy = U->getType(); 3725 const SCEV *Z0 = Z->getOperand(); 3726 Type *Z0Ty = Z0->getType(); 3727 unsigned Z0TySize = getTypeSizeInBits(Z0Ty); 3728 3729 // If C is a low-bits mask, the zero extend is serving to 3730 // mask off the high bits. Complement the operand and 3731 // re-apply the zext. 3732 if (APIntOps::isMask(Z0TySize, CI->getValue())) 3733 return getZeroExtendExpr(getNotSCEV(Z0), UTy); 3734 3735 // If C is a single bit, it may be in the sign-bit position 3736 // before the zero-extend. In this case, represent the xor 3737 // using an add, which is equivalent, and re-apply the zext. 3738 APInt Trunc = CI->getValue().trunc(Z0TySize); 3739 if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() && 3740 Trunc.isSignBit()) 3741 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)), 3742 UTy); 3743 } 3744 } 3745 break; 3746 3747 case Instruction::Shl: 3748 // Turn shift left of a constant amount into a multiply. 3749 if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) { 3750 uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth(); 3751 3752 // If the shift count is not less than the bitwidth, the result of 3753 // the shift is undefined. Don't try to analyze it, because the 3754 // resolution chosen here may differ from the resolution chosen in 3755 // other parts of the compiler. 3756 if (SA->getValue().uge(BitWidth)) 3757 break; 3758 3759 Constant *X = ConstantInt::get(getContext(), 3760 APInt(BitWidth, 1).shl(SA->getZExtValue())); 3761 return getMulExpr(getSCEV(U->getOperand(0)), getSCEV(X)); 3762 } 3763 break; 3764 3765 case Instruction::LShr: 3766 // Turn logical shift right of a constant into a unsigned divide. 3767 if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) { 3768 uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth(); 3769 3770 // If the shift count is not less than the bitwidth, the result of 3771 // the shift is undefined. Don't try to analyze it, because the 3772 // resolution chosen here may differ from the resolution chosen in 3773 // other parts of the compiler. 3774 if (SA->getValue().uge(BitWidth)) 3775 break; 3776 3777 Constant *X = ConstantInt::get(getContext(), 3778 APInt(BitWidth, 1).shl(SA->getZExtValue())); 3779 return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(X)); 3780 } 3781 break; 3782 3783 case Instruction::AShr: 3784 // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression. 3785 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) 3786 if (Operator *L = dyn_cast<Operator>(U->getOperand(0))) 3787 if (L->getOpcode() == Instruction::Shl && 3788 L->getOperand(1) == U->getOperand(1)) { 3789 uint64_t BitWidth = getTypeSizeInBits(U->getType()); 3790 3791 // If the shift count is not less than the bitwidth, the result of 3792 // the shift is undefined. Don't try to analyze it, because the 3793 // resolution chosen here may differ from the resolution chosen in 3794 // other parts of the compiler. 3795 if (CI->getValue().uge(BitWidth)) 3796 break; 3797 3798 uint64_t Amt = BitWidth - CI->getZExtValue(); 3799 if (Amt == BitWidth) 3800 return getSCEV(L->getOperand(0)); // shift by zero --> noop 3801 return 3802 getSignExtendExpr(getTruncateExpr(getSCEV(L->getOperand(0)), 3803 IntegerType::get(getContext(), 3804 Amt)), 3805 U->getType()); 3806 } 3807 break; 3808 3809 case Instruction::Trunc: 3810 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType()); 3811 3812 case Instruction::ZExt: 3813 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 3814 3815 case Instruction::SExt: 3816 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 3817 3818 case Instruction::BitCast: 3819 // BitCasts are no-op casts so we just eliminate the cast. 3820 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) 3821 return getSCEV(U->getOperand(0)); 3822 break; 3823 3824 // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can 3825 // lead to pointer expressions which cannot safely be expanded to GEPs, 3826 // because ScalarEvolution doesn't respect the GEP aliasing rules when 3827 // simplifying integer expressions. 3828 3829 case Instruction::GetElementPtr: 3830 return createNodeForGEP(cast<GEPOperator>(U)); 3831 3832 case Instruction::PHI: 3833 return createNodeForPHI(cast<PHINode>(U)); 3834 3835 case Instruction::Select: 3836 // This could be a smax or umax that was lowered earlier. 3837 // Try to recover it. 3838 if (ICmpInst *ICI = dyn_cast<ICmpInst>(U->getOperand(0))) { 3839 Value *LHS = ICI->getOperand(0); 3840 Value *RHS = ICI->getOperand(1); 3841 switch (ICI->getPredicate()) { 3842 case ICmpInst::ICMP_SLT: 3843 case ICmpInst::ICMP_SLE: 3844 std::swap(LHS, RHS); 3845 // fall through 3846 case ICmpInst::ICMP_SGT: 3847 case ICmpInst::ICMP_SGE: 3848 // a >s b ? a+x : b+x -> smax(a, b)+x 3849 // a >s b ? b+x : a+x -> smin(a, b)+x 3850 if (LHS->getType() == U->getType()) { 3851 const SCEV *LS = getSCEV(LHS); 3852 const SCEV *RS = getSCEV(RHS); 3853 const SCEV *LA = getSCEV(U->getOperand(1)); 3854 const SCEV *RA = getSCEV(U->getOperand(2)); 3855 const SCEV *LDiff = getMinusSCEV(LA, LS); 3856 const SCEV *RDiff = getMinusSCEV(RA, RS); 3857 if (LDiff == RDiff) 3858 return getAddExpr(getSMaxExpr(LS, RS), LDiff); 3859 LDiff = getMinusSCEV(LA, RS); 3860 RDiff = getMinusSCEV(RA, LS); 3861 if (LDiff == RDiff) 3862 return getAddExpr(getSMinExpr(LS, RS), LDiff); 3863 } 3864 break; 3865 case ICmpInst::ICMP_ULT: 3866 case ICmpInst::ICMP_ULE: 3867 std::swap(LHS, RHS); 3868 // fall through 3869 case ICmpInst::ICMP_UGT: 3870 case ICmpInst::ICMP_UGE: 3871 // a >u b ? a+x : b+x -> umax(a, b)+x 3872 // a >u b ? b+x : a+x -> umin(a, b)+x 3873 if (LHS->getType() == U->getType()) { 3874 const SCEV *LS = getSCEV(LHS); 3875 const SCEV *RS = getSCEV(RHS); 3876 const SCEV *LA = getSCEV(U->getOperand(1)); 3877 const SCEV *RA = getSCEV(U->getOperand(2)); 3878 const SCEV *LDiff = getMinusSCEV(LA, LS); 3879 const SCEV *RDiff = getMinusSCEV(RA, RS); 3880 if (LDiff == RDiff) 3881 return getAddExpr(getUMaxExpr(LS, RS), LDiff); 3882 LDiff = getMinusSCEV(LA, RS); 3883 RDiff = getMinusSCEV(RA, LS); 3884 if (LDiff == RDiff) 3885 return getAddExpr(getUMinExpr(LS, RS), LDiff); 3886 } 3887 break; 3888 case ICmpInst::ICMP_NE: 3889 // n != 0 ? n+x : 1+x -> umax(n, 1)+x 3890 if (LHS->getType() == U->getType() && 3891 isa<ConstantInt>(RHS) && 3892 cast<ConstantInt>(RHS)->isZero()) { 3893 const SCEV *One = getConstant(LHS->getType(), 1); 3894 const SCEV *LS = getSCEV(LHS); 3895 const SCEV *LA = getSCEV(U->getOperand(1)); 3896 const SCEV *RA = getSCEV(U->getOperand(2)); 3897 const SCEV *LDiff = getMinusSCEV(LA, LS); 3898 const SCEV *RDiff = getMinusSCEV(RA, One); 3899 if (LDiff == RDiff) 3900 return getAddExpr(getUMaxExpr(One, LS), LDiff); 3901 } 3902 break; 3903 case ICmpInst::ICMP_EQ: 3904 // n == 0 ? 1+x : n+x -> umax(n, 1)+x 3905 if (LHS->getType() == U->getType() && 3906 isa<ConstantInt>(RHS) && 3907 cast<ConstantInt>(RHS)->isZero()) { 3908 const SCEV *One = getConstant(LHS->getType(), 1); 3909 const SCEV *LS = getSCEV(LHS); 3910 const SCEV *LA = getSCEV(U->getOperand(1)); 3911 const SCEV *RA = getSCEV(U->getOperand(2)); 3912 const SCEV *LDiff = getMinusSCEV(LA, One); 3913 const SCEV *RDiff = getMinusSCEV(RA, LS); 3914 if (LDiff == RDiff) 3915 return getAddExpr(getUMaxExpr(One, LS), LDiff); 3916 } 3917 break; 3918 default: 3919 break; 3920 } 3921 } 3922 3923 default: // We cannot analyze this expression. 3924 break; 3925 } 3926 3927 return getUnknown(V); 3928} 3929 3930 3931 3932//===----------------------------------------------------------------------===// 3933// Iteration Count Computation Code 3934// 3935 3936/// getSmallConstantTripCount - Returns the maximum trip count of this loop as a 3937/// normal unsigned value. Returns 0 if the trip count is unknown or not 3938/// constant. Will also return 0 if the maximum trip count is very large (>= 3939/// 2^32). 3940/// 3941/// This "trip count" assumes that control exits via ExitingBlock. More 3942/// precisely, it is the number of times that control may reach ExitingBlock 3943/// before taking the branch. For loops with multiple exits, it may not be the 3944/// number times that the loop header executes because the loop may exit 3945/// prematurely via another branch. 3946unsigned ScalarEvolution:: 3947getSmallConstantTripCount(Loop *L, BasicBlock *ExitingBlock) { 3948 const SCEVConstant *ExitCount = 3949 dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock)); 3950 if (!ExitCount) 3951 return 0; 3952 3953 ConstantInt *ExitConst = ExitCount->getValue(); 3954 3955 // Guard against huge trip counts. 3956 if (ExitConst->getValue().getActiveBits() > 32) 3957 return 0; 3958 3959 // In case of integer overflow, this returns 0, which is correct. 3960 return ((unsigned)ExitConst->getZExtValue()) + 1; 3961} 3962 3963/// getSmallConstantTripMultiple - Returns the largest constant divisor of the 3964/// trip count of this loop as a normal unsigned value, if possible. This 3965/// means that the actual trip count is always a multiple of the returned 3966/// value (don't forget the trip count could very well be zero as well!). 3967/// 3968/// Returns 1 if the trip count is unknown or not guaranteed to be the 3969/// multiple of a constant (which is also the case if the trip count is simply 3970/// constant, use getSmallConstantTripCount for that case), Will also return 1 3971/// if the trip count is very large (>= 2^32). 3972/// 3973/// As explained in the comments for getSmallConstantTripCount, this assumes 3974/// that control exits the loop via ExitingBlock. 3975unsigned ScalarEvolution:: 3976getSmallConstantTripMultiple(Loop *L, BasicBlock *ExitingBlock) { 3977 const SCEV *ExitCount = getExitCount(L, ExitingBlock); 3978 if (ExitCount == getCouldNotCompute()) 3979 return 1; 3980 3981 // Get the trip count from the BE count by adding 1. 3982 const SCEV *TCMul = getAddExpr(ExitCount, 3983 getConstant(ExitCount->getType(), 1)); 3984 // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt 3985 // to factor simple cases. 3986 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul)) 3987 TCMul = Mul->getOperand(0); 3988 3989 const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul); 3990 if (!MulC) 3991 return 1; 3992 3993 ConstantInt *Result = MulC->getValue(); 3994 3995 // Guard against huge trip counts. 3996 if (!Result || Result->getValue().getActiveBits() > 32) 3997 return 1; 3998 3999 return (unsigned)Result->getZExtValue(); 4000} 4001 4002// getExitCount - Get the expression for the number of loop iterations for which 4003// this loop is guaranteed not to exit via ExitintBlock. Otherwise return 4004// SCEVCouldNotCompute. 4005const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) { 4006 return getBackedgeTakenInfo(L).getExact(ExitingBlock, this); 4007} 4008 4009/// getBackedgeTakenCount - If the specified loop has a predictable 4010/// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute 4011/// object. The backedge-taken count is the number of times the loop header 4012/// will be branched to from within the loop. This is one less than the 4013/// trip count of the loop, since it doesn't count the first iteration, 4014/// when the header is branched to from outside the loop. 4015/// 4016/// Note that it is not valid to call this method on a loop without a 4017/// loop-invariant backedge-taken count (see 4018/// hasLoopInvariantBackedgeTakenCount). 4019/// 4020const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) { 4021 return getBackedgeTakenInfo(L).getExact(this); 4022} 4023 4024/// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except 4025/// return the least SCEV value that is known never to be less than the 4026/// actual backedge taken count. 4027const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) { 4028 return getBackedgeTakenInfo(L).getMax(this); 4029} 4030 4031/// PushLoopPHIs - Push PHI nodes in the header of the given loop 4032/// onto the given Worklist. 4033static void 4034PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) { 4035 BasicBlock *Header = L->getHeader(); 4036 4037 // Push all Loop-header PHIs onto the Worklist stack. 4038 for (BasicBlock::iterator I = Header->begin(); 4039 PHINode *PN = dyn_cast<PHINode>(I); ++I) 4040 Worklist.push_back(PN); 4041} 4042 4043const ScalarEvolution::BackedgeTakenInfo & 4044ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { 4045 // Initially insert an invalid entry for this loop. If the insertion 4046 // succeeds, proceed to actually compute a backedge-taken count and 4047 // update the value. The temporary CouldNotCompute value tells SCEV 4048 // code elsewhere that it shouldn't attempt to request a new 4049 // backedge-taken count, which could result in infinite recursion. 4050 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair = 4051 BackedgeTakenCounts.insert(std::make_pair(L, BackedgeTakenInfo())); 4052 if (!Pair.second) 4053 return Pair.first->second; 4054 4055 // ComputeBackedgeTakenCount may allocate memory for its result. Inserting it 4056 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result 4057 // must be cleared in this scope. 4058 BackedgeTakenInfo Result = ComputeBackedgeTakenCount(L); 4059 4060 if (Result.getExact(this) != getCouldNotCompute()) { 4061 assert(isLoopInvariant(Result.getExact(this), L) && 4062 isLoopInvariant(Result.getMax(this), L) && 4063 "Computed backedge-taken count isn't loop invariant for loop!"); 4064 ++NumTripCountsComputed; 4065 } 4066 else if (Result.getMax(this) == getCouldNotCompute() && 4067 isa<PHINode>(L->getHeader()->begin())) { 4068 // Only count loops that have phi nodes as not being computable. 4069 ++NumTripCountsNotComputed; 4070 } 4071 4072 // Now that we know more about the trip count for this loop, forget any 4073 // existing SCEV values for PHI nodes in this loop since they are only 4074 // conservative estimates made without the benefit of trip count 4075 // information. This is similar to the code in forgetLoop, except that 4076 // it handles SCEVUnknown PHI nodes specially. 4077 if (Result.hasAnyInfo()) { 4078 SmallVector<Instruction *, 16> Worklist; 4079 PushLoopPHIs(L, Worklist); 4080 4081 SmallPtrSet<Instruction *, 8> Visited; 4082 while (!Worklist.empty()) { 4083 Instruction *I = Worklist.pop_back_val(); 4084 if (!Visited.insert(I)) continue; 4085 4086 ValueExprMapType::iterator It = 4087 ValueExprMap.find(static_cast<Value *>(I)); 4088 if (It != ValueExprMap.end()) { 4089 const SCEV *Old = It->second; 4090 4091 // SCEVUnknown for a PHI either means that it has an unrecognized 4092 // structure, or it's a PHI that's in the progress of being computed 4093 // by createNodeForPHI. In the former case, additional loop trip 4094 // count information isn't going to change anything. In the later 4095 // case, createNodeForPHI will perform the necessary updates on its 4096 // own when it gets to that point. 4097 if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) { 4098 forgetMemoizedResults(Old); 4099 ValueExprMap.erase(It); 4100 } 4101 if (PHINode *PN = dyn_cast<PHINode>(I)) 4102 ConstantEvolutionLoopExitValue.erase(PN); 4103 } 4104 4105 PushDefUseChildren(I, Worklist); 4106 } 4107 } 4108 4109 // Re-lookup the insert position, since the call to 4110 // ComputeBackedgeTakenCount above could result in a 4111 // recusive call to getBackedgeTakenInfo (on a different 4112 // loop), which would invalidate the iterator computed 4113 // earlier. 4114 return BackedgeTakenCounts.find(L)->second = Result; 4115} 4116 4117/// forgetLoop - This method should be called by the client when it has 4118/// changed a loop in a way that may effect ScalarEvolution's ability to 4119/// compute a trip count, or if the loop is deleted. 4120void ScalarEvolution::forgetLoop(const Loop *L) { 4121 // Drop any stored trip count value. 4122 DenseMap<const Loop*, BackedgeTakenInfo>::iterator BTCPos = 4123 BackedgeTakenCounts.find(L); 4124 if (BTCPos != BackedgeTakenCounts.end()) { 4125 BTCPos->second.clear(); 4126 BackedgeTakenCounts.erase(BTCPos); 4127 } 4128 4129 // Drop information about expressions based on loop-header PHIs. 4130 SmallVector<Instruction *, 16> Worklist; 4131 PushLoopPHIs(L, Worklist); 4132 4133 SmallPtrSet<Instruction *, 8> Visited; 4134 while (!Worklist.empty()) { 4135 Instruction *I = Worklist.pop_back_val(); 4136 if (!Visited.insert(I)) continue; 4137 4138 ValueExprMapType::iterator It = ValueExprMap.find(static_cast<Value *>(I)); 4139 if (It != ValueExprMap.end()) { 4140 forgetMemoizedResults(It->second); 4141 ValueExprMap.erase(It); 4142 if (PHINode *PN = dyn_cast<PHINode>(I)) 4143 ConstantEvolutionLoopExitValue.erase(PN); 4144 } 4145 4146 PushDefUseChildren(I, Worklist); 4147 } 4148 4149 // Forget all contained loops too, to avoid dangling entries in the 4150 // ValuesAtScopes map. 4151 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 4152 forgetLoop(*I); 4153} 4154 4155/// forgetValue - This method should be called by the client when it has 4156/// changed a value in a way that may effect its value, or which may 4157/// disconnect it from a def-use chain linking it to a loop. 4158void ScalarEvolution::forgetValue(Value *V) { 4159 Instruction *I = dyn_cast<Instruction>(V); 4160 if (!I) return; 4161 4162 // Drop information about expressions based on loop-header PHIs. 4163 SmallVector<Instruction *, 16> Worklist; 4164 Worklist.push_back(I); 4165 4166 SmallPtrSet<Instruction *, 8> Visited; 4167 while (!Worklist.empty()) { 4168 I = Worklist.pop_back_val(); 4169 if (!Visited.insert(I)) continue; 4170 4171 ValueExprMapType::iterator It = ValueExprMap.find(static_cast<Value *>(I)); 4172 if (It != ValueExprMap.end()) { 4173 forgetMemoizedResults(It->second); 4174 ValueExprMap.erase(It); 4175 if (PHINode *PN = dyn_cast<PHINode>(I)) 4176 ConstantEvolutionLoopExitValue.erase(PN); 4177 } 4178 4179 PushDefUseChildren(I, Worklist); 4180 } 4181} 4182 4183/// getExact - Get the exact loop backedge taken count considering all loop 4184/// exits. A computable result can only be return for loops with a single exit. 4185/// Returning the minimum taken count among all exits is incorrect because one 4186/// of the loop's exit limit's may have been skipped. HowFarToZero assumes that 4187/// the limit of each loop test is never skipped. This is a valid assumption as 4188/// long as the loop exits via that test. For precise results, it is the 4189/// caller's responsibility to specify the relevant loop exit using 4190/// getExact(ExitingBlock, SE). 4191const SCEV * 4192ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const { 4193 // If any exits were not computable, the loop is not computable. 4194 if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute(); 4195 4196 // We need exactly one computable exit. 4197 if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute(); 4198 assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info"); 4199 4200 const SCEV *BECount = 0; 4201 for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4202 ENT != 0; ENT = ENT->getNextExit()) { 4203 4204 assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV"); 4205 4206 if (!BECount) 4207 BECount = ENT->ExactNotTaken; 4208 else if (BECount != ENT->ExactNotTaken) 4209 return SE->getCouldNotCompute(); 4210 } 4211 assert(BECount && "Invalid not taken count for loop exit"); 4212 return BECount; 4213} 4214 4215/// getExact - Get the exact not taken count for this loop exit. 4216const SCEV * 4217ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock, 4218 ScalarEvolution *SE) const { 4219 for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4220 ENT != 0; ENT = ENT->getNextExit()) { 4221 4222 if (ENT->ExitingBlock == ExitingBlock) 4223 return ENT->ExactNotTaken; 4224 } 4225 return SE->getCouldNotCompute(); 4226} 4227 4228/// getMax - Get the max backedge taken count for the loop. 4229const SCEV * 4230ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const { 4231 return Max ? Max : SE->getCouldNotCompute(); 4232} 4233 4234/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each 4235/// computable exit into a persistent ExitNotTakenInfo array. 4236ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( 4237 SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts, 4238 bool Complete, const SCEV *MaxCount) : Max(MaxCount) { 4239 4240 if (!Complete) 4241 ExitNotTaken.setIncomplete(); 4242 4243 unsigned NumExits = ExitCounts.size(); 4244 if (NumExits == 0) return; 4245 4246 ExitNotTaken.ExitingBlock = ExitCounts[0].first; 4247 ExitNotTaken.ExactNotTaken = ExitCounts[0].second; 4248 if (NumExits == 1) return; 4249 4250 // Handle the rare case of multiple computable exits. 4251 ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1]; 4252 4253 ExitNotTakenInfo *PrevENT = &ExitNotTaken; 4254 for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) { 4255 PrevENT->setNextExit(ENT); 4256 ENT->ExitingBlock = ExitCounts[i].first; 4257 ENT->ExactNotTaken = ExitCounts[i].second; 4258 } 4259} 4260 4261/// clear - Invalidate this result and free the ExitNotTakenInfo array. 4262void ScalarEvolution::BackedgeTakenInfo::clear() { 4263 ExitNotTaken.ExitingBlock = 0; 4264 ExitNotTaken.ExactNotTaken = 0; 4265 delete[] ExitNotTaken.getNextExit(); 4266} 4267 4268/// ComputeBackedgeTakenCount - Compute the number of times the backedge 4269/// of the specified loop will execute. 4270ScalarEvolution::BackedgeTakenInfo 4271ScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) { 4272 SmallVector<BasicBlock *, 8> ExitingBlocks; 4273 L->getExitingBlocks(ExitingBlocks); 4274 4275 // Examine all exits and pick the most conservative values. 4276 const SCEV *MaxBECount = getCouldNotCompute(); 4277 bool CouldComputeBECount = true; 4278 SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts; 4279 for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 4280 ExitLimit EL = ComputeExitLimit(L, ExitingBlocks[i]); 4281 if (EL.Exact == getCouldNotCompute()) 4282 // We couldn't compute an exact value for this exit, so 4283 // we won't be able to compute an exact value for the loop. 4284 CouldComputeBECount = false; 4285 else 4286 ExitCounts.push_back(std::make_pair(ExitingBlocks[i], EL.Exact)); 4287 4288 if (MaxBECount == getCouldNotCompute()) 4289 MaxBECount = EL.Max; 4290 else if (EL.Max != getCouldNotCompute()) { 4291 // We cannot take the "min" MaxBECount, because non-unit stride loops may 4292 // skip some loop tests. Taking the max over the exits is sufficiently 4293 // conservative. TODO: We could do better taking into consideration 4294 // that (1) the loop has unit stride (2) the last loop test is 4295 // less-than/greater-than (3) any loop test is less-than/greater-than AND 4296 // falls-through some constant times less then the other tests. 4297 MaxBECount = getUMaxFromMismatchedTypes(MaxBECount, EL.Max); 4298 } 4299 } 4300 4301 return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount); 4302} 4303 4304/// ComputeExitLimit - Compute the number of times the backedge of the specified 4305/// loop will execute if it exits via the specified block. 4306ScalarEvolution::ExitLimit 4307ScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) { 4308 4309 // Okay, we've chosen an exiting block. See what condition causes us to 4310 // exit at this block. 4311 // 4312 // FIXME: we should be able to handle switch instructions (with a single exit) 4313 BranchInst *ExitBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); 4314 if (ExitBr == 0) return getCouldNotCompute(); 4315 assert(ExitBr->isConditional() && "If unconditional, it can't be in loop!"); 4316 4317 // At this point, we know we have a conditional branch that determines whether 4318 // the loop is exited. However, we don't know if the branch is executed each 4319 // time through the loop. If not, then the execution count of the branch will 4320 // not be equal to the trip count of the loop. 4321 // 4322 // Currently we check for this by checking to see if the Exit branch goes to 4323 // the loop header. If so, we know it will always execute the same number of 4324 // times as the loop. We also handle the case where the exit block *is* the 4325 // loop header. This is common for un-rotated loops. 4326 // 4327 // If both of those tests fail, walk up the unique predecessor chain to the 4328 // header, stopping if there is an edge that doesn't exit the loop. If the 4329 // header is reached, the execution count of the branch will be equal to the 4330 // trip count of the loop. 4331 // 4332 // More extensive analysis could be done to handle more cases here. 4333 // 4334 if (ExitBr->getSuccessor(0) != L->getHeader() && 4335 ExitBr->getSuccessor(1) != L->getHeader() && 4336 ExitBr->getParent() != L->getHeader()) { 4337 // The simple checks failed, try climbing the unique predecessor chain 4338 // up to the header. 4339 bool Ok = false; 4340 for (BasicBlock *BB = ExitBr->getParent(); BB; ) { 4341 BasicBlock *Pred = BB->getUniquePredecessor(); 4342 if (!Pred) 4343 return getCouldNotCompute(); 4344 TerminatorInst *PredTerm = Pred->getTerminator(); 4345 for (unsigned i = 0, e = PredTerm->getNumSuccessors(); i != e; ++i) { 4346 BasicBlock *PredSucc = PredTerm->getSuccessor(i); 4347 if (PredSucc == BB) 4348 continue; 4349 // If the predecessor has a successor that isn't BB and isn't 4350 // outside the loop, assume the worst. 4351 if (L->contains(PredSucc)) 4352 return getCouldNotCompute(); 4353 } 4354 if (Pred == L->getHeader()) { 4355 Ok = true; 4356 break; 4357 } 4358 BB = Pred; 4359 } 4360 if (!Ok) 4361 return getCouldNotCompute(); 4362 } 4363 4364 // Proceed to the next level to examine the exit condition expression. 4365 return ComputeExitLimitFromCond(L, ExitBr->getCondition(), 4366 ExitBr->getSuccessor(0), 4367 ExitBr->getSuccessor(1)); 4368} 4369 4370/// ComputeExitLimitFromCond - Compute the number of times the 4371/// backedge of the specified loop will execute if its exit condition 4372/// were a conditional branch of ExitCond, TBB, and FBB. 4373ScalarEvolution::ExitLimit 4374ScalarEvolution::ComputeExitLimitFromCond(const Loop *L, 4375 Value *ExitCond, 4376 BasicBlock *TBB, 4377 BasicBlock *FBB) { 4378 // Check if the controlling expression for this loop is an And or Or. 4379 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) { 4380 if (BO->getOpcode() == Instruction::And) { 4381 // Recurse on the operands of the and. 4382 ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB); 4383 ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB); 4384 const SCEV *BECount = getCouldNotCompute(); 4385 const SCEV *MaxBECount = getCouldNotCompute(); 4386 if (L->contains(TBB)) { 4387 // Both conditions must be true for the loop to continue executing. 4388 // Choose the less conservative count. 4389 if (EL0.Exact == getCouldNotCompute() || 4390 EL1.Exact == getCouldNotCompute()) 4391 BECount = getCouldNotCompute(); 4392 else 4393 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 4394 if (EL0.Max == getCouldNotCompute()) 4395 MaxBECount = EL1.Max; 4396 else if (EL1.Max == getCouldNotCompute()) 4397 MaxBECount = EL0.Max; 4398 else 4399 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 4400 } else { 4401 // Both conditions must be true at the same time for the loop to exit. 4402 // For now, be conservative. 4403 assert(L->contains(FBB) && "Loop block has no successor in loop!"); 4404 if (EL0.Max == EL1.Max) 4405 MaxBECount = EL0.Max; 4406 if (EL0.Exact == EL1.Exact) 4407 BECount = EL0.Exact; 4408 } 4409 4410 return ExitLimit(BECount, MaxBECount); 4411 } 4412 if (BO->getOpcode() == Instruction::Or) { 4413 // Recurse on the operands of the or. 4414 ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB); 4415 ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB); 4416 const SCEV *BECount = getCouldNotCompute(); 4417 const SCEV *MaxBECount = getCouldNotCompute(); 4418 if (L->contains(FBB)) { 4419 // Both conditions must be false for the loop to continue executing. 4420 // Choose the less conservative count. 4421 if (EL0.Exact == getCouldNotCompute() || 4422 EL1.Exact == getCouldNotCompute()) 4423 BECount = getCouldNotCompute(); 4424 else 4425 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 4426 if (EL0.Max == getCouldNotCompute()) 4427 MaxBECount = EL1.Max; 4428 else if (EL1.Max == getCouldNotCompute()) 4429 MaxBECount = EL0.Max; 4430 else 4431 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 4432 } else { 4433 // Both conditions must be false at the same time for the loop to exit. 4434 // For now, be conservative. 4435 assert(L->contains(TBB) && "Loop block has no successor in loop!"); 4436 if (EL0.Max == EL1.Max) 4437 MaxBECount = EL0.Max; 4438 if (EL0.Exact == EL1.Exact) 4439 BECount = EL0.Exact; 4440 } 4441 4442 return ExitLimit(BECount, MaxBECount); 4443 } 4444 } 4445 4446 // With an icmp, it may be feasible to compute an exact backedge-taken count. 4447 // Proceed to the next level to examine the icmp. 4448 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) 4449 return ComputeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB); 4450 4451 // Check for a constant condition. These are normally stripped out by 4452 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to 4453 // preserve the CFG and is temporarily leaving constant conditions 4454 // in place. 4455 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) { 4456 if (L->contains(FBB) == !CI->getZExtValue()) 4457 // The backedge is always taken. 4458 return getCouldNotCompute(); 4459 else 4460 // The backedge is never taken. 4461 return getConstant(CI->getType(), 0); 4462 } 4463 4464 // If it's not an integer or pointer comparison then compute it the hard way. 4465 return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 4466} 4467 4468/// ComputeExitLimitFromICmp - Compute the number of times the 4469/// backedge of the specified loop will execute if its exit condition 4470/// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB. 4471ScalarEvolution::ExitLimit 4472ScalarEvolution::ComputeExitLimitFromICmp(const Loop *L, 4473 ICmpInst *ExitCond, 4474 BasicBlock *TBB, 4475 BasicBlock *FBB) { 4476 4477 // If the condition was exit on true, convert the condition to exit on false 4478 ICmpInst::Predicate Cond; 4479 if (!L->contains(FBB)) 4480 Cond = ExitCond->getPredicate(); 4481 else 4482 Cond = ExitCond->getInversePredicate(); 4483 4484 // Handle common loops like: for (X = "string"; *X; ++X) 4485 if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0))) 4486 if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) { 4487 ExitLimit ItCnt = 4488 ComputeLoadConstantCompareExitLimit(LI, RHS, L, Cond); 4489 if (ItCnt.hasAnyInfo()) 4490 return ItCnt; 4491 } 4492 4493 const SCEV *LHS = getSCEV(ExitCond->getOperand(0)); 4494 const SCEV *RHS = getSCEV(ExitCond->getOperand(1)); 4495 4496 // Try to evaluate any dependencies out of the loop. 4497 LHS = getSCEVAtScope(LHS, L); 4498 RHS = getSCEVAtScope(RHS, L); 4499 4500 // At this point, we would like to compute how many iterations of the 4501 // loop the predicate will return true for these inputs. 4502 if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) { 4503 // If there is a loop-invariant, force it into the RHS. 4504 std::swap(LHS, RHS); 4505 Cond = ICmpInst::getSwappedPredicate(Cond); 4506 } 4507 4508 // Simplify the operands before analyzing them. 4509 (void)SimplifyICmpOperands(Cond, LHS, RHS); 4510 4511 // If we have a comparison of a chrec against a constant, try to use value 4512 // ranges to answer this query. 4513 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) 4514 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS)) 4515 if (AddRec->getLoop() == L) { 4516 // Form the constant range. 4517 ConstantRange CompRange( 4518 ICmpInst::makeConstantRange(Cond, RHSC->getValue()->getValue())); 4519 4520 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this); 4521 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret; 4522 } 4523 4524 switch (Cond) { 4525 case ICmpInst::ICMP_NE: { // while (X != Y) 4526 // Convert to: while (X-Y != 0) 4527 ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L); 4528 if (EL.hasAnyInfo()) return EL; 4529 break; 4530 } 4531 case ICmpInst::ICMP_EQ: { // while (X == Y) 4532 // Convert to: while (X-Y == 0) 4533 ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L); 4534 if (EL.hasAnyInfo()) return EL; 4535 break; 4536 } 4537 case ICmpInst::ICMP_SLT: { 4538 ExitLimit EL = HowManyLessThans(LHS, RHS, L, true); 4539 if (EL.hasAnyInfo()) return EL; 4540 break; 4541 } 4542 case ICmpInst::ICMP_SGT: { 4543 ExitLimit EL = HowManyLessThans(getNotSCEV(LHS), 4544 getNotSCEV(RHS), L, true); 4545 if (EL.hasAnyInfo()) return EL; 4546 break; 4547 } 4548 case ICmpInst::ICMP_ULT: { 4549 ExitLimit EL = HowManyLessThans(LHS, RHS, L, false); 4550 if (EL.hasAnyInfo()) return EL; 4551 break; 4552 } 4553 case ICmpInst::ICMP_UGT: { 4554 ExitLimit EL = HowManyLessThans(getNotSCEV(LHS), 4555 getNotSCEV(RHS), L, false); 4556 if (EL.hasAnyInfo()) return EL; 4557 break; 4558 } 4559 default: 4560#if 0 4561 dbgs() << "ComputeBackedgeTakenCount "; 4562 if (ExitCond->getOperand(0)->getType()->isUnsigned()) 4563 dbgs() << "[unsigned] "; 4564 dbgs() << *LHS << " " 4565 << Instruction::getOpcodeName(Instruction::ICmp) 4566 << " " << *RHS << "\n"; 4567#endif 4568 break; 4569 } 4570 return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 4571} 4572 4573static ConstantInt * 4574EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, 4575 ScalarEvolution &SE) { 4576 const SCEV *InVal = SE.getConstant(C); 4577 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE); 4578 assert(isa<SCEVConstant>(Val) && 4579 "Evaluation of SCEV at constant didn't fold correctly?"); 4580 return cast<SCEVConstant>(Val)->getValue(); 4581} 4582 4583/// ComputeLoadConstantCompareExitLimit - Given an exit condition of 4584/// 'icmp op load X, cst', try to see if we can compute the backedge 4585/// execution count. 4586ScalarEvolution::ExitLimit 4587ScalarEvolution::ComputeLoadConstantCompareExitLimit( 4588 LoadInst *LI, 4589 Constant *RHS, 4590 const Loop *L, 4591 ICmpInst::Predicate predicate) { 4592 4593 if (LI->isVolatile()) return getCouldNotCompute(); 4594 4595 // Check to see if the loaded pointer is a getelementptr of a global. 4596 // TODO: Use SCEV instead of manually grubbing with GEPs. 4597 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)); 4598 if (!GEP) return getCouldNotCompute(); 4599 4600 // Make sure that it is really a constant global we are gepping, with an 4601 // initializer, and make sure the first IDX is really 0. 4602 GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)); 4603 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || 4604 GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) || 4605 !cast<Constant>(GEP->getOperand(1))->isNullValue()) 4606 return getCouldNotCompute(); 4607 4608 // Okay, we allow one non-constant index into the GEP instruction. 4609 Value *VarIdx = 0; 4610 std::vector<Constant*> Indexes; 4611 unsigned VarIdxNum = 0; 4612 for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i) 4613 if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { 4614 Indexes.push_back(CI); 4615 } else if (!isa<ConstantInt>(GEP->getOperand(i))) { 4616 if (VarIdx) return getCouldNotCompute(); // Multiple non-constant idx's. 4617 VarIdx = GEP->getOperand(i); 4618 VarIdxNum = i-2; 4619 Indexes.push_back(0); 4620 } 4621 4622 // Loop-invariant loads may be a byproduct of loop optimization. Skip them. 4623 if (!VarIdx) 4624 return getCouldNotCompute(); 4625 4626 // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant. 4627 // Check to see if X is a loop variant variable value now. 4628 const SCEV *Idx = getSCEV(VarIdx); 4629 Idx = getSCEVAtScope(Idx, L); 4630 4631 // We can only recognize very limited forms of loop index expressions, in 4632 // particular, only affine AddRec's like {C1,+,C2}. 4633 const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx); 4634 if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) || 4635 !isa<SCEVConstant>(IdxExpr->getOperand(0)) || 4636 !isa<SCEVConstant>(IdxExpr->getOperand(1))) 4637 return getCouldNotCompute(); 4638 4639 unsigned MaxSteps = MaxBruteForceIterations; 4640 for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) { 4641 ConstantInt *ItCst = ConstantInt::get( 4642 cast<IntegerType>(IdxExpr->getType()), IterationNum); 4643 ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this); 4644 4645 // Form the GEP offset. 4646 Indexes[VarIdxNum] = Val; 4647 4648 Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(), 4649 Indexes); 4650 if (Result == 0) break; // Cannot compute! 4651 4652 // Evaluate the condition for this iteration. 4653 Result = ConstantExpr::getICmp(predicate, Result, RHS); 4654 if (!isa<ConstantInt>(Result)) break; // Couldn't decide for sure 4655 if (cast<ConstantInt>(Result)->getValue().isMinValue()) { 4656#if 0 4657 dbgs() << "\n***\n*** Computed loop count " << *ItCst 4658 << "\n*** From global " << *GV << "*** BB: " << *L->getHeader() 4659 << "***\n"; 4660#endif 4661 ++NumArrayLenItCounts; 4662 return getConstant(ItCst); // Found terminating iteration! 4663 } 4664 } 4665 return getCouldNotCompute(); 4666} 4667 4668 4669/// CanConstantFold - Return true if we can constant fold an instruction of the 4670/// specified type, assuming that all operands were constants. 4671static bool CanConstantFold(const Instruction *I) { 4672 if (isa<BinaryOperator>(I) || isa<CmpInst>(I) || 4673 isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) || 4674 isa<LoadInst>(I)) 4675 return true; 4676 4677 if (const CallInst *CI = dyn_cast<CallInst>(I)) 4678 if (const Function *F = CI->getCalledFunction()) 4679 return canConstantFoldCallTo(F); 4680 return false; 4681} 4682 4683/// Determine whether this instruction can constant evolve within this loop 4684/// assuming its operands can all constant evolve. 4685static bool canConstantEvolve(Instruction *I, const Loop *L) { 4686 // An instruction outside of the loop can't be derived from a loop PHI. 4687 if (!L->contains(I)) return false; 4688 4689 if (isa<PHINode>(I)) { 4690 if (L->getHeader() == I->getParent()) 4691 return true; 4692 else 4693 // We don't currently keep track of the control flow needed to evaluate 4694 // PHIs, so we cannot handle PHIs inside of loops. 4695 return false; 4696 } 4697 4698 // If we won't be able to constant fold this expression even if the operands 4699 // are constants, bail early. 4700 return CanConstantFold(I); 4701} 4702 4703/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by 4704/// recursing through each instruction operand until reaching a loop header phi. 4705static PHINode * 4706getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, 4707 DenseMap<Instruction *, PHINode *> &PHIMap) { 4708 4709 // Otherwise, we can evaluate this instruction if all of its operands are 4710 // constant or derived from a PHI node themselves. 4711 PHINode *PHI = 0; 4712 for (Instruction::op_iterator OpI = UseInst->op_begin(), 4713 OpE = UseInst->op_end(); OpI != OpE; ++OpI) { 4714 4715 if (isa<Constant>(*OpI)) continue; 4716 4717 Instruction *OpInst = dyn_cast<Instruction>(*OpI); 4718 if (!OpInst || !canConstantEvolve(OpInst, L)) return 0; 4719 4720 PHINode *P = dyn_cast<PHINode>(OpInst); 4721 if (!P) 4722 // If this operand is already visited, reuse the prior result. 4723 // We may have P != PHI if this is the deepest point at which the 4724 // inconsistent paths meet. 4725 P = PHIMap.lookup(OpInst); 4726 if (!P) { 4727 // Recurse and memoize the results, whether a phi is found or not. 4728 // This recursive call invalidates pointers into PHIMap. 4729 P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap); 4730 PHIMap[OpInst] = P; 4731 } 4732 if (P == 0) return 0; // Not evolving from PHI 4733 if (PHI && PHI != P) return 0; // Evolving from multiple different PHIs. 4734 PHI = P; 4735 } 4736 // This is a expression evolving from a constant PHI! 4737 return PHI; 4738} 4739 4740/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node 4741/// in the loop that V is derived from. We allow arbitrary operations along the 4742/// way, but the operands of an operation must either be constants or a value 4743/// derived from a constant PHI. If this expression does not fit with these 4744/// constraints, return null. 4745static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) { 4746 Instruction *I = dyn_cast<Instruction>(V); 4747 if (I == 0 || !canConstantEvolve(I, L)) return 0; 4748 4749 if (PHINode *PN = dyn_cast<PHINode>(I)) { 4750 return PN; 4751 } 4752 4753 // Record non-constant instructions contained by the loop. 4754 DenseMap<Instruction *, PHINode *> PHIMap; 4755 return getConstantEvolvingPHIOperands(I, L, PHIMap); 4756} 4757 4758/// EvaluateExpression - Given an expression that passes the 4759/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node 4760/// in the loop has the value PHIVal. If we can't fold this expression for some 4761/// reason, return null. 4762static Constant *EvaluateExpression(Value *V, const Loop *L, 4763 DenseMap<Instruction *, Constant *> &Vals, 4764 const TargetData *TD, 4765 const TargetLibraryInfo *TLI) { 4766 // Convenient constant check, but redundant for recursive calls. 4767 if (Constant *C = dyn_cast<Constant>(V)) return C; 4768 Instruction *I = dyn_cast<Instruction>(V); 4769 if (!I) return 0; 4770 4771 if (Constant *C = Vals.lookup(I)) return C; 4772 4773 // An instruction inside the loop depends on a value outside the loop that we 4774 // weren't given a mapping for, or a value such as a call inside the loop. 4775 if (!canConstantEvolve(I, L)) return 0; 4776 4777 // An unmapped PHI can be due to a branch or another loop inside this loop, 4778 // or due to this not being the initial iteration through a loop where we 4779 // couldn't compute the evolution of this particular PHI last time. 4780 if (isa<PHINode>(I)) return 0; 4781 4782 std::vector<Constant*> Operands(I->getNumOperands()); 4783 4784 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 4785 Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i)); 4786 if (!Operand) { 4787 Operands[i] = dyn_cast<Constant>(I->getOperand(i)); 4788 if (!Operands[i]) return 0; 4789 continue; 4790 } 4791 Constant *C = EvaluateExpression(Operand, L, Vals, TD, TLI); 4792 Vals[Operand] = C; 4793 if (!C) return 0; 4794 Operands[i] = C; 4795 } 4796 4797 if (CmpInst *CI = dyn_cast<CmpInst>(I)) 4798 return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], 4799 Operands[1], TD, TLI); 4800 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 4801 if (!LI->isVolatile()) 4802 return ConstantFoldLoadFromConstPtr(Operands[0], TD); 4803 } 4804 return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, TD, 4805 TLI); 4806} 4807 4808/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is 4809/// in the header of its containing loop, we know the loop executes a 4810/// constant number of times, and the PHI node is just a recurrence 4811/// involving constants, fold it. 4812Constant * 4813ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN, 4814 const APInt &BEs, 4815 const Loop *L) { 4816 DenseMap<PHINode*, Constant*>::const_iterator I = 4817 ConstantEvolutionLoopExitValue.find(PN); 4818 if (I != ConstantEvolutionLoopExitValue.end()) 4819 return I->second; 4820 4821 if (BEs.ugt(MaxBruteForceIterations)) 4822 return ConstantEvolutionLoopExitValue[PN] = 0; // Not going to evaluate it. 4823 4824 Constant *&RetVal = ConstantEvolutionLoopExitValue[PN]; 4825 4826 DenseMap<Instruction *, Constant *> CurrentIterVals; 4827 BasicBlock *Header = L->getHeader(); 4828 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 4829 4830 // Since the loop is canonicalized, the PHI node must have two entries. One 4831 // entry must be a constant (coming in from outside of the loop), and the 4832 // second must be derived from the same PHI. 4833 bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1)); 4834 PHINode *PHI = 0; 4835 for (BasicBlock::iterator I = Header->begin(); 4836 (PHI = dyn_cast<PHINode>(I)); ++I) { 4837 Constant *StartCST = 4838 dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge)); 4839 if (StartCST == 0) continue; 4840 CurrentIterVals[PHI] = StartCST; 4841 } 4842 if (!CurrentIterVals.count(PN)) 4843 return RetVal = 0; 4844 4845 Value *BEValue = PN->getIncomingValue(SecondIsBackedge); 4846 4847 // Execute the loop symbolically to determine the exit value. 4848 if (BEs.getActiveBits() >= 32) 4849 return RetVal = 0; // More than 2^32-1 iterations?? Not doing it! 4850 4851 unsigned NumIterations = BEs.getZExtValue(); // must be in range 4852 unsigned IterationNum = 0; 4853 for (; ; ++IterationNum) { 4854 if (IterationNum == NumIterations) 4855 return RetVal = CurrentIterVals[PN]; // Got exit value! 4856 4857 // Compute the value of the PHIs for the next iteration. 4858 // EvaluateExpression adds non-phi values to the CurrentIterVals map. 4859 DenseMap<Instruction *, Constant *> NextIterVals; 4860 Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, 4861 TLI); 4862 if (NextPHI == 0) 4863 return 0; // Couldn't evaluate! 4864 NextIterVals[PN] = NextPHI; 4865 4866 bool StoppedEvolving = NextPHI == CurrentIterVals[PN]; 4867 4868 // Also evaluate the other PHI nodes. However, we don't get to stop if we 4869 // cease to be able to evaluate one of them or if they stop evolving, 4870 // because that doesn't necessarily prevent us from computing PN. 4871 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute; 4872 for (DenseMap<Instruction *, Constant *>::const_iterator 4873 I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){ 4874 PHINode *PHI = dyn_cast<PHINode>(I->first); 4875 if (!PHI || PHI == PN || PHI->getParent() != Header) continue; 4876 PHIsToCompute.push_back(std::make_pair(PHI, I->second)); 4877 } 4878 // We use two distinct loops because EvaluateExpression may invalidate any 4879 // iterators into CurrentIterVals. 4880 for (SmallVectorImpl<std::pair<PHINode *, Constant*> >::const_iterator 4881 I = PHIsToCompute.begin(), E = PHIsToCompute.end(); I != E; ++I) { 4882 PHINode *PHI = I->first; 4883 Constant *&NextPHI = NextIterVals[PHI]; 4884 if (!NextPHI) { // Not already computed. 4885 Value *BEValue = PHI->getIncomingValue(SecondIsBackedge); 4886 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, TLI); 4887 } 4888 if (NextPHI != I->second) 4889 StoppedEvolving = false; 4890 } 4891 4892 // If all entries in CurrentIterVals == NextIterVals then we can stop 4893 // iterating, the loop can't continue to change. 4894 if (StoppedEvolving) 4895 return RetVal = CurrentIterVals[PN]; 4896 4897 CurrentIterVals.swap(NextIterVals); 4898 } 4899} 4900 4901/// ComputeExitCountExhaustively - If the loop is known to execute a 4902/// constant number of times (the condition evolves only from constants), 4903/// try to evaluate a few iterations of the loop until we get the exit 4904/// condition gets a value of ExitWhen (true or false). If we cannot 4905/// evaluate the trip count of the loop, return getCouldNotCompute(). 4906const SCEV *ScalarEvolution::ComputeExitCountExhaustively(const Loop *L, 4907 Value *Cond, 4908 bool ExitWhen) { 4909 PHINode *PN = getConstantEvolvingPHI(Cond, L); 4910 if (PN == 0) return getCouldNotCompute(); 4911 4912 // If the loop is canonicalized, the PHI will have exactly two entries. 4913 // That's the only form we support here. 4914 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute(); 4915 4916 DenseMap<Instruction *, Constant *> CurrentIterVals; 4917 BasicBlock *Header = L->getHeader(); 4918 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 4919 4920 // One entry must be a constant (coming in from outside of the loop), and the 4921 // second must be derived from the same PHI. 4922 bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1)); 4923 PHINode *PHI = 0; 4924 for (BasicBlock::iterator I = Header->begin(); 4925 (PHI = dyn_cast<PHINode>(I)); ++I) { 4926 Constant *StartCST = 4927 dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge)); 4928 if (StartCST == 0) continue; 4929 CurrentIterVals[PHI] = StartCST; 4930 } 4931 if (!CurrentIterVals.count(PN)) 4932 return getCouldNotCompute(); 4933 4934 // Okay, we find a PHI node that defines the trip count of this loop. Execute 4935 // the loop symbolically to determine when the condition gets a value of 4936 // "ExitWhen". 4937 4938 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis. 4939 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){ 4940 ConstantInt *CondVal = 4941 dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, L, CurrentIterVals, 4942 TD, TLI)); 4943 4944 // Couldn't symbolically evaluate. 4945 if (!CondVal) return getCouldNotCompute(); 4946 4947 if (CondVal->getValue() == uint64_t(ExitWhen)) { 4948 ++NumBruteForceTripCountsComputed; 4949 return getConstant(Type::getInt32Ty(getContext()), IterationNum); 4950 } 4951 4952 // Update all the PHI nodes for the next iteration. 4953 DenseMap<Instruction *, Constant *> NextIterVals; 4954 4955 // Create a list of which PHIs we need to compute. We want to do this before 4956 // calling EvaluateExpression on them because that may invalidate iterators 4957 // into CurrentIterVals. 4958 SmallVector<PHINode *, 8> PHIsToCompute; 4959 for (DenseMap<Instruction *, Constant *>::const_iterator 4960 I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){ 4961 PHINode *PHI = dyn_cast<PHINode>(I->first); 4962 if (!PHI || PHI->getParent() != Header) continue; 4963 PHIsToCompute.push_back(PHI); 4964 } 4965 for (SmallVectorImpl<PHINode *>::const_iterator I = PHIsToCompute.begin(), 4966 E = PHIsToCompute.end(); I != E; ++I) { 4967 PHINode *PHI = *I; 4968 Constant *&NextPHI = NextIterVals[PHI]; 4969 if (NextPHI) continue; // Already computed! 4970 4971 Value *BEValue = PHI->getIncomingValue(SecondIsBackedge); 4972 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, TLI); 4973 } 4974 CurrentIterVals.swap(NextIterVals); 4975 } 4976 4977 // Too many iterations were needed to evaluate. 4978 return getCouldNotCompute(); 4979} 4980 4981/// getSCEVAtScope - Return a SCEV expression for the specified value 4982/// at the specified scope in the program. The L value specifies a loop 4983/// nest to evaluate the expression at, where null is the top-level or a 4984/// specified loop is immediately inside of the loop. 4985/// 4986/// This method can be used to compute the exit value for a variable defined 4987/// in a loop by querying what the value will hold in the parent loop. 4988/// 4989/// In the case that a relevant loop exit value cannot be computed, the 4990/// original value V is returned. 4991const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) { 4992 // Check to see if we've folded this expression at this loop before. 4993 std::map<const Loop *, const SCEV *> &Values = ValuesAtScopes[V]; 4994 std::pair<std::map<const Loop *, const SCEV *>::iterator, bool> Pair = 4995 Values.insert(std::make_pair(L, static_cast<const SCEV *>(0))); 4996 if (!Pair.second) 4997 return Pair.first->second ? Pair.first->second : V; 4998 4999 // Otherwise compute it. 5000 const SCEV *C = computeSCEVAtScope(V, L); 5001 ValuesAtScopes[V][L] = C; 5002 return C; 5003} 5004 5005/// This builds up a Constant using the ConstantExpr interface. That way, we 5006/// will return Constants for objects which aren't represented by a 5007/// SCEVConstant, because SCEVConstant is restricted to ConstantInt. 5008/// Returns NULL if the SCEV isn't representable as a Constant. 5009static Constant *BuildConstantFromSCEV(const SCEV *V) { 5010 switch (V->getSCEVType()) { 5011 default: // TODO: smax, umax. 5012 case scCouldNotCompute: 5013 case scAddRecExpr: 5014 break; 5015 case scConstant: 5016 return cast<SCEVConstant>(V)->getValue(); 5017 case scUnknown: 5018 return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue()); 5019 case scSignExtend: { 5020 const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V); 5021 if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand())) 5022 return ConstantExpr::getSExt(CastOp, SS->getType()); 5023 break; 5024 } 5025 case scZeroExtend: { 5026 const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V); 5027 if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand())) 5028 return ConstantExpr::getZExt(CastOp, SZ->getType()); 5029 break; 5030 } 5031 case scTruncate: { 5032 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V); 5033 if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand())) 5034 return ConstantExpr::getTrunc(CastOp, ST->getType()); 5035 break; 5036 } 5037 case scAddExpr: { 5038 const SCEVAddExpr *SA = cast<SCEVAddExpr>(V); 5039 if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) { 5040 if (C->getType()->isPointerTy()) 5041 C = ConstantExpr::getBitCast(C, Type::getInt8PtrTy(C->getContext())); 5042 for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) { 5043 Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i)); 5044 if (!C2) return 0; 5045 5046 // First pointer! 5047 if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) { 5048 std::swap(C, C2); 5049 // The offsets have been converted to bytes. We can add bytes to an 5050 // i8* by GEP with the byte count in the first index. 5051 C = ConstantExpr::getBitCast(C,Type::getInt8PtrTy(C->getContext())); 5052 } 5053 5054 // Don't bother trying to sum two pointers. We probably can't 5055 // statically compute a load that results from it anyway. 5056 if (C2->getType()->isPointerTy()) 5057 return 0; 5058 5059 if (C->getType()->isPointerTy()) { 5060 if (cast<PointerType>(C->getType())->getElementType()->isStructTy()) 5061 C2 = ConstantExpr::getIntegerCast( 5062 C2, Type::getInt32Ty(C->getContext()), true); 5063 C = ConstantExpr::getGetElementPtr(C, C2); 5064 } else 5065 C = ConstantExpr::getAdd(C, C2); 5066 } 5067 return C; 5068 } 5069 break; 5070 } 5071 case scMulExpr: { 5072 const SCEVMulExpr *SM = cast<SCEVMulExpr>(V); 5073 if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) { 5074 // Don't bother with pointers at all. 5075 if (C->getType()->isPointerTy()) return 0; 5076 for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) { 5077 Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i)); 5078 if (!C2 || C2->getType()->isPointerTy()) return 0; 5079 C = ConstantExpr::getMul(C, C2); 5080 } 5081 return C; 5082 } 5083 break; 5084 } 5085 case scUDivExpr: { 5086 const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V); 5087 if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS())) 5088 if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS())) 5089 if (LHS->getType() == RHS->getType()) 5090 return ConstantExpr::getUDiv(LHS, RHS); 5091 break; 5092 } 5093 } 5094 return 0; 5095} 5096 5097const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { 5098 if (isa<SCEVConstant>(V)) return V; 5099 5100 // If this instruction is evolved from a constant-evolving PHI, compute the 5101 // exit value from the loop without using SCEVs. 5102 if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) { 5103 if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) { 5104 const Loop *LI = (*this->LI)[I->getParent()]; 5105 if (LI && LI->getParentLoop() == L) // Looking for loop exit value. 5106 if (PHINode *PN = dyn_cast<PHINode>(I)) 5107 if (PN->getParent() == LI->getHeader()) { 5108 // Okay, there is no closed form solution for the PHI node. Check 5109 // to see if the loop that contains it has a known backedge-taken 5110 // count. If so, we may be able to force computation of the exit 5111 // value. 5112 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI); 5113 if (const SCEVConstant *BTCC = 5114 dyn_cast<SCEVConstant>(BackedgeTakenCount)) { 5115 // Okay, we know how many times the containing loop executes. If 5116 // this is a constant evolving PHI node, get the final value at 5117 // the specified iteration number. 5118 Constant *RV = getConstantEvolutionLoopExitValue(PN, 5119 BTCC->getValue()->getValue(), 5120 LI); 5121 if (RV) return getSCEV(RV); 5122 } 5123 } 5124 5125 // Okay, this is an expression that we cannot symbolically evaluate 5126 // into a SCEV. Check to see if it's possible to symbolically evaluate 5127 // the arguments into constants, and if so, try to constant propagate the 5128 // result. This is particularly useful for computing loop exit values. 5129 if (CanConstantFold(I)) { 5130 SmallVector<Constant *, 4> Operands; 5131 bool MadeImprovement = false; 5132 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 5133 Value *Op = I->getOperand(i); 5134 if (Constant *C = dyn_cast<Constant>(Op)) { 5135 Operands.push_back(C); 5136 continue; 5137 } 5138 5139 // If any of the operands is non-constant and if they are 5140 // non-integer and non-pointer, don't even try to analyze them 5141 // with scev techniques. 5142 if (!isSCEVable(Op->getType())) 5143 return V; 5144 5145 const SCEV *OrigV = getSCEV(Op); 5146 const SCEV *OpV = getSCEVAtScope(OrigV, L); 5147 MadeImprovement |= OrigV != OpV; 5148 5149 Constant *C = BuildConstantFromSCEV(OpV); 5150 if (!C) return V; 5151 if (C->getType() != Op->getType()) 5152 C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 5153 Op->getType(), 5154 false), 5155 C, Op->getType()); 5156 Operands.push_back(C); 5157 } 5158 5159 // Check to see if getSCEVAtScope actually made an improvement. 5160 if (MadeImprovement) { 5161 Constant *C = 0; 5162 if (const CmpInst *CI = dyn_cast<CmpInst>(I)) 5163 C = ConstantFoldCompareInstOperands(CI->getPredicate(), 5164 Operands[0], Operands[1], TD, 5165 TLI); 5166 else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) { 5167 if (!LI->isVolatile()) 5168 C = ConstantFoldLoadFromConstPtr(Operands[0], TD); 5169 } else 5170 C = ConstantFoldInstOperands(I->getOpcode(), I->getType(), 5171 Operands, TD, TLI); 5172 if (!C) return V; 5173 return getSCEV(C); 5174 } 5175 } 5176 } 5177 5178 // This is some other type of SCEVUnknown, just return it. 5179 return V; 5180 } 5181 5182 if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) { 5183 // Avoid performing the look-up in the common case where the specified 5184 // expression has no loop-variant portions. 5185 for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) { 5186 const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 5187 if (OpAtScope != Comm->getOperand(i)) { 5188 // Okay, at least one of these operands is loop variant but might be 5189 // foldable. Build a new instance of the folded commutative expression. 5190 SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(), 5191 Comm->op_begin()+i); 5192 NewOps.push_back(OpAtScope); 5193 5194 for (++i; i != e; ++i) { 5195 OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 5196 NewOps.push_back(OpAtScope); 5197 } 5198 if (isa<SCEVAddExpr>(Comm)) 5199 return getAddExpr(NewOps); 5200 if (isa<SCEVMulExpr>(Comm)) 5201 return getMulExpr(NewOps); 5202 if (isa<SCEVSMaxExpr>(Comm)) 5203 return getSMaxExpr(NewOps); 5204 if (isa<SCEVUMaxExpr>(Comm)) 5205 return getUMaxExpr(NewOps); 5206 llvm_unreachable("Unknown commutative SCEV type!"); 5207 } 5208 } 5209 // If we got here, all operands are loop invariant. 5210 return Comm; 5211 } 5212 5213 if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) { 5214 const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L); 5215 const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L); 5216 if (LHS == Div->getLHS() && RHS == Div->getRHS()) 5217 return Div; // must be loop invariant 5218 return getUDivExpr(LHS, RHS); 5219 } 5220 5221 // If this is a loop recurrence for a loop that does not contain L, then we 5222 // are dealing with the final value computed by the loop. 5223 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) { 5224 // First, attempt to evaluate each operand. 5225 // Avoid performing the look-up in the common case where the specified 5226 // expression has no loop-variant portions. 5227 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 5228 const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L); 5229 if (OpAtScope == AddRec->getOperand(i)) 5230 continue; 5231 5232 // Okay, at least one of these operands is loop variant but might be 5233 // foldable. Build a new instance of the folded commutative expression. 5234 SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(), 5235 AddRec->op_begin()+i); 5236 NewOps.push_back(OpAtScope); 5237 for (++i; i != e; ++i) 5238 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L)); 5239 5240 const SCEV *FoldedRec = 5241 getAddRecExpr(NewOps, AddRec->getLoop(), 5242 AddRec->getNoWrapFlags(SCEV::FlagNW)); 5243 AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec); 5244 // The addrec may be folded to a nonrecurrence, for example, if the 5245 // induction variable is multiplied by zero after constant folding. Go 5246 // ahead and return the folded value. 5247 if (!AddRec) 5248 return FoldedRec; 5249 break; 5250 } 5251 5252 // If the scope is outside the addrec's loop, evaluate it by using the 5253 // loop exit value of the addrec. 5254 if (!AddRec->getLoop()->contains(L)) { 5255 // To evaluate this recurrence, we need to know how many times the AddRec 5256 // loop iterates. Compute this now. 5257 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop()); 5258 if (BackedgeTakenCount == getCouldNotCompute()) return AddRec; 5259 5260 // Then, evaluate the AddRec. 5261 return AddRec->evaluateAtIteration(BackedgeTakenCount, *this); 5262 } 5263 5264 return AddRec; 5265 } 5266 5267 if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) { 5268 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5269 if (Op == Cast->getOperand()) 5270 return Cast; // must be loop invariant 5271 return getZeroExtendExpr(Op, Cast->getType()); 5272 } 5273 5274 if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) { 5275 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5276 if (Op == Cast->getOperand()) 5277 return Cast; // must be loop invariant 5278 return getSignExtendExpr(Op, Cast->getType()); 5279 } 5280 5281 if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) { 5282 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5283 if (Op == Cast->getOperand()) 5284 return Cast; // must be loop invariant 5285 return getTruncateExpr(Op, Cast->getType()); 5286 } 5287 5288 llvm_unreachable("Unknown SCEV type!"); 5289} 5290 5291/// getSCEVAtScope - This is a convenience function which does 5292/// getSCEVAtScope(getSCEV(V), L). 5293const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) { 5294 return getSCEVAtScope(getSCEV(V), L); 5295} 5296 5297/// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the 5298/// following equation: 5299/// 5300/// A * X = B (mod N) 5301/// 5302/// where N = 2^BW and BW is the common bit width of A and B. The signedness of 5303/// A and B isn't important. 5304/// 5305/// If the equation does not have a solution, SCEVCouldNotCompute is returned. 5306static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B, 5307 ScalarEvolution &SE) { 5308 uint32_t BW = A.getBitWidth(); 5309 assert(BW == B.getBitWidth() && "Bit widths must be the same."); 5310 assert(A != 0 && "A must be non-zero."); 5311 5312 // 1. D = gcd(A, N) 5313 // 5314 // The gcd of A and N may have only one prime factor: 2. The number of 5315 // trailing zeros in A is its multiplicity 5316 uint32_t Mult2 = A.countTrailingZeros(); 5317 // D = 2^Mult2 5318 5319 // 2. Check if B is divisible by D. 5320 // 5321 // B is divisible by D if and only if the multiplicity of prime factor 2 for B 5322 // is not less than multiplicity of this prime factor for D. 5323 if (B.countTrailingZeros() < Mult2) 5324 return SE.getCouldNotCompute(); 5325 5326 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic 5327 // modulo (N / D). 5328 // 5329 // (N / D) may need BW+1 bits in its representation. Hence, we'll use this 5330 // bit width during computations. 5331 APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D 5332 APInt Mod(BW + 1, 0); 5333 Mod.setBit(BW - Mult2); // Mod = N / D 5334 APInt I = AD.multiplicativeInverse(Mod); 5335 5336 // 4. Compute the minimum unsigned root of the equation: 5337 // I * (B / D) mod (N / D) 5338 APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod); 5339 5340 // The result is guaranteed to be less than 2^BW so we may truncate it to BW 5341 // bits. 5342 return SE.getConstant(Result.trunc(BW)); 5343} 5344 5345/// SolveQuadraticEquation - Find the roots of the quadratic equation for the 5346/// given quadratic chrec {L,+,M,+,N}. This returns either the two roots (which 5347/// might be the same) or two SCEVCouldNotCompute objects. 5348/// 5349static std::pair<const SCEV *,const SCEV *> 5350SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) { 5351 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!"); 5352 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0)); 5353 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1)); 5354 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2)); 5355 5356 // We currently can only solve this if the coefficients are constants. 5357 if (!LC || !MC || !NC) { 5358 const SCEV *CNC = SE.getCouldNotCompute(); 5359 return std::make_pair(CNC, CNC); 5360 } 5361 5362 uint32_t BitWidth = LC->getValue()->getValue().getBitWidth(); 5363 const APInt &L = LC->getValue()->getValue(); 5364 const APInt &M = MC->getValue()->getValue(); 5365 const APInt &N = NC->getValue()->getValue(); 5366 APInt Two(BitWidth, 2); 5367 APInt Four(BitWidth, 4); 5368 5369 { 5370 using namespace APIntOps; 5371 const APInt& C = L; 5372 // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C 5373 // The B coefficient is M-N/2 5374 APInt B(M); 5375 B -= sdiv(N,Two); 5376 5377 // The A coefficient is N/2 5378 APInt A(N.sdiv(Two)); 5379 5380 // Compute the B^2-4ac term. 5381 APInt SqrtTerm(B); 5382 SqrtTerm *= B; 5383 SqrtTerm -= Four * (A * C); 5384 5385 // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest 5386 // integer value or else APInt::sqrt() will assert. 5387 APInt SqrtVal(SqrtTerm.sqrt()); 5388 5389 // Compute the two solutions for the quadratic formula. 5390 // The divisions must be performed as signed divisions. 5391 APInt NegB(-B); 5392 APInt TwoA(A << 1); 5393 if (TwoA.isMinValue()) { 5394 const SCEV *CNC = SE.getCouldNotCompute(); 5395 return std::make_pair(CNC, CNC); 5396 } 5397 5398 LLVMContext &Context = SE.getContext(); 5399 5400 ConstantInt *Solution1 = 5401 ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA)); 5402 ConstantInt *Solution2 = 5403 ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA)); 5404 5405 return std::make_pair(SE.getConstant(Solution1), 5406 SE.getConstant(Solution2)); 5407 } // end APIntOps namespace 5408} 5409 5410/// HowFarToZero - Return the number of times a backedge comparing the specified 5411/// value to zero will execute. If not computable, return CouldNotCompute. 5412/// 5413/// This is only used for loops with a "x != y" exit test. The exit condition is 5414/// now expressed as a single expression, V = x-y. So the exit test is 5415/// effectively V != 0. We know and take advantage of the fact that this 5416/// expression only being used in a comparison by zero context. 5417ScalarEvolution::ExitLimit 5418ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) { 5419 // If the value is a constant 5420 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 5421 // If the value is already zero, the branch will execute zero times. 5422 if (C->getValue()->isZero()) return C; 5423 return getCouldNotCompute(); // Otherwise it will loop infinitely. 5424 } 5425 5426 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V); 5427 if (!AddRec || AddRec->getLoop() != L) 5428 return getCouldNotCompute(); 5429 5430 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of 5431 // the quadratic equation to solve it. 5432 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) { 5433 std::pair<const SCEV *,const SCEV *> Roots = 5434 SolveQuadraticEquation(AddRec, *this); 5435 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 5436 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 5437 if (R1 && R2) { 5438#if 0 5439 dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1 5440 << " sol#2: " << *R2 << "\n"; 5441#endif 5442 // Pick the smallest positive root value. 5443 if (ConstantInt *CB = 5444 dyn_cast<ConstantInt>(ConstantExpr::getICmp(CmpInst::ICMP_ULT, 5445 R1->getValue(), 5446 R2->getValue()))) { 5447 if (CB->getZExtValue() == false) 5448 std::swap(R1, R2); // R1 is the minimum root now. 5449 5450 // We can only use this value if the chrec ends up with an exact zero 5451 // value at this index. When solving for "X*X != 5", for example, we 5452 // should not accept a root of 2. 5453 const SCEV *Val = AddRec->evaluateAtIteration(R1, *this); 5454 if (Val->isZero()) 5455 return R1; // We found a quadratic root! 5456 } 5457 } 5458 return getCouldNotCompute(); 5459 } 5460 5461 // Otherwise we can only handle this if it is affine. 5462 if (!AddRec->isAffine()) 5463 return getCouldNotCompute(); 5464 5465 // If this is an affine expression, the execution count of this branch is 5466 // the minimum unsigned root of the following equation: 5467 // 5468 // Start + Step*N = 0 (mod 2^BW) 5469 // 5470 // equivalent to: 5471 // 5472 // Step*N = -Start (mod 2^BW) 5473 // 5474 // where BW is the common bit width of Start and Step. 5475 5476 // Get the initial value for the loop. 5477 const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop()); 5478 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop()); 5479 5480 // For now we handle only constant steps. 5481 // 5482 // TODO: Handle a nonconstant Step given AddRec<NUW>. If the 5483 // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap 5484 // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step. 5485 // We have not yet seen any such cases. 5486 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step); 5487 if (StepC == 0 || StepC->getValue()->equalsInt(0)) 5488 return getCouldNotCompute(); 5489 5490 // For positive steps (counting up until unsigned overflow): 5491 // N = -Start/Step (as unsigned) 5492 // For negative steps (counting down to zero): 5493 // N = Start/-Step 5494 // First compute the unsigned distance from zero in the direction of Step. 5495 bool CountDown = StepC->getValue()->getValue().isNegative(); 5496 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start); 5497 5498 // Handle unitary steps, which cannot wraparound. 5499 // 1*N = -Start; -1*N = Start (mod 2^BW), so: 5500 // N = Distance (as unsigned) 5501 if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) { 5502 ConstantRange CR = getUnsignedRange(Start); 5503 const SCEV *MaxBECount; 5504 if (!CountDown && CR.getUnsignedMin().isMinValue()) 5505 // When counting up, the worst starting value is 1, not 0. 5506 MaxBECount = CR.getUnsignedMax().isMinValue() 5507 ? getConstant(APInt::getMinValue(CR.getBitWidth())) 5508 : getConstant(APInt::getMaxValue(CR.getBitWidth())); 5509 else 5510 MaxBECount = getConstant(CountDown ? CR.getUnsignedMax() 5511 : -CR.getUnsignedMin()); 5512 return ExitLimit(Distance, MaxBECount); 5513 } 5514 5515 // If the recurrence is known not to wraparound, unsigned divide computes the 5516 // back edge count. We know that the value will either become zero (and thus 5517 // the loop terminates), that the loop will terminate through some other exit 5518 // condition first, or that the loop has undefined behavior. This means 5519 // we can't "miss" the exit value, even with nonunit stride. 5520 // 5521 // FIXME: Prove that loops always exhibits *acceptable* undefined 5522 // behavior. Loops must exhibit defined behavior until a wrapped value is 5523 // actually used. So the trip count computed by udiv could be smaller than the 5524 // number of well-defined iterations. 5525 if (AddRec->getNoWrapFlags(SCEV::FlagNW)) { 5526 // FIXME: We really want an "isexact" bit for udiv. 5527 return getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step); 5528 } 5529 // Then, try to solve the above equation provided that Start is constant. 5530 if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start)) 5531 return SolveLinEquationWithOverflow(StepC->getValue()->getValue(), 5532 -StartC->getValue()->getValue(), 5533 *this); 5534 return getCouldNotCompute(); 5535} 5536 5537/// HowFarToNonZero - Return the number of times a backedge checking the 5538/// specified value for nonzero will execute. If not computable, return 5539/// CouldNotCompute 5540ScalarEvolution::ExitLimit 5541ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) { 5542 // Loops that look like: while (X == 0) are very strange indeed. We don't 5543 // handle them yet except for the trivial case. This could be expanded in the 5544 // future as needed. 5545 5546 // If the value is a constant, check to see if it is known to be non-zero 5547 // already. If so, the backedge will execute zero times. 5548 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 5549 if (!C->getValue()->isNullValue()) 5550 return getConstant(C->getType(), 0); 5551 return getCouldNotCompute(); // Otherwise it will loop infinitely. 5552 } 5553 5554 // We could implement others, but I really doubt anyone writes loops like 5555 // this, and if they did, they would already be constant folded. 5556 return getCouldNotCompute(); 5557} 5558 5559/// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB 5560/// (which may not be an immediate predecessor) which has exactly one 5561/// successor from which BB is reachable, or null if no such block is 5562/// found. 5563/// 5564std::pair<BasicBlock *, BasicBlock *> 5565ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) { 5566 // If the block has a unique predecessor, then there is no path from the 5567 // predecessor to the block that does not go through the direct edge 5568 // from the predecessor to the block. 5569 if (BasicBlock *Pred = BB->getSinglePredecessor()) 5570 return std::make_pair(Pred, BB); 5571 5572 // A loop's header is defined to be a block that dominates the loop. 5573 // If the header has a unique predecessor outside the loop, it must be 5574 // a block that has exactly one successor that can reach the loop. 5575 if (Loop *L = LI->getLoopFor(BB)) 5576 return std::make_pair(L->getLoopPredecessor(), L->getHeader()); 5577 5578 return std::pair<BasicBlock *, BasicBlock *>(); 5579} 5580 5581/// HasSameValue - SCEV structural equivalence is usually sufficient for 5582/// testing whether two expressions are equal, however for the purposes of 5583/// looking for a condition guarding a loop, it can be useful to be a little 5584/// more general, since a front-end may have replicated the controlling 5585/// expression. 5586/// 5587static bool HasSameValue(const SCEV *A, const SCEV *B) { 5588 // Quick check to see if they are the same SCEV. 5589 if (A == B) return true; 5590 5591 // Otherwise, if they're both SCEVUnknown, it's possible that they hold 5592 // two different instructions with the same value. Check for this case. 5593 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A)) 5594 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B)) 5595 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue())) 5596 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue())) 5597 if (AI->isIdenticalTo(BI) && !AI->mayReadFromMemory()) 5598 return true; 5599 5600 // Otherwise assume they may have a different value. 5601 return false; 5602} 5603 5604/// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with 5605/// predicate Pred. Return true iff any changes were made. 5606/// 5607bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, 5608 const SCEV *&LHS, const SCEV *&RHS, 5609 unsigned Depth) { 5610 bool Changed = false; 5611 5612 // If we hit the max recursion limit bail out. 5613 if (Depth >= 3) 5614 return false; 5615 5616 // Canonicalize a constant to the right side. 5617 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 5618 // Check for both operands constant. 5619 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 5620 if (ConstantExpr::getICmp(Pred, 5621 LHSC->getValue(), 5622 RHSC->getValue())->isNullValue()) 5623 goto trivially_false; 5624 else 5625 goto trivially_true; 5626 } 5627 // Otherwise swap the operands to put the constant on the right. 5628 std::swap(LHS, RHS); 5629 Pred = ICmpInst::getSwappedPredicate(Pred); 5630 Changed = true; 5631 } 5632 5633 // If we're comparing an addrec with a value which is loop-invariant in the 5634 // addrec's loop, put the addrec on the left. Also make a dominance check, 5635 // as both operands could be addrecs loop-invariant in each other's loop. 5636 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) { 5637 const Loop *L = AR->getLoop(); 5638 if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) { 5639 std::swap(LHS, RHS); 5640 Pred = ICmpInst::getSwappedPredicate(Pred); 5641 Changed = true; 5642 } 5643 } 5644 5645 // If there's a constant operand, canonicalize comparisons with boundary 5646 // cases, and canonicalize *-or-equal comparisons to regular comparisons. 5647 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) { 5648 const APInt &RA = RC->getValue()->getValue(); 5649 switch (Pred) { 5650 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 5651 case ICmpInst::ICMP_EQ: 5652 case ICmpInst::ICMP_NE: 5653 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b. 5654 if (!RA) 5655 if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS)) 5656 if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0))) 5657 if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 && 5658 ME->getOperand(0)->isAllOnesValue()) { 5659 RHS = AE->getOperand(1); 5660 LHS = ME->getOperand(1); 5661 Changed = true; 5662 } 5663 break; 5664 case ICmpInst::ICMP_UGE: 5665 if ((RA - 1).isMinValue()) { 5666 Pred = ICmpInst::ICMP_NE; 5667 RHS = getConstant(RA - 1); 5668 Changed = true; 5669 break; 5670 } 5671 if (RA.isMaxValue()) { 5672 Pred = ICmpInst::ICMP_EQ; 5673 Changed = true; 5674 break; 5675 } 5676 if (RA.isMinValue()) goto trivially_true; 5677 5678 Pred = ICmpInst::ICMP_UGT; 5679 RHS = getConstant(RA - 1); 5680 Changed = true; 5681 break; 5682 case ICmpInst::ICMP_ULE: 5683 if ((RA + 1).isMaxValue()) { 5684 Pred = ICmpInst::ICMP_NE; 5685 RHS = getConstant(RA + 1); 5686 Changed = true; 5687 break; 5688 } 5689 if (RA.isMinValue()) { 5690 Pred = ICmpInst::ICMP_EQ; 5691 Changed = true; 5692 break; 5693 } 5694 if (RA.isMaxValue()) goto trivially_true; 5695 5696 Pred = ICmpInst::ICMP_ULT; 5697 RHS = getConstant(RA + 1); 5698 Changed = true; 5699 break; 5700 case ICmpInst::ICMP_SGE: 5701 if ((RA - 1).isMinSignedValue()) { 5702 Pred = ICmpInst::ICMP_NE; 5703 RHS = getConstant(RA - 1); 5704 Changed = true; 5705 break; 5706 } 5707 if (RA.isMaxSignedValue()) { 5708 Pred = ICmpInst::ICMP_EQ; 5709 Changed = true; 5710 break; 5711 } 5712 if (RA.isMinSignedValue()) goto trivially_true; 5713 5714 Pred = ICmpInst::ICMP_SGT; 5715 RHS = getConstant(RA - 1); 5716 Changed = true; 5717 break; 5718 case ICmpInst::ICMP_SLE: 5719 if ((RA + 1).isMaxSignedValue()) { 5720 Pred = ICmpInst::ICMP_NE; 5721 RHS = getConstant(RA + 1); 5722 Changed = true; 5723 break; 5724 } 5725 if (RA.isMinSignedValue()) { 5726 Pred = ICmpInst::ICMP_EQ; 5727 Changed = true; 5728 break; 5729 } 5730 if (RA.isMaxSignedValue()) goto trivially_true; 5731 5732 Pred = ICmpInst::ICMP_SLT; 5733 RHS = getConstant(RA + 1); 5734 Changed = true; 5735 break; 5736 case ICmpInst::ICMP_UGT: 5737 if (RA.isMinValue()) { 5738 Pred = ICmpInst::ICMP_NE; 5739 Changed = true; 5740 break; 5741 } 5742 if ((RA + 1).isMaxValue()) { 5743 Pred = ICmpInst::ICMP_EQ; 5744 RHS = getConstant(RA + 1); 5745 Changed = true; 5746 break; 5747 } 5748 if (RA.isMaxValue()) goto trivially_false; 5749 break; 5750 case ICmpInst::ICMP_ULT: 5751 if (RA.isMaxValue()) { 5752 Pred = ICmpInst::ICMP_NE; 5753 Changed = true; 5754 break; 5755 } 5756 if ((RA - 1).isMinValue()) { 5757 Pred = ICmpInst::ICMP_EQ; 5758 RHS = getConstant(RA - 1); 5759 Changed = true; 5760 break; 5761 } 5762 if (RA.isMinValue()) goto trivially_false; 5763 break; 5764 case ICmpInst::ICMP_SGT: 5765 if (RA.isMinSignedValue()) { 5766 Pred = ICmpInst::ICMP_NE; 5767 Changed = true; 5768 break; 5769 } 5770 if ((RA + 1).isMaxSignedValue()) { 5771 Pred = ICmpInst::ICMP_EQ; 5772 RHS = getConstant(RA + 1); 5773 Changed = true; 5774 break; 5775 } 5776 if (RA.isMaxSignedValue()) goto trivially_false; 5777 break; 5778 case ICmpInst::ICMP_SLT: 5779 if (RA.isMaxSignedValue()) { 5780 Pred = ICmpInst::ICMP_NE; 5781 Changed = true; 5782 break; 5783 } 5784 if ((RA - 1).isMinSignedValue()) { 5785 Pred = ICmpInst::ICMP_EQ; 5786 RHS = getConstant(RA - 1); 5787 Changed = true; 5788 break; 5789 } 5790 if (RA.isMinSignedValue()) goto trivially_false; 5791 break; 5792 } 5793 } 5794 5795 // Check for obvious equality. 5796 if (HasSameValue(LHS, RHS)) { 5797 if (ICmpInst::isTrueWhenEqual(Pred)) 5798 goto trivially_true; 5799 if (ICmpInst::isFalseWhenEqual(Pred)) 5800 goto trivially_false; 5801 } 5802 5803 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by 5804 // adding or subtracting 1 from one of the operands. 5805 switch (Pred) { 5806 case ICmpInst::ICMP_SLE: 5807 if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) { 5808 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 5809 SCEV::FlagNSW); 5810 Pred = ICmpInst::ICMP_SLT; 5811 Changed = true; 5812 } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) { 5813 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 5814 SCEV::FlagNSW); 5815 Pred = ICmpInst::ICMP_SLT; 5816 Changed = true; 5817 } 5818 break; 5819 case ICmpInst::ICMP_SGE: 5820 if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) { 5821 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 5822 SCEV::FlagNSW); 5823 Pred = ICmpInst::ICMP_SGT; 5824 Changed = true; 5825 } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) { 5826 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 5827 SCEV::FlagNSW); 5828 Pred = ICmpInst::ICMP_SGT; 5829 Changed = true; 5830 } 5831 break; 5832 case ICmpInst::ICMP_ULE: 5833 if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) { 5834 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 5835 SCEV::FlagNUW); 5836 Pred = ICmpInst::ICMP_ULT; 5837 Changed = true; 5838 } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) { 5839 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 5840 SCEV::FlagNUW); 5841 Pred = ICmpInst::ICMP_ULT; 5842 Changed = true; 5843 } 5844 break; 5845 case ICmpInst::ICMP_UGE: 5846 if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) { 5847 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 5848 SCEV::FlagNUW); 5849 Pred = ICmpInst::ICMP_UGT; 5850 Changed = true; 5851 } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) { 5852 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 5853 SCEV::FlagNUW); 5854 Pred = ICmpInst::ICMP_UGT; 5855 Changed = true; 5856 } 5857 break; 5858 default: 5859 break; 5860 } 5861 5862 // TODO: More simplifications are possible here. 5863 5864 // Recursively simplify until we either hit a recursion limit or nothing 5865 // changes. 5866 if (Changed) 5867 return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1); 5868 5869 return Changed; 5870 5871trivially_true: 5872 // Return 0 == 0. 5873 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 5874 Pred = ICmpInst::ICMP_EQ; 5875 return true; 5876 5877trivially_false: 5878 // Return 0 != 0. 5879 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 5880 Pred = ICmpInst::ICMP_NE; 5881 return true; 5882} 5883 5884bool ScalarEvolution::isKnownNegative(const SCEV *S) { 5885 return getSignedRange(S).getSignedMax().isNegative(); 5886} 5887 5888bool ScalarEvolution::isKnownPositive(const SCEV *S) { 5889 return getSignedRange(S).getSignedMin().isStrictlyPositive(); 5890} 5891 5892bool ScalarEvolution::isKnownNonNegative(const SCEV *S) { 5893 return !getSignedRange(S).getSignedMin().isNegative(); 5894} 5895 5896bool ScalarEvolution::isKnownNonPositive(const SCEV *S) { 5897 return !getSignedRange(S).getSignedMax().isStrictlyPositive(); 5898} 5899 5900bool ScalarEvolution::isKnownNonZero(const SCEV *S) { 5901 return isKnownNegative(S) || isKnownPositive(S); 5902} 5903 5904bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred, 5905 const SCEV *LHS, const SCEV *RHS) { 5906 // Canonicalize the inputs first. 5907 (void)SimplifyICmpOperands(Pred, LHS, RHS); 5908 5909 // If LHS or RHS is an addrec, check to see if the condition is true in 5910 // every iteration of the loop. 5911 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) 5912 if (isLoopEntryGuardedByCond( 5913 AR->getLoop(), Pred, AR->getStart(), RHS) && 5914 isLoopBackedgeGuardedByCond( 5915 AR->getLoop(), Pred, AR->getPostIncExpr(*this), RHS)) 5916 return true; 5917 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) 5918 if (isLoopEntryGuardedByCond( 5919 AR->getLoop(), Pred, LHS, AR->getStart()) && 5920 isLoopBackedgeGuardedByCond( 5921 AR->getLoop(), Pred, LHS, AR->getPostIncExpr(*this))) 5922 return true; 5923 5924 // Otherwise see what can be done with known constant ranges. 5925 return isKnownPredicateWithRanges(Pred, LHS, RHS); 5926} 5927 5928bool 5929ScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred, 5930 const SCEV *LHS, const SCEV *RHS) { 5931 if (HasSameValue(LHS, RHS)) 5932 return ICmpInst::isTrueWhenEqual(Pred); 5933 5934 // This code is split out from isKnownPredicate because it is called from 5935 // within isLoopEntryGuardedByCond. 5936 switch (Pred) { 5937 default: 5938 llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 5939 case ICmpInst::ICMP_SGT: 5940 Pred = ICmpInst::ICMP_SLT; 5941 std::swap(LHS, RHS); 5942 case ICmpInst::ICMP_SLT: { 5943 ConstantRange LHSRange = getSignedRange(LHS); 5944 ConstantRange RHSRange = getSignedRange(RHS); 5945 if (LHSRange.getSignedMax().slt(RHSRange.getSignedMin())) 5946 return true; 5947 if (LHSRange.getSignedMin().sge(RHSRange.getSignedMax())) 5948 return false; 5949 break; 5950 } 5951 case ICmpInst::ICMP_SGE: 5952 Pred = ICmpInst::ICMP_SLE; 5953 std::swap(LHS, RHS); 5954 case ICmpInst::ICMP_SLE: { 5955 ConstantRange LHSRange = getSignedRange(LHS); 5956 ConstantRange RHSRange = getSignedRange(RHS); 5957 if (LHSRange.getSignedMax().sle(RHSRange.getSignedMin())) 5958 return true; 5959 if (LHSRange.getSignedMin().sgt(RHSRange.getSignedMax())) 5960 return false; 5961 break; 5962 } 5963 case ICmpInst::ICMP_UGT: 5964 Pred = ICmpInst::ICMP_ULT; 5965 std::swap(LHS, RHS); 5966 case ICmpInst::ICMP_ULT: { 5967 ConstantRange LHSRange = getUnsignedRange(LHS); 5968 ConstantRange RHSRange = getUnsignedRange(RHS); 5969 if (LHSRange.getUnsignedMax().ult(RHSRange.getUnsignedMin())) 5970 return true; 5971 if (LHSRange.getUnsignedMin().uge(RHSRange.getUnsignedMax())) 5972 return false; 5973 break; 5974 } 5975 case ICmpInst::ICMP_UGE: 5976 Pred = ICmpInst::ICMP_ULE; 5977 std::swap(LHS, RHS); 5978 case ICmpInst::ICMP_ULE: { 5979 ConstantRange LHSRange = getUnsignedRange(LHS); 5980 ConstantRange RHSRange = getUnsignedRange(RHS); 5981 if (LHSRange.getUnsignedMax().ule(RHSRange.getUnsignedMin())) 5982 return true; 5983 if (LHSRange.getUnsignedMin().ugt(RHSRange.getUnsignedMax())) 5984 return false; 5985 break; 5986 } 5987 case ICmpInst::ICMP_NE: { 5988 if (getUnsignedRange(LHS).intersectWith(getUnsignedRange(RHS)).isEmptySet()) 5989 return true; 5990 if (getSignedRange(LHS).intersectWith(getSignedRange(RHS)).isEmptySet()) 5991 return true; 5992 5993 const SCEV *Diff = getMinusSCEV(LHS, RHS); 5994 if (isKnownNonZero(Diff)) 5995 return true; 5996 break; 5997 } 5998 case ICmpInst::ICMP_EQ: 5999 // The check at the top of the function catches the case where 6000 // the values are known to be equal. 6001 break; 6002 } 6003 return false; 6004} 6005 6006/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is 6007/// protected by a conditional between LHS and RHS. This is used to 6008/// to eliminate casts. 6009bool 6010ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L, 6011 ICmpInst::Predicate Pred, 6012 const SCEV *LHS, const SCEV *RHS) { 6013 // Interpret a null as meaning no loop, where there is obviously no guard 6014 // (interprocedural conditions notwithstanding). 6015 if (!L) return true; 6016 6017 BasicBlock *Latch = L->getLoopLatch(); 6018 if (!Latch) 6019 return false; 6020 6021 BranchInst *LoopContinuePredicate = 6022 dyn_cast<BranchInst>(Latch->getTerminator()); 6023 if (!LoopContinuePredicate || 6024 LoopContinuePredicate->isUnconditional()) 6025 return false; 6026 6027 return isImpliedCond(Pred, LHS, RHS, 6028 LoopContinuePredicate->getCondition(), 6029 LoopContinuePredicate->getSuccessor(0) != L->getHeader()); 6030} 6031 6032/// isLoopEntryGuardedByCond - Test whether entry to the loop is protected 6033/// by a conditional between LHS and RHS. This is used to help avoid max 6034/// expressions in loop trip counts, and to eliminate casts. 6035bool 6036ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, 6037 ICmpInst::Predicate Pred, 6038 const SCEV *LHS, const SCEV *RHS) { 6039 // Interpret a null as meaning no loop, where there is obviously no guard 6040 // (interprocedural conditions notwithstanding). 6041 if (!L) return false; 6042 6043 // Starting at the loop predecessor, climb up the predecessor chain, as long 6044 // as there are predecessors that can be found that have unique successors 6045 // leading to the original header. 6046 for (std::pair<BasicBlock *, BasicBlock *> 6047 Pair(L->getLoopPredecessor(), L->getHeader()); 6048 Pair.first; 6049 Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { 6050 6051 BranchInst *LoopEntryPredicate = 6052 dyn_cast<BranchInst>(Pair.first->getTerminator()); 6053 if (!LoopEntryPredicate || 6054 LoopEntryPredicate->isUnconditional()) 6055 continue; 6056 6057 if (isImpliedCond(Pred, LHS, RHS, 6058 LoopEntryPredicate->getCondition(), 6059 LoopEntryPredicate->getSuccessor(0) != Pair.second)) 6060 return true; 6061 } 6062 6063 return false; 6064} 6065 6066/// RAII wrapper to prevent recursive application of isImpliedCond. 6067/// ScalarEvolution's PendingLoopPredicates set must be empty unless we are 6068/// currently evaluating isImpliedCond. 6069struct MarkPendingLoopPredicate { 6070 Value *Cond; 6071 DenseSet<Value*> &LoopPreds; 6072 bool Pending; 6073 6074 MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP) 6075 : Cond(C), LoopPreds(LP) { 6076 Pending = !LoopPreds.insert(Cond).second; 6077 } 6078 ~MarkPendingLoopPredicate() { 6079 if (!Pending) 6080 LoopPreds.erase(Cond); 6081 } 6082}; 6083 6084/// isImpliedCond - Test whether the condition described by Pred, LHS, 6085/// and RHS is true whenever the given Cond value evaluates to true. 6086bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, 6087 const SCEV *LHS, const SCEV *RHS, 6088 Value *FoundCondValue, 6089 bool Inverse) { 6090 MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates); 6091 if (Mark.Pending) 6092 return false; 6093 6094 // Recursively handle And and Or conditions. 6095 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) { 6096 if (BO->getOpcode() == Instruction::And) { 6097 if (!Inverse) 6098 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 6099 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 6100 } else if (BO->getOpcode() == Instruction::Or) { 6101 if (Inverse) 6102 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 6103 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 6104 } 6105 } 6106 6107 ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue); 6108 if (!ICI) return false; 6109 6110 // Bail if the ICmp's operands' types are wider than the needed type 6111 // before attempting to call getSCEV on them. This avoids infinite 6112 // recursion, since the analysis of widening casts can require loop 6113 // exit condition information for overflow checking, which would 6114 // lead back here. 6115 if (getTypeSizeInBits(LHS->getType()) < 6116 getTypeSizeInBits(ICI->getOperand(0)->getType())) 6117 return false; 6118 6119 // Now that we found a conditional branch that dominates the loop, check to 6120 // see if it is the comparison we are looking for. 6121 ICmpInst::Predicate FoundPred; 6122 if (Inverse) 6123 FoundPred = ICI->getInversePredicate(); 6124 else 6125 FoundPred = ICI->getPredicate(); 6126 6127 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0)); 6128 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1)); 6129 6130 // Balance the types. The case where FoundLHS' type is wider than 6131 // LHS' type is checked for above. 6132 if (getTypeSizeInBits(LHS->getType()) > 6133 getTypeSizeInBits(FoundLHS->getType())) { 6134 if (CmpInst::isSigned(Pred)) { 6135 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType()); 6136 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType()); 6137 } else { 6138 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType()); 6139 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType()); 6140 } 6141 } 6142 6143 // Canonicalize the query to match the way instcombine will have 6144 // canonicalized the comparison. 6145 if (SimplifyICmpOperands(Pred, LHS, RHS)) 6146 if (LHS == RHS) 6147 return CmpInst::isTrueWhenEqual(Pred); 6148 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS)) 6149 if (FoundLHS == FoundRHS) 6150 return CmpInst::isFalseWhenEqual(Pred); 6151 6152 // Check to see if we can make the LHS or RHS match. 6153 if (LHS == FoundRHS || RHS == FoundLHS) { 6154 if (isa<SCEVConstant>(RHS)) { 6155 std::swap(FoundLHS, FoundRHS); 6156 FoundPred = ICmpInst::getSwappedPredicate(FoundPred); 6157 } else { 6158 std::swap(LHS, RHS); 6159 Pred = ICmpInst::getSwappedPredicate(Pred); 6160 } 6161 } 6162 6163 // Check whether the found predicate is the same as the desired predicate. 6164 if (FoundPred == Pred) 6165 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS); 6166 6167 // Check whether swapping the found predicate makes it the same as the 6168 // desired predicate. 6169 if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) { 6170 if (isa<SCEVConstant>(RHS)) 6171 return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS); 6172 else 6173 return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred), 6174 RHS, LHS, FoundLHS, FoundRHS); 6175 } 6176 6177 // Check whether the actual condition is beyond sufficient. 6178 if (FoundPred == ICmpInst::ICMP_EQ) 6179 if (ICmpInst::isTrueWhenEqual(Pred)) 6180 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS)) 6181 return true; 6182 if (Pred == ICmpInst::ICMP_NE) 6183 if (!ICmpInst::isTrueWhenEqual(FoundPred)) 6184 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS)) 6185 return true; 6186 6187 // Otherwise assume the worst. 6188 return false; 6189} 6190 6191/// isImpliedCondOperands - Test whether the condition described by Pred, 6192/// LHS, and RHS is true whenever the condition described by Pred, FoundLHS, 6193/// and FoundRHS is true. 6194bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, 6195 const SCEV *LHS, const SCEV *RHS, 6196 const SCEV *FoundLHS, 6197 const SCEV *FoundRHS) { 6198 return isImpliedCondOperandsHelper(Pred, LHS, RHS, 6199 FoundLHS, FoundRHS) || 6200 // ~x < ~y --> x > y 6201 isImpliedCondOperandsHelper(Pred, LHS, RHS, 6202 getNotSCEV(FoundRHS), 6203 getNotSCEV(FoundLHS)); 6204} 6205 6206/// isImpliedCondOperandsHelper - Test whether the condition described by 6207/// Pred, LHS, and RHS is true whenever the condition described by Pred, 6208/// FoundLHS, and FoundRHS is true. 6209bool 6210ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, 6211 const SCEV *LHS, const SCEV *RHS, 6212 const SCEV *FoundLHS, 6213 const SCEV *FoundRHS) { 6214 switch (Pred) { 6215 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 6216 case ICmpInst::ICMP_EQ: 6217 case ICmpInst::ICMP_NE: 6218 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS)) 6219 return true; 6220 break; 6221 case ICmpInst::ICMP_SLT: 6222 case ICmpInst::ICMP_SLE: 6223 if (isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, LHS, FoundLHS) && 6224 isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, RHS, FoundRHS)) 6225 return true; 6226 break; 6227 case ICmpInst::ICMP_SGT: 6228 case ICmpInst::ICMP_SGE: 6229 if (isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, LHS, FoundLHS) && 6230 isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, RHS, FoundRHS)) 6231 return true; 6232 break; 6233 case ICmpInst::ICMP_ULT: 6234 case ICmpInst::ICMP_ULE: 6235 if (isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, LHS, FoundLHS) && 6236 isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, RHS, FoundRHS)) 6237 return true; 6238 break; 6239 case ICmpInst::ICMP_UGT: 6240 case ICmpInst::ICMP_UGE: 6241 if (isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, LHS, FoundLHS) && 6242 isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, RHS, FoundRHS)) 6243 return true; 6244 break; 6245 } 6246 6247 return false; 6248} 6249 6250/// getBECount - Subtract the end and start values and divide by the step, 6251/// rounding up, to get the number of times the backedge is executed. Return 6252/// CouldNotCompute if an intermediate computation overflows. 6253const SCEV *ScalarEvolution::getBECount(const SCEV *Start, 6254 const SCEV *End, 6255 const SCEV *Step, 6256 bool NoWrap) { 6257 assert(!isKnownNegative(Step) && 6258 "This code doesn't handle negative strides yet!"); 6259 6260 Type *Ty = Start->getType(); 6261 6262 // When Start == End, we have an exact BECount == 0. Short-circuit this case 6263 // here because SCEV may not be able to determine that the unsigned division 6264 // after rounding is zero. 6265 if (Start == End) 6266 return getConstant(Ty, 0); 6267 6268 const SCEV *NegOne = getConstant(Ty, (uint64_t)-1); 6269 const SCEV *Diff = getMinusSCEV(End, Start); 6270 const SCEV *RoundUp = getAddExpr(Step, NegOne); 6271 6272 // Add an adjustment to the difference between End and Start so that 6273 // the division will effectively round up. 6274 const SCEV *Add = getAddExpr(Diff, RoundUp); 6275 6276 if (!NoWrap) { 6277 // Check Add for unsigned overflow. 6278 // TODO: More sophisticated things could be done here. 6279 Type *WideTy = IntegerType::get(getContext(), 6280 getTypeSizeInBits(Ty) + 1); 6281 const SCEV *EDiff = getZeroExtendExpr(Diff, WideTy); 6282 const SCEV *ERoundUp = getZeroExtendExpr(RoundUp, WideTy); 6283 const SCEV *OperandExtendedAdd = getAddExpr(EDiff, ERoundUp); 6284 if (getZeroExtendExpr(Add, WideTy) != OperandExtendedAdd) 6285 return getCouldNotCompute(); 6286 } 6287 6288 return getUDivExpr(Add, Step); 6289} 6290 6291/// HowManyLessThans - Return the number of times a backedge containing the 6292/// specified less-than comparison will execute. If not computable, return 6293/// CouldNotCompute. 6294ScalarEvolution::ExitLimit 6295ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, 6296 const Loop *L, bool isSigned) { 6297 // Only handle: "ADDREC < LoopInvariant". 6298 if (!isLoopInvariant(RHS, L)) return getCouldNotCompute(); 6299 6300 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS); 6301 if (!AddRec || AddRec->getLoop() != L) 6302 return getCouldNotCompute(); 6303 6304 // Check to see if we have a flag which makes analysis easy. 6305 bool NoWrap = isSigned ? 6306 AddRec->getNoWrapFlags((SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNW)) : 6307 AddRec->getNoWrapFlags((SCEV::NoWrapFlags)(SCEV::FlagNUW | SCEV::FlagNW)); 6308 6309 if (AddRec->isAffine()) { 6310 unsigned BitWidth = getTypeSizeInBits(AddRec->getType()); 6311 const SCEV *Step = AddRec->getStepRecurrence(*this); 6312 6313 if (Step->isZero()) 6314 return getCouldNotCompute(); 6315 if (Step->isOne()) { 6316 // With unit stride, the iteration never steps past the limit value. 6317 } else if (isKnownPositive(Step)) { 6318 // Test whether a positive iteration can step past the limit 6319 // value and past the maximum value for its type in a single step. 6320 // Note that it's not sufficient to check NoWrap here, because even 6321 // though the value after a wrap is undefined, it's not undefined 6322 // behavior, so if wrap does occur, the loop could either terminate or 6323 // loop infinitely, but in either case, the loop is guaranteed to 6324 // iterate at least until the iteration where the wrapping occurs. 6325 const SCEV *One = getConstant(Step->getType(), 1); 6326 if (isSigned) { 6327 APInt Max = APInt::getSignedMaxValue(BitWidth); 6328 if ((Max - getSignedRange(getMinusSCEV(Step, One)).getSignedMax()) 6329 .slt(getSignedRange(RHS).getSignedMax())) 6330 return getCouldNotCompute(); 6331 } else { 6332 APInt Max = APInt::getMaxValue(BitWidth); 6333 if ((Max - getUnsignedRange(getMinusSCEV(Step, One)).getUnsignedMax()) 6334 .ult(getUnsignedRange(RHS).getUnsignedMax())) 6335 return getCouldNotCompute(); 6336 } 6337 } else 6338 // TODO: Handle negative strides here and below. 6339 return getCouldNotCompute(); 6340 6341 // We know the LHS is of the form {n,+,s} and the RHS is some loop-invariant 6342 // m. So, we count the number of iterations in which {n,+,s} < m is true. 6343 // Note that we cannot simply return max(m-n,0)/s because it's not safe to 6344 // treat m-n as signed nor unsigned due to overflow possibility. 6345 6346 // First, we get the value of the LHS in the first iteration: n 6347 const SCEV *Start = AddRec->getOperand(0); 6348 6349 // Determine the minimum constant start value. 6350 const SCEV *MinStart = getConstant(isSigned ? 6351 getSignedRange(Start).getSignedMin() : 6352 getUnsignedRange(Start).getUnsignedMin()); 6353 6354 // If we know that the condition is true in order to enter the loop, 6355 // then we know that it will run exactly (m-n)/s times. Otherwise, we 6356 // only know that it will execute (max(m,n)-n)/s times. In both cases, 6357 // the division must round up. 6358 const SCEV *End = RHS; 6359 if (!isLoopEntryGuardedByCond(L, 6360 isSigned ? ICmpInst::ICMP_SLT : 6361 ICmpInst::ICMP_ULT, 6362 getMinusSCEV(Start, Step), RHS)) 6363 End = isSigned ? getSMaxExpr(RHS, Start) 6364 : getUMaxExpr(RHS, Start); 6365 6366 // Determine the maximum constant end value. 6367 const SCEV *MaxEnd = getConstant(isSigned ? 6368 getSignedRange(End).getSignedMax() : 6369 getUnsignedRange(End).getUnsignedMax()); 6370 6371 // If MaxEnd is within a step of the maximum integer value in its type, 6372 // adjust it down to the minimum value which would produce the same effect. 6373 // This allows the subsequent ceiling division of (N+(step-1))/step to 6374 // compute the correct value. 6375 const SCEV *StepMinusOne = getMinusSCEV(Step, 6376 getConstant(Step->getType(), 1)); 6377 MaxEnd = isSigned ? 6378 getSMinExpr(MaxEnd, 6379 getMinusSCEV(getConstant(APInt::getSignedMaxValue(BitWidth)), 6380 StepMinusOne)) : 6381 getUMinExpr(MaxEnd, 6382 getMinusSCEV(getConstant(APInt::getMaxValue(BitWidth)), 6383 StepMinusOne)); 6384 6385 // Finally, we subtract these two values and divide, rounding up, to get 6386 // the number of times the backedge is executed. 6387 const SCEV *BECount = getBECount(Start, End, Step, NoWrap); 6388 6389 // The maximum backedge count is similar, except using the minimum start 6390 // value and the maximum end value. 6391 // If we already have an exact constant BECount, use it instead. 6392 const SCEV *MaxBECount = isa<SCEVConstant>(BECount) ? BECount 6393 : getBECount(MinStart, MaxEnd, Step, NoWrap); 6394 6395 // If the stride is nonconstant, and NoWrap == true, then 6396 // getBECount(MinStart, MaxEnd) may not compute. This would result in an 6397 // exact BECount and invalid MaxBECount, which should be avoided to catch 6398 // more optimization opportunities. 6399 if (isa<SCEVCouldNotCompute>(MaxBECount)) 6400 MaxBECount = BECount; 6401 6402 return ExitLimit(BECount, MaxBECount); 6403 } 6404 6405 return getCouldNotCompute(); 6406} 6407 6408/// getNumIterationsInRange - Return the number of iterations of this loop that 6409/// produce values in the specified constant range. Another way of looking at 6410/// this is that it returns the first iteration number where the value is not in 6411/// the condition, thus computing the exit count. If the iteration count can't 6412/// be computed, an instance of SCEVCouldNotCompute is returned. 6413const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range, 6414 ScalarEvolution &SE) const { 6415 if (Range.isFullSet()) // Infinite loop. 6416 return SE.getCouldNotCompute(); 6417 6418 // If the start is a non-zero constant, shift the range to simplify things. 6419 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart())) 6420 if (!SC->getValue()->isZero()) { 6421 SmallVector<const SCEV *, 4> Operands(op_begin(), op_end()); 6422 Operands[0] = SE.getConstant(SC->getType(), 0); 6423 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(), 6424 getNoWrapFlags(FlagNW)); 6425 if (const SCEVAddRecExpr *ShiftedAddRec = 6426 dyn_cast<SCEVAddRecExpr>(Shifted)) 6427 return ShiftedAddRec->getNumIterationsInRange( 6428 Range.subtract(SC->getValue()->getValue()), SE); 6429 // This is strange and shouldn't happen. 6430 return SE.getCouldNotCompute(); 6431 } 6432 6433 // The only time we can solve this is when we have all constant indices. 6434 // Otherwise, we cannot determine the overflow conditions. 6435 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6436 if (!isa<SCEVConstant>(getOperand(i))) 6437 return SE.getCouldNotCompute(); 6438 6439 6440 // Okay at this point we know that all elements of the chrec are constants and 6441 // that the start element is zero. 6442 6443 // First check to see if the range contains zero. If not, the first 6444 // iteration exits. 6445 unsigned BitWidth = SE.getTypeSizeInBits(getType()); 6446 if (!Range.contains(APInt(BitWidth, 0))) 6447 return SE.getConstant(getType(), 0); 6448 6449 if (isAffine()) { 6450 // If this is an affine expression then we have this situation: 6451 // Solve {0,+,A} in Range === Ax in Range 6452 6453 // We know that zero is in the range. If A is positive then we know that 6454 // the upper value of the range must be the first possible exit value. 6455 // If A is negative then the lower of the range is the last possible loop 6456 // value. Also note that we already checked for a full range. 6457 APInt One(BitWidth,1); 6458 APInt A = cast<SCEVConstant>(getOperand(1))->getValue()->getValue(); 6459 APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower(); 6460 6461 // The exit value should be (End+A)/A. 6462 APInt ExitVal = (End + A).udiv(A); 6463 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal); 6464 6465 // Evaluate at the exit value. If we really did fall out of the valid 6466 // range, then we computed our trip count, otherwise wrap around or other 6467 // things must have happened. 6468 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE); 6469 if (Range.contains(Val->getValue())) 6470 return SE.getCouldNotCompute(); // Something strange happened 6471 6472 // Ensure that the previous value is in the range. This is a sanity check. 6473 assert(Range.contains( 6474 EvaluateConstantChrecAtConstant(this, 6475 ConstantInt::get(SE.getContext(), ExitVal - One), SE)->getValue()) && 6476 "Linear scev computation is off in a bad way!"); 6477 return SE.getConstant(ExitValue); 6478 } else if (isQuadratic()) { 6479 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of the 6480 // quadratic equation to solve it. To do this, we must frame our problem in 6481 // terms of figuring out when zero is crossed, instead of when 6482 // Range.getUpper() is crossed. 6483 SmallVector<const SCEV *, 4> NewOps(op_begin(), op_end()); 6484 NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper())); 6485 const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop(), 6486 // getNoWrapFlags(FlagNW) 6487 FlagAnyWrap); 6488 6489 // Next, solve the constructed addrec 6490 std::pair<const SCEV *,const SCEV *> Roots = 6491 SolveQuadraticEquation(cast<SCEVAddRecExpr>(NewAddRec), SE); 6492 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 6493 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 6494 if (R1) { 6495 // Pick the smallest positive root value. 6496 if (ConstantInt *CB = 6497 dyn_cast<ConstantInt>(ConstantExpr::getICmp(ICmpInst::ICMP_ULT, 6498 R1->getValue(), R2->getValue()))) { 6499 if (CB->getZExtValue() == false) 6500 std::swap(R1, R2); // R1 is the minimum root now. 6501 6502 // Make sure the root is not off by one. The returned iteration should 6503 // not be in the range, but the previous one should be. When solving 6504 // for "X*X < 5", for example, we should not return a root of 2. 6505 ConstantInt *R1Val = EvaluateConstantChrecAtConstant(this, 6506 R1->getValue(), 6507 SE); 6508 if (Range.contains(R1Val->getValue())) { 6509 // The next iteration must be out of the range... 6510 ConstantInt *NextVal = 6511 ConstantInt::get(SE.getContext(), R1->getValue()->getValue()+1); 6512 6513 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 6514 if (!Range.contains(R1Val->getValue())) 6515 return SE.getConstant(NextVal); 6516 return SE.getCouldNotCompute(); // Something strange happened 6517 } 6518 6519 // If R1 was not in the range, then it is a good return value. Make 6520 // sure that R1-1 WAS in the range though, just in case. 6521 ConstantInt *NextVal = 6522 ConstantInt::get(SE.getContext(), R1->getValue()->getValue()-1); 6523 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 6524 if (Range.contains(R1Val->getValue())) 6525 return R1; 6526 return SE.getCouldNotCompute(); // Something strange happened 6527 } 6528 } 6529 } 6530 6531 return SE.getCouldNotCompute(); 6532} 6533 6534 6535 6536//===----------------------------------------------------------------------===// 6537// SCEVCallbackVH Class Implementation 6538//===----------------------------------------------------------------------===// 6539 6540void ScalarEvolution::SCEVCallbackVH::deleted() { 6541 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 6542 if (PHINode *PN = dyn_cast<PHINode>(getValPtr())) 6543 SE->ConstantEvolutionLoopExitValue.erase(PN); 6544 SE->ValueExprMap.erase(getValPtr()); 6545 // this now dangles! 6546} 6547 6548void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) { 6549 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 6550 6551 // Forget all the expressions associated with users of the old value, 6552 // so that future queries will recompute the expressions using the new 6553 // value. 6554 Value *Old = getValPtr(); 6555 SmallVector<User *, 16> Worklist; 6556 SmallPtrSet<User *, 8> Visited; 6557 for (Value::use_iterator UI = Old->use_begin(), UE = Old->use_end(); 6558 UI != UE; ++UI) 6559 Worklist.push_back(*UI); 6560 while (!Worklist.empty()) { 6561 User *U = Worklist.pop_back_val(); 6562 // Deleting the Old value will cause this to dangle. Postpone 6563 // that until everything else is done. 6564 if (U == Old) 6565 continue; 6566 if (!Visited.insert(U)) 6567 continue; 6568 if (PHINode *PN = dyn_cast<PHINode>(U)) 6569 SE->ConstantEvolutionLoopExitValue.erase(PN); 6570 SE->ValueExprMap.erase(U); 6571 for (Value::use_iterator UI = U->use_begin(), UE = U->use_end(); 6572 UI != UE; ++UI) 6573 Worklist.push_back(*UI); 6574 } 6575 // Delete the Old value. 6576 if (PHINode *PN = dyn_cast<PHINode>(Old)) 6577 SE->ConstantEvolutionLoopExitValue.erase(PN); 6578 SE->ValueExprMap.erase(Old); 6579 // this now dangles! 6580} 6581 6582ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se) 6583 : CallbackVH(V), SE(se) {} 6584 6585//===----------------------------------------------------------------------===// 6586// ScalarEvolution Class Implementation 6587//===----------------------------------------------------------------------===// 6588 6589ScalarEvolution::ScalarEvolution() 6590 : FunctionPass(ID), FirstUnknown(0) { 6591 initializeScalarEvolutionPass(*PassRegistry::getPassRegistry()); 6592} 6593 6594bool ScalarEvolution::runOnFunction(Function &F) { 6595 this->F = &F; 6596 LI = &getAnalysis<LoopInfo>(); 6597 TD = getAnalysisIfAvailable<TargetData>(); 6598 TLI = &getAnalysis<TargetLibraryInfo>(); 6599 DT = &getAnalysis<DominatorTree>(); 6600 return false; 6601} 6602 6603void ScalarEvolution::releaseMemory() { 6604 // Iterate through all the SCEVUnknown instances and call their 6605 // destructors, so that they release their references to their values. 6606 for (SCEVUnknown *U = FirstUnknown; U; U = U->Next) 6607 U->~SCEVUnknown(); 6608 FirstUnknown = 0; 6609 6610 ValueExprMap.clear(); 6611 6612 // Free any extra memory created for ExitNotTakenInfo in the unlikely event 6613 // that a loop had multiple computable exits. 6614 for (DenseMap<const Loop*, BackedgeTakenInfo>::iterator I = 6615 BackedgeTakenCounts.begin(), E = BackedgeTakenCounts.end(); 6616 I != E; ++I) { 6617 I->second.clear(); 6618 } 6619 6620 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage"); 6621 6622 BackedgeTakenCounts.clear(); 6623 ConstantEvolutionLoopExitValue.clear(); 6624 ValuesAtScopes.clear(); 6625 LoopDispositions.clear(); 6626 BlockDispositions.clear(); 6627 UnsignedRanges.clear(); 6628 SignedRanges.clear(); 6629 UniqueSCEVs.clear(); 6630 SCEVAllocator.Reset(); 6631} 6632 6633void ScalarEvolution::getAnalysisUsage(AnalysisUsage &AU) const { 6634 AU.setPreservesAll(); 6635 AU.addRequiredTransitive<LoopInfo>(); 6636 AU.addRequiredTransitive<DominatorTree>(); 6637 AU.addRequired<TargetLibraryInfo>(); 6638} 6639 6640bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) { 6641 return !isa<SCEVCouldNotCompute>(getBackedgeTakenCount(L)); 6642} 6643 6644static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, 6645 const Loop *L) { 6646 // Print all inner loops first 6647 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 6648 PrintLoopInfo(OS, SE, *I); 6649 6650 OS << "Loop "; 6651 WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false); 6652 OS << ": "; 6653 6654 SmallVector<BasicBlock *, 8> ExitBlocks; 6655 L->getExitBlocks(ExitBlocks); 6656 if (ExitBlocks.size() != 1) 6657 OS << "<multiple exits> "; 6658 6659 if (SE->hasLoopInvariantBackedgeTakenCount(L)) { 6660 OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L); 6661 } else { 6662 OS << "Unpredictable backedge-taken count. "; 6663 } 6664 6665 OS << "\n" 6666 "Loop "; 6667 WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false); 6668 OS << ": "; 6669 6670 if (!isa<SCEVCouldNotCompute>(SE->getMaxBackedgeTakenCount(L))) { 6671 OS << "max backedge-taken count is " << *SE->getMaxBackedgeTakenCount(L); 6672 } else { 6673 OS << "Unpredictable max backedge-taken count. "; 6674 } 6675 6676 OS << "\n"; 6677} 6678 6679void ScalarEvolution::print(raw_ostream &OS, const Module *) const { 6680 // ScalarEvolution's implementation of the print method is to print 6681 // out SCEV values of all instructions that are interesting. Doing 6682 // this potentially causes it to create new SCEV objects though, 6683 // which technically conflicts with the const qualifier. This isn't 6684 // observable from outside the class though, so casting away the 6685 // const isn't dangerous. 6686 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 6687 6688 OS << "Classifying expressions for: "; 6689 WriteAsOperand(OS, F, /*PrintType=*/false); 6690 OS << "\n"; 6691 for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) 6692 if (isSCEVable(I->getType()) && !isa<CmpInst>(*I)) { 6693 OS << *I << '\n'; 6694 OS << " --> "; 6695 const SCEV *SV = SE.getSCEV(&*I); 6696 SV->print(OS); 6697 6698 const Loop *L = LI->getLoopFor((*I).getParent()); 6699 6700 const SCEV *AtUse = SE.getSCEVAtScope(SV, L); 6701 if (AtUse != SV) { 6702 OS << " --> "; 6703 AtUse->print(OS); 6704 } 6705 6706 if (L) { 6707 OS << "\t\t" "Exits: "; 6708 const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop()); 6709 if (!SE.isLoopInvariant(ExitValue, L)) { 6710 OS << "<<Unknown>>"; 6711 } else { 6712 OS << *ExitValue; 6713 } 6714 } 6715 6716 OS << "\n"; 6717 } 6718 6719 OS << "Determining loop execution counts for: "; 6720 WriteAsOperand(OS, F, /*PrintType=*/false); 6721 OS << "\n"; 6722 for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I) 6723 PrintLoopInfo(OS, &SE, *I); 6724} 6725 6726ScalarEvolution::LoopDisposition 6727ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) { 6728 std::map<const Loop *, LoopDisposition> &Values = LoopDispositions[S]; 6729 std::pair<std::map<const Loop *, LoopDisposition>::iterator, bool> Pair = 6730 Values.insert(std::make_pair(L, LoopVariant)); 6731 if (!Pair.second) 6732 return Pair.first->second; 6733 6734 LoopDisposition D = computeLoopDisposition(S, L); 6735 return LoopDispositions[S][L] = D; 6736} 6737 6738ScalarEvolution::LoopDisposition 6739ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) { 6740 switch (S->getSCEVType()) { 6741 case scConstant: 6742 return LoopInvariant; 6743 case scTruncate: 6744 case scZeroExtend: 6745 case scSignExtend: 6746 return getLoopDisposition(cast<SCEVCastExpr>(S)->getOperand(), L); 6747 case scAddRecExpr: { 6748 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 6749 6750 // If L is the addrec's loop, it's computable. 6751 if (AR->getLoop() == L) 6752 return LoopComputable; 6753 6754 // Add recurrences are never invariant in the function-body (null loop). 6755 if (!L) 6756 return LoopVariant; 6757 6758 // This recurrence is variant w.r.t. L if L contains AR's loop. 6759 if (L->contains(AR->getLoop())) 6760 return LoopVariant; 6761 6762 // This recurrence is invariant w.r.t. L if AR's loop contains L. 6763 if (AR->getLoop()->contains(L)) 6764 return LoopInvariant; 6765 6766 // This recurrence is variant w.r.t. L if any of its operands 6767 // are variant. 6768 for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end(); 6769 I != E; ++I) 6770 if (!isLoopInvariant(*I, L)) 6771 return LoopVariant; 6772 6773 // Otherwise it's loop-invariant. 6774 return LoopInvariant; 6775 } 6776 case scAddExpr: 6777 case scMulExpr: 6778 case scUMaxExpr: 6779 case scSMaxExpr: { 6780 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 6781 bool HasVarying = false; 6782 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 6783 I != E; ++I) { 6784 LoopDisposition D = getLoopDisposition(*I, L); 6785 if (D == LoopVariant) 6786 return LoopVariant; 6787 if (D == LoopComputable) 6788 HasVarying = true; 6789 } 6790 return HasVarying ? LoopComputable : LoopInvariant; 6791 } 6792 case scUDivExpr: { 6793 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 6794 LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L); 6795 if (LD == LoopVariant) 6796 return LoopVariant; 6797 LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L); 6798 if (RD == LoopVariant) 6799 return LoopVariant; 6800 return (LD == LoopInvariant && RD == LoopInvariant) ? 6801 LoopInvariant : LoopComputable; 6802 } 6803 case scUnknown: 6804 // All non-instruction values are loop invariant. All instructions are loop 6805 // invariant if they are not contained in the specified loop. 6806 // Instructions are never considered invariant in the function body 6807 // (null loop) because they are defined within the "loop". 6808 if (Instruction *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) 6809 return (L && !L->contains(I)) ? LoopInvariant : LoopVariant; 6810 return LoopInvariant; 6811 case scCouldNotCompute: 6812 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 6813 default: llvm_unreachable("Unknown SCEV kind!"); 6814 } 6815} 6816 6817bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) { 6818 return getLoopDisposition(S, L) == LoopInvariant; 6819} 6820 6821bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) { 6822 return getLoopDisposition(S, L) == LoopComputable; 6823} 6824 6825ScalarEvolution::BlockDisposition 6826ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) { 6827 std::map<const BasicBlock *, BlockDisposition> &Values = BlockDispositions[S]; 6828 std::pair<std::map<const BasicBlock *, BlockDisposition>::iterator, bool> 6829 Pair = Values.insert(std::make_pair(BB, DoesNotDominateBlock)); 6830 if (!Pair.second) 6831 return Pair.first->second; 6832 6833 BlockDisposition D = computeBlockDisposition(S, BB); 6834 return BlockDispositions[S][BB] = D; 6835} 6836 6837ScalarEvolution::BlockDisposition 6838ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { 6839 switch (S->getSCEVType()) { 6840 case scConstant: 6841 return ProperlyDominatesBlock; 6842 case scTruncate: 6843 case scZeroExtend: 6844 case scSignExtend: 6845 return getBlockDisposition(cast<SCEVCastExpr>(S)->getOperand(), BB); 6846 case scAddRecExpr: { 6847 // This uses a "dominates" query instead of "properly dominates" query 6848 // to test for proper dominance too, because the instruction which 6849 // produces the addrec's value is a PHI, and a PHI effectively properly 6850 // dominates its entire containing block. 6851 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 6852 if (!DT->dominates(AR->getLoop()->getHeader(), BB)) 6853 return DoesNotDominateBlock; 6854 } 6855 // FALL THROUGH into SCEVNAryExpr handling. 6856 case scAddExpr: 6857 case scMulExpr: 6858 case scUMaxExpr: 6859 case scSMaxExpr: { 6860 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 6861 bool Proper = true; 6862 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 6863 I != E; ++I) { 6864 BlockDisposition D = getBlockDisposition(*I, BB); 6865 if (D == DoesNotDominateBlock) 6866 return DoesNotDominateBlock; 6867 if (D == DominatesBlock) 6868 Proper = false; 6869 } 6870 return Proper ? ProperlyDominatesBlock : DominatesBlock; 6871 } 6872 case scUDivExpr: { 6873 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 6874 const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS(); 6875 BlockDisposition LD = getBlockDisposition(LHS, BB); 6876 if (LD == DoesNotDominateBlock) 6877 return DoesNotDominateBlock; 6878 BlockDisposition RD = getBlockDisposition(RHS, BB); 6879 if (RD == DoesNotDominateBlock) 6880 return DoesNotDominateBlock; 6881 return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ? 6882 ProperlyDominatesBlock : DominatesBlock; 6883 } 6884 case scUnknown: 6885 if (Instruction *I = 6886 dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) { 6887 if (I->getParent() == BB) 6888 return DominatesBlock; 6889 if (DT->properlyDominates(I->getParent(), BB)) 6890 return ProperlyDominatesBlock; 6891 return DoesNotDominateBlock; 6892 } 6893 return ProperlyDominatesBlock; 6894 case scCouldNotCompute: 6895 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 6896 default: 6897 llvm_unreachable("Unknown SCEV kind!"); 6898 } 6899} 6900 6901bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) { 6902 return getBlockDisposition(S, BB) >= DominatesBlock; 6903} 6904 6905bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) { 6906 return getBlockDisposition(S, BB) == ProperlyDominatesBlock; 6907} 6908 6909bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { 6910 SmallVector<const SCEV *, 8> Worklist; 6911 Worklist.push_back(S); 6912 do { 6913 S = Worklist.pop_back_val(); 6914 6915 switch (S->getSCEVType()) { 6916 case scConstant: 6917 break; 6918 case scTruncate: 6919 case scZeroExtend: 6920 case scSignExtend: { 6921 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(S); 6922 const SCEV *CastOp = Cast->getOperand(); 6923 if (Op == CastOp) 6924 return true; 6925 Worklist.push_back(CastOp); 6926 break; 6927 } 6928 case scAddRecExpr: 6929 case scAddExpr: 6930 case scMulExpr: 6931 case scUMaxExpr: 6932 case scSMaxExpr: { 6933 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 6934 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 6935 I != E; ++I) { 6936 const SCEV *NAryOp = *I; 6937 if (NAryOp == Op) 6938 return true; 6939 Worklist.push_back(NAryOp); 6940 } 6941 break; 6942 } 6943 case scUDivExpr: { 6944 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 6945 const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS(); 6946 if (LHS == Op || RHS == Op) 6947 return true; 6948 Worklist.push_back(LHS); 6949 Worklist.push_back(RHS); 6950 break; 6951 } 6952 case scUnknown: 6953 break; 6954 case scCouldNotCompute: 6955 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 6956 default: 6957 llvm_unreachable("Unknown SCEV kind!"); 6958 } 6959 } while (!Worklist.empty()); 6960 6961 return false; 6962} 6963 6964void ScalarEvolution::forgetMemoizedResults(const SCEV *S) { 6965 ValuesAtScopes.erase(S); 6966 LoopDispositions.erase(S); 6967 BlockDispositions.erase(S); 6968 UnsignedRanges.erase(S); 6969 SignedRanges.erase(S); 6970} 6971