136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===- CodeGenPrepare.cpp - Prepare a function for code generation --------===// 236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// The LLVM Compiler Infrastructure 436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This file is distributed under the University of Illinois Open Source 636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// License. See LICENSE.TXT for details. 736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 1036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This pass munges the code in the input function to better prepare it for 1136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// SelectionDAG-based code generation. This works around limitations in it's 1236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// basic-block-at-a-time approach. It should eventually be removed. 1336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 1436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 1536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/CodeGen/Passes.h" 1736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/ADT/DenseMap.h" 1836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/ADT/SmallSet.h" 1936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/ADT/Statistic.h" 2036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Analysis/InstructionSimplify.h" 2136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/CallSite.h" 2236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Constants.h" 2336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/DataLayout.h" 2436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/DerivedTypes.h" 2536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Dominators.h" 2636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Function.h" 2736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/GetElementPtrTypeIterator.h" 2836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/IRBuilder.h" 2936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/InlineAsm.h" 3036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Instructions.h" 3136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/IntrinsicInst.h" 3236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/PatternMatch.h" 3336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/ValueHandle.h" 3436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/ValueMap.h" 3536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Pass.h" 3636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Support/CommandLine.h" 3736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Support/Debug.h" 3836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Support/raw_ostream.h" 3936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Target/TargetLibraryInfo.h" 4036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Target/TargetLowering.h" 41dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Target/TargetSubtargetInfo.h" 4236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/BasicBlockUtils.h" 4336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/BuildLibCalls.h" 4436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/BypassSlowDivision.h" 4536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/Local.h" 4636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesusing namespace llvm; 4736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesusing namespace llvm::PatternMatch; 4836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 49dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "codegenprepare" 50dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumBlocksElim, "Number of blocks eliminated"); 5236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumPHIsElim, "Number of trivial PHIs eliminated"); 5336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumGEPsElim, "Number of GEPs converted to casts"); 5436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of " 5536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "sunken Cmps"); 5636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses " 5736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "of sunken Casts"); 5836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumMemoryInsts, "Number of memory instructions whose address " 5936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "computations were sunk"); 6036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumExtsMoved, "Number of [s|z]ext instructions combined with loads"); 6136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumExtUses, "Number of uses of [s|z]ext instructions optimized"); 6236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumRetsDup, "Number of return instructions duplicated"); 6336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumDbgValueMoved, "Number of debug value instructions moved"); 6436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumSelectsExpanded, "Number of selects turned into branches"); 6536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumAndCmpsMoved, "Number of and/cmp's pushed into branches"); 6636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 6736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<bool> DisableBranchOpts( 6836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "disable-cgp-branch-opts", cl::Hidden, cl::init(false), 6936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Disable branch optimizations in CodeGenPrepare")); 7036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 7136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<bool> DisableSelectToBranch( 7236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "disable-cgp-select2branch", cl::Hidden, cl::init(false), 7336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Disable select to branch conversion.")); 7436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 75dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic cl::opt<bool> AddrSinkUsingGEPs( 76dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "addr-sink-using-gep", cl::Hidden, cl::init(false), 77dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cl::desc("Address sinking in CGP using GEPs.")); 78dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 7936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<bool> EnableAndCmpSinking( 8036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "enable-andcmp-sinking", cl::Hidden, cl::init(true), 8136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Enable sinkinig and/cmp into branches.")); 8236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesnamespace { 8436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinestypedef SmallPtrSet<Instruction *, 16> SetOfInstrs; 8536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinestypedef DenseMap<Instruction *, Type *> InstrToOrigTy; 8636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class CodeGenPrepare : public FunctionPass { 8836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// TLI - Keep a pointer of a TargetLowering to consult for determining 8936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// transformation profitability. 9036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetMachine *TM; 9136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering *TLI; 9236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLibraryInfo *TLInfo; 9336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DominatorTree *DT; 9436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 9536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// CurInstIterator - As we scan instructions optimizing them, this is the 9636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// next instruction to optimize. Xforms that can invalidate this should 9736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// update it. 9836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator CurInstIterator; 9936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 10036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keeps track of non-local addresses that have been sunk into a block. 10136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// This allows us to avoid inserting duplicate code for blocks with 10236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// multiple load/stores of the same address. 10336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueMap<Value*, Value*> SunkAddrs; 10436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 10536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keeps track of all truncates inserted for the current function. 10636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SetOfInstrs InsertedTruncsSet; 10736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keeps track of the type of the related instruction before their 10836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// promotion for the current function. 10936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy PromotedInsts; 11036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 11136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// ModifiedDT - If CFG is modified in anyway, dominator tree may need to 11236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// be updated. 11336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool ModifiedDT; 11436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 11536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// OptSize - True if optimizing for size. 11636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptSize; 11736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 11836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 11936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static char ID; // Pass identification, replacement for typeid 120dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines explicit CodeGenPrepare(const TargetMachine *TM = nullptr) 121dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : FunctionPass(ID), TM(TM), TLI(nullptr) { 12236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines initializeCodeGenPreparePass(*PassRegistry::getPassRegistry()); 12336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 12436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool runOnFunction(Function &F) override; 12536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const char *getPassName() const override { return "CodeGen Prepare"; } 12736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void getAnalysisUsage(AnalysisUsage &AU) const override { 12936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AU.addPreserved<DominatorTreeWrapperPass>(); 13036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AU.addRequired<TargetLibraryInfo>(); 13136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 13236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 13336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines private: 13436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool EliminateFallThrough(Function &F); 13536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool EliminateMostlyEmptyBlocks(Function &F); 13636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool CanMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const; 13736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void EliminateMostlyEmptyBlock(BasicBlock *BB); 13836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeBlock(BasicBlock &BB); 13936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeInst(Instruction *I); 14036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeMemoryInst(Instruction *I, Value *Addr, Type *AccessTy); 14136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeInlineAsmInst(CallInst *CS); 14236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeCallInst(CallInst *CI); 14336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MoveExtToFormExtLoad(Instruction *I); 14436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeExtUses(Instruction *I); 14536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeSelectInst(SelectInst *SI); 14636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeShuffleVectorInst(ShuffleVectorInst *SI); 14736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool DupRetToEnableTailCallOpts(BasicBlock *BB); 14836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool PlaceDbgValues(Function &F); 14936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool sinkAndCmp(Function &F); 15036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 15136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 15236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 15336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hineschar CodeGenPrepare::ID = 0; 154cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen HinesINITIALIZE_TM_PASS(CodeGenPrepare, "codegenprepare", 155cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines "Optimize for code generation", false, false) 15636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 15736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesFunctionPass *llvm::createCodeGenPreparePass(const TargetMachine *TM) { 15836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return new CodeGenPrepare(TM); 15936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 16036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 16136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::runOnFunction(Function &F) { 16236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (skipOptnoneFunction(F)) 16336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 16436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 16536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool EverMadeChange = false; 16636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Clear per function information. 16736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTruncsSet.clear(); 16836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts.clear(); 16936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 17036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ModifiedDT = false; 17136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TM) TLI = TM->getTargetLowering(); 17236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLInfo = &getAnalysis<TargetLibraryInfo>(); 17336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DominatorTreeWrapperPass *DTWP = 17436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines getAnalysisIfAvailable<DominatorTreeWrapperPass>(); 175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DT = DTWP ? &DTWP->getDomTree() : nullptr; 17636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OptSize = F.getAttributes().hasAttribute(AttributeSet::FunctionIndex, 17736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Attribute::OptimizeForSize); 17836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 17936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// This optimization identifies DIV instructions that can be 18036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// profitably bypassed and carried out with a shorter, faster divide. 18136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!OptSize && TLI && TLI->isSlowDivBypassed()) { 18236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const DenseMap<unsigned int, unsigned int> &BypassWidths = 18336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->getBypassSlowDivWidths(); 18436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = F.begin(); I != F.end(); I++) 18536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= bypassSlowDivision(F, I, BypassWidths); 18636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 18736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 18836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Eliminate blocks that contain only PHI nodes and an 18936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // unconditional branch. 19036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= EliminateMostlyEmptyBlocks(F); 19136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 19236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // llvm.dbg.value is far away from the value then iSel may not be able 19336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // handle it properly. iSel will drop llvm.dbg.value if it can not 19436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // find a node corresponding to the value. 19536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= PlaceDbgValues(F); 19636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 19736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If there is a mask, compare against zero, and branch that can be combined 19836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // into a single target instruction, push the mask and compare into branch 19936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // users. Do this before OptimizeBlock -> OptimizeInst -> 20036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // OptimizeCmpExpression, which perturbs the pattern being searched for. 20136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DisableBranchOpts) 20236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= sinkAndCmp(F); 20336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 20436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = true; 20536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (MadeChange) { 20636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = false; 20736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = F.begin(); I != F.end(); ) { 20836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 20936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= OptimizeBlock(*BB); 21036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 21136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= MadeChange; 21236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 21336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 21436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 21536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 21636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DisableBranchOpts) { 21736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = false; 21836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<BasicBlock*, 8> WorkList; 21936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) { 22036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<BasicBlock*, 2> Successors(succ_begin(BB), succ_end(BB)); 22136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= ConstantFoldTerminator(BB, true); 22236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MadeChange) continue; 22336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 22436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallVectorImpl<BasicBlock*>::iterator 22536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines II = Successors.begin(), IE = Successors.end(); II != IE; ++II) 22636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (pred_begin(*II) == pred_end(*II)) 22736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WorkList.insert(*II); 22836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 22936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Delete the dead blocks and any of their dead successors. 23136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= !WorkList.empty(); 23236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (!WorkList.empty()) { 23336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = *WorkList.begin(); 23436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WorkList.erase(BB); 23536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<BasicBlock*, 2> Successors(succ_begin(BB), succ_end(BB)); 23636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DeleteDeadBlock(BB); 23836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallVectorImpl<BasicBlock*>::iterator 24036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines II = Successors.begin(), IE = Successors.end(); II != IE; ++II) 24136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (pred_begin(*II) == pred_end(*II)) 24236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WorkList.insert(*II); 24336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 24436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 24536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Merge pairs of basic blocks with unconditional branches, connected by 24636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // a single edge. 24736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (EverMadeChange || MadeChange) 24836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= EliminateFallThrough(F); 24936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 25036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MadeChange) 25136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ModifiedDT = true; 25236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= MadeChange; 25336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 25436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 25536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ModifiedDT && DT) 25636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DT->recalculate(F); 25736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 25836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return EverMadeChange; 25936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 26036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 26136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// EliminateFallThrough - Merge basic blocks which are connected 26236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// by a single edge, where one of the basic blocks has a single successor 26336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// pointing to the other basic block, which has a single predecessor. 26436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::EliminateFallThrough(Function &F) { 26536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Changed = false; 26636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Scan all of the blocks in the function, except for the entry block. 26736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) { 26836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 26936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the destination block has a single pred, then this is a trivial 27036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // edge, just collapse it. 27136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *SinglePred = BB->getSinglePredecessor(); 27236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 27336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't merge if BB's address is taken. 27436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue; 27536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 27636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator()); 27736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Term && !Term->isConditional()) { 27836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Changed = true; 27936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "To merge:\n"<< *SinglePred << "\n\n\n"); 28036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember if SinglePred was the entry block of the function. 28136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If so, we will need to move BB back to the entry position. 28236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); 28336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MergeBasicBlockIntoOnlyPred(BB, this); 28436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 28536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isEntry && BB != &BB->getParent()->getEntryBlock()) 28636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->moveBefore(&BB->getParent()->getEntryBlock()); 28736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 28836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We have erased a block. Update the iterator. 28936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I = BB; 29036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 29136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 29236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Changed; 29336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 29436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 29536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// EliminateMostlyEmptyBlocks - eliminate blocks that contain only PHI nodes, 29636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// debug info directives, and an unconditional branch. Passes before isel 29736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (e.g. LSR/loopsimplify) often split edges in ways that are non-optimal for 29836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// isel. Start by eliminating these blocks so we can split them the way we 29936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// want them. 30036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::EliminateMostlyEmptyBlocks(Function &F) { 30136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 30236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Note that this intentionally skips the entry block. 30336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) { 30436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 30536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 30636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this block doesn't end with an uncond branch, ignore it. 30736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); 30836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!BI || !BI->isUnconditional()) 30936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 31036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 31136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the instruction before the branch (skipping debug info) isn't a phi 31236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // node, then other stuff is happening here. 31336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator BBI = BI; 31436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BBI != BB->begin()) { 31536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines --BBI; 31636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (isa<DbgInfoIntrinsic>(BBI)) { 31736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BBI == BB->begin()) 31836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 31936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines --BBI; 32036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 32136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI)) 32236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 32336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 32436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 32536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do not break infinite loops. 32636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DestBB = BI->getSuccessor(0); 32736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (DestBB == BB) 32836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 32936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 33036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!CanMergeBlocks(BB, DestBB)) 33136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 33236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 33336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EliminateMostlyEmptyBlock(BB); 33436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 33536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 33636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 33736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 33836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 33936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// CanMergeBlocks - Return true if we can merge BB into DestBB if there is a 34036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// single uncond branch between them, and BB contains no other non-phi 34136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// instructions. 34236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::CanMergeBlocks(const BasicBlock *BB, 34336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const BasicBlock *DestBB) const { 34436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We only want to eliminate blocks whose phi nodes are used by phi nodes in 34536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the successor. If there are more complex condition (e.g. preheaders), 34636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // don't mess around with them. 34736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::const_iterator BBI = BB->begin(); 34836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) { 34936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (const User *U : PN->users()) { 35036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Instruction *UI = cast<Instruction>(U); 35136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UI->getParent() != DestBB || !isa<PHINode>(UI)) 35236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 35336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If User is inside DestBB block and it is a PHINode then check 35436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // incoming value. If incoming value is not from BB then this is 35536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // a complex condition (e.g. preheaders) we want to avoid here. 35636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UI->getParent() == DestBB) { 35736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const PHINode *UPN = dyn_cast<PHINode>(UI)) 35836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) { 35936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I)); 36036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Insn && Insn->getParent() == BB && 36136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Insn->getParent() != UPN->getIncomingBlock(I)) 36236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 36336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 36436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 36536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 36636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 36736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 36836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If BB and DestBB contain any common predecessors, then the phi nodes in BB 36936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // and DestBB may have conflicting incoming values for the block. If so, we 37036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // can't merge the block. 37136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin()); 37236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DestBBPN) return true; // no conflict. 37336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 37436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Collect the preds of BB. 37536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<const BasicBlock*, 16> BBPreds; 37636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { 37736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It is faster to get preds from a PHI than with pred_iterator. 37836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) 37936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BBPreds.insert(BBPN->getIncomingBlock(i)); 38036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 38136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BBPreds.insert(pred_begin(BB), pred_end(BB)); 38236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 38336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 38436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Walk the preds of DestBB. 38536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) { 38636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *Pred = DestBBPN->getIncomingBlock(i); 38736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BBPreds.count(Pred)) { // Common predecessor? 38836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BBI = DestBB->begin(); 38936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) { 39036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Value *V1 = PN->getIncomingValueForBlock(Pred); 39136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Value *V2 = PN->getIncomingValueForBlock(BB); 39236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 39336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If V2 is a phi node in BB, look up what the mapped value will be. 39436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const PHINode *V2PN = dyn_cast<PHINode>(V2)) 39536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V2PN->getParent() == BB) 39636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V2 = V2PN->getIncomingValueForBlock(Pred); 39736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 39836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If there is a conflict, bail out. 39936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V1 != V2) return false; 40036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 40136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 40236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 40336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 40436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 40536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 40636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 40736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 40836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// EliminateMostlyEmptyBlock - Eliminate a basic block that have only phi's and 40936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// an unconditional branch in it. 41036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid CodeGenPrepare::EliminateMostlyEmptyBlock(BasicBlock *BB) { 41136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BI = cast<BranchInst>(BB->getTerminator()); 41236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DestBB = BI->getSuccessor(0); 41336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 41436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" << *BB << *DestBB); 41536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 41636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the destination block has a single pred, then this is a trivial edge, 41736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // just collapse it. 41836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) { 41936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SinglePred != DestBB) { 42036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember if SinglePred was the entry block of the function. If so, we 42136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // will need to move BB back to the entry position. 42236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); 42336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MergeBasicBlockIntoOnlyPred(DestBB, this); 42436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 42536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isEntry && BB != &BB->getParent()->getEntryBlock()) 42636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->moveBefore(&BB->getParent()->getEntryBlock()); 42736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 42836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n"); 42936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 43036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 43136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 43236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 43336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB 43436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to handle the new incoming edges it is about to have. 43536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PHINode *PN; 43636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (BasicBlock::iterator BBI = DestBB->begin(); 43736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (PN = dyn_cast<PHINode>(BBI)); ++BBI) { 43836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remove the incoming value for BB, and remember it. 43936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *InVal = PN->removeIncomingValue(BB, false); 44036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 44136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Two options: either the InVal is a phi node defined in BB or it is some 44236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // value that dominates BB. 44336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PHINode *InValPhi = dyn_cast<PHINode>(InVal); 44436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (InValPhi && InValPhi->getParent() == BB) { 44536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add all of the input values of the input PHI as inputs of this phi. 44636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i) 44736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(InValPhi->getIncomingValue(i), 44836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InValPhi->getIncomingBlock(i)); 44936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 45036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, add one instance of the dominating value for each edge that 45136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // we will be adding. 45236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { 45336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) 45436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(InVal, BBPN->getIncomingBlock(i)); 45536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 45636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) 45736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(InVal, *PI); 45836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 45936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 46036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 46136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 46236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The PHIs are now updated, change everything that refers to BB to use 46336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // DestBB and remove BB. 46436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->replaceAllUsesWith(DestBB); 46536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (DT && !ModifiedDT) { 46636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BBIDom = DT->getNode(BB)->getIDom()->getBlock(); 46736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DestBBIDom = DT->getNode(DestBB)->getIDom()->getBlock(); 46836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *NewIDom = DT->findNearestCommonDominator(BBIDom, DestBBIDom); 46936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DT->changeImmediateDominator(DestBB, NewIDom); 47036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DT->eraseNode(BB); 47136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 47236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->eraseFromParent(); 47336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumBlocksElim; 47436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 47536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n"); 47636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 47736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 47836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// SinkCast - Sink the specified cast instruction into its user blocks 47936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool SinkCast(CastInst *CI) { 48036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = CI->getParent(); 48136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 48236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// InsertedCasts - Only insert a cast in each block once. 48336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, CastInst*> InsertedCasts; 48436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 48536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 48636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); 48736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI != E; ) { 48836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Use &TheUse = UI.getUse(); 48936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = cast<Instruction>(*UI); 49036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 49136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this cast is used in. For PHI's this is the 49236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // appropriate predecessor block. 49336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = User->getParent(); 49436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *PN = dyn_cast<PHINode>(User)) { 49536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UserBB = PN->getIncomingBlock(TheUse); 49636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 49736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 49836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Preincrement use iterator so we don't invalidate it. 49936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++UI; 50036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 50136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this user is in the same block as the cast, don't change the cast. 50236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 50336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 50436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we have already inserted a cast into this block, use it. 50536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CastInst *&InsertedCast = InsertedCasts[UserBB]; 50636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 50736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedCast) { 50836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 50936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedCast = 51036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CastInst::Create(CI->getOpcode(), CI->getOperand(0), CI->getType(), "", 51136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertPt); 51236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 51336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 51436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 51536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Replace a use of the cast with a use of the new cast. 51636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TheUse = InsertedCast; 51736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumCastUses; 51836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 51936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 52036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we removed all uses, nuke the cast. 52136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CI->use_empty()) { 52236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->eraseFromParent(); 52336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 52436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 52536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 52636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 52736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 52836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 52936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeNoopCopyExpression - If the specified cast instruction is a noop 53036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// copy (e.g. it's casting from one pointer type to another, i32->i8 on PPC), 53136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// sink it into user blocks to reduce the number of virtual 53236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// registers that must be created and coalesced. 53336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 53436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Return true if any changes are made. 53536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 53636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI){ 53736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is a noop copy, 53836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT SrcVT = TLI.getValueType(CI->getOperand(0)->getType()); 53936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT DstVT = TLI.getValueType(CI->getType()); 54036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 54136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This is an fp<->int conversion? 54236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SrcVT.isInteger() != DstVT.isInteger()) 54336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 54436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 54536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is an extension, it will be a zero or sign extension, which 54636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // isn't a noop. 54736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SrcVT.bitsLT(DstVT)) return false; 54836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 54936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If these values will be promoted, find out what they will be promoted 55036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to. This helps us consider truncates on PPC as noop copies when they 55136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // are. 55236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.getTypeAction(CI->getContext(), SrcVT) == 55336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::TypePromoteInteger) 55436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT); 55536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.getTypeAction(CI->getContext(), DstVT) == 55636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::TypePromoteInteger) 55736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT); 55836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 55936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If, after promotion, these are the same types, this is a noop copy. 56036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SrcVT != DstVT) 56136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 56236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 56336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SinkCast(CI); 56436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 56536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 56636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeCmpExpression - sink the given CmpInst into user blocks to reduce 56736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the number of virtual registers that must be created and coalesced. This is 56836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// a clear win except on targets with multiple condition code registers 56936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (PowerPC), where it might lose; some adjustment may be wanted there. 57036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 57136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Return true if any changes are made. 57236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool OptimizeCmpExpression(CmpInst *CI) { 57336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = CI->getParent(); 57436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 57536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// InsertedCmp - Only insert a cmp in each block once. 57636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, CmpInst*> InsertedCmps; 57736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 57836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 57936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); 58036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI != E; ) { 58136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Use &TheUse = UI.getUse(); 58236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = cast<Instruction>(*UI); 58336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 58436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Preincrement use iterator so we don't invalidate it. 58536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++UI; 58636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 58736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't bother for PHI nodes. 58836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<PHINode>(User)) 58936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 59036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 59136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this cmp is used in. 59236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = User->getParent(); 59336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 59436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this user is in the same block as the cmp, don't change the cmp. 59536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 59636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 59736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we have already inserted a cmp into this block, use it. 59836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst *&InsertedCmp = InsertedCmps[UserBB]; 59936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 60036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedCmp) { 60136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 60236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedCmp = 60336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst::Create(CI->getOpcode(), 60436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->getPredicate(), CI->getOperand(0), 60536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->getOperand(1), "", InsertPt); 60636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 60736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 60836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 60936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Replace a use of the cmp with a use of the new cmp. 61036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TheUse = InsertedCmp; 61136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumCmpUses; 61236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 61336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 61436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we removed all uses, nuke the cmp. 61536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CI->use_empty()) 61636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->eraseFromParent(); 61736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 61836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 61936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 62036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isExtractBitsCandidateUse - Check if the candidates could 622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// be combined with shift instruction, which includes: 623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 1. Truncate instruction 624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 2. And instruction and the imm is a mask of the low bits: 625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// imm & (imm+1) == 0 626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isExtractBitsCandidateUse(Instruction *User) { 627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isa<TruncInst>(User)) { 628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (User->getOpcode() != Instruction::And || 629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !isa<ConstantInt>(User->getOperand(1))) 630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue(); 633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((Cimm & (Cimm + 1)).getBoolValue()) 635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// SinkShiftAndTruncate - sink both shift and truncate instruction 641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// to the use of truncate's BB. 642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool 643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI, 644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts, 645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLowering &TLI) { 646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *UserBB = User->getParent(); 647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<BasicBlock *, CastInst *> InsertedTruncs; 648dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncInst *TruncI = dyn_cast<TruncInst>(User); 649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool MadeChange = false; 650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (Value::user_iterator TruncUI = TruncI->user_begin(), 652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncE = TruncI->user_end(); 653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncUI != TruncE;) { 654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Use &TruncTheUse = TruncUI.getUse(); 656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *TruncUser = cast<Instruction>(*TruncUI); 657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Preincrement use iterator so we don't invalidate it. 658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++TruncUI; 660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode()); 662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ISDOpcode) 663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the use is actually a legal node, there will not be an implicit 666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // truncate. 667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TLI.isOperationLegalOrCustom(ISDOpcode, 668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines EVT::getEVT(TruncUser->getType()))) 669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't bother for PHI nodes. 672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<PHINode>(TruncUser)) 673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 674dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *TruncUserBB = TruncUser->getParent(); 676dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (UserBB == TruncUserBB) 678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB]; 681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB]; 682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!InsertedShift && !InsertedTrunc) { 684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt(); 685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Sink the shift 686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShiftI->getOpcode() == Instruction::AShr) 687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "", InsertPt); 689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "", InsertPt); 692dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Sink the trunc 694dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt(); 695dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncInsertPt++; 696dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift, 698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncI->getType(), "", TruncInsertPt); 699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MadeChange = true; 701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncTheUse = InsertedTrunc; 703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 704dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 705dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MadeChange; 706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// OptimizeExtractBits - sink the shift *right* instruction into user blocks if 709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// the uses could potentially be combined with this shift instruction and 710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// generate BitExtract instruction. It will only be applied if the architecture 711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// supports BitExtract instruction. Here is an example: 712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// BB1: 713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.shift = lshr i64 %arg1, 32 714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// BB2: 715dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.trunc = trunc i64 %x.extract.shift to i16 716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// ==> 717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// BB2: 719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.shift.1 = lshr i64 %arg1, 32 720dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16 721dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// CodeGen will recoginze the pattern in BB2 and generate BitExtract 723dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// instruction. 724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Return true if any changes are made. 725dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI, 726dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLowering &TLI) { 727dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *DefBB = ShiftI->getParent(); 728dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 729dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines /// Only insert instructions in each block once. 730dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts; 731dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 732dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(ShiftI->getType())); 733dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 734dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool MadeChange = false; 735dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end(); 736dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UI != E;) { 737dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Use &TheUse = UI.getUse(); 738dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *User = cast<Instruction>(*UI); 739dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Preincrement use iterator so we don't invalidate it. 740dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++UI; 741dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 742dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't bother for PHI nodes. 743dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<PHINode>(User)) 744dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 745dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 746dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isExtractBitsCandidateUse(User)) 747dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 748dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 749dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *UserBB = User->getParent(); 750dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 751dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (UserBB == DefBB) { 752dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the shift and truncate instruction are in the same BB. The use of 753dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the truncate(TruncUse) may still introduce another truncate if not 754dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // legal. In this case, we would like to sink both shift and truncate 755dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instruction to the BB of TruncUse. 756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // for example: 757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // BB1: 758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i64 shift.result = lshr i64 opnd, imm 759dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // trunc.result = trunc shift.result to i16 760dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 761dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // BB2: 762dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ----> We will have an implicit truncate here if the architecture does 763dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // not have i16 compare. 764dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // cmp i16 trunc.result, opnd2 765dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 766dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<TruncInst>(User) && shiftIsLegal 767dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the type of the truncate is legal, no trucate will be 768dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // introduced in other basic blocks. 769dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines && (!TLI.isTypeLegal(TLI.getValueType(User->getType())))) 770dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MadeChange = 771dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI); 772dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 773dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 774dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 775dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we have already inserted a shift into this block, use it. 776dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator *&InsertedShift = InsertedShifts[UserBB]; 777dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 778dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!InsertedShift) { 779dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 780dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 781dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShiftI->getOpcode() == Instruction::AShr) 782dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 783dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "", InsertPt); 784dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 785dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 786dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "", InsertPt); 787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MadeChange = true; 789dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 790dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 791dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Replace a use of the shift with a use of the new shift. 792dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TheUse = InsertedShift; 793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 794dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 795dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we removed all uses, nuke the shift. 796dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShiftI->use_empty()) 797dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShiftI->eraseFromParent(); 798dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 799dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MadeChange; 800dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 801dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 80236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesnamespace { 80336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass CodeGenPrepareFortifiedLibCalls : public SimplifyFortifiedLibCalls { 80436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesprotected: 80536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void replaceCall(Value *With) override { 80636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->replaceAllUsesWith(With); 80736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->eraseFromParent(); 80836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 80936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool isFoldable(unsigned SizeCIOp, unsigned, bool) const override { 81036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstantInt *SizeCI = 81136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines dyn_cast<ConstantInt>(CI->getArgOperand(SizeCIOp))) 81236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SizeCI->isAllOnesValue(); 81336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 81436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 81536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 81636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} // end anonymous namespace 81736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 81836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeCallInst(CallInst *CI) { 81936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = CI->getParent(); 82036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 82136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower inline assembly if we can. 82236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we found an inline asm expession, and if the target knows how to 82336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // lower it to normal LLVM code, do so now. 82436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && isa<InlineAsm>(CI->getCalledValue())) { 82536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI->ExpandInlineAsm(CI)) { 82636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Avoid invalidating the iterator. 82736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = BB->begin(); 82836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Avoid processing instructions out of order, which could cause 82936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // reuse before a value is defined. 83036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 83136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 83236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 83336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Sink address computing for memory operands into the block. 83436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OptimizeInlineAsmInst(CI)) 83536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 83636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 83736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 83836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower all uses of llvm.objectsize.* 83936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI); 84036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (II && II->getIntrinsicID() == Intrinsic::objectsize) { 84136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Min = (cast<ConstantInt>(II->getArgOperand(1))->getZExtValue() == 1); 84236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *ReturnTy = CI->getType(); 84336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Constant *RetVal = ConstantInt::get(ReturnTy, Min ? 0 : -1ULL); 84436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 84536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Substituting this can cause recursive simplifications, which can 84636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // invalidate our iterator. Use a WeakVH to hold onto it in case this 84736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // happens. 84836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WeakVH IterHandle(CurInstIterator); 84936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines replaceAndRecursivelySimplify(CI, RetVal, 851dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TLI ? TLI->getDataLayout() : nullptr, 852dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TLInfo, ModifiedDT ? nullptr : DT); 85336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 85436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the iterator instruction was recursively deleted, start over at the 85536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // start of the block. 85636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IterHandle != CurInstIterator) { 85736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = BB->begin(); 85836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 85936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 86036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 86136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 86236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 86336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (II && TLI) { 86436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Value*, 2> PtrOps; 86536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *AccessTy; 86636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy)) 86736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (!PtrOps.empty()) 86836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OptimizeMemoryInst(II, PtrOps.pop_back_val(), AccessTy)) 86936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 87036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 87136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 87236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // From here on out we're working with named functions. 873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CI->getCalledFunction()) return false; 87436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 87536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We'll need DataLayout from here on out. 876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const DataLayout *TD = TLI ? TLI->getDataLayout() : nullptr; 87736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TD) return false; 87836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 87936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower all default uses of _chk calls. This is very similar 88036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to what InstCombineCalls does, but here we are only lowering calls 88136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // that have the default "don't know" as the objectsize. Anything else 88236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // should be left alone. 88336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CodeGenPrepareFortifiedLibCalls Simplifier; 88436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Simplifier.fold(CI, TD, TLInfo); 88536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 88636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 88736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// DupRetToEnableTailCallOpts - Look for opportunities to duplicate return 88836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// instructions to the predecessor to enable tail call optimizations. The 88936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// case it is currently looking for is: 89036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @code 89136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb0: 89236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp0 = tail call i32 @f0() 89336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// br label %return 89436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb1: 89536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp1 = tail call i32 @f1() 89636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// br label %return 89736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb2: 89836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp2 = tail call i32 @f2() 89936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// br label %return 90036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// return: 90136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ] 90236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %retval 90336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @endcode 90436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 90536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// => 90636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 90736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @code 90836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb0: 90936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp0 = tail call i32 @f0() 91036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %tmp0 91136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb1: 91236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp1 = tail call i32 @f1() 91336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %tmp1 91436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb2: 91536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp2 = tail call i32 @f2() 91636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %tmp2 91736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @endcode 91836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::DupRetToEnableTailCallOpts(BasicBlock *BB) { 91936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI) 92036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 92136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 92236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator()); 92336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!RI) 92436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 92536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PHINode *PN = nullptr; 927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BitCastInst *BCI = nullptr; 92836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = RI->getReturnValue(); 92936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V) { 93036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BCI = dyn_cast<BitCastInst>(V); 93136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BCI) 93236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = BCI->getOperand(0); 93336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 93436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN = dyn_cast<PHINode>(V); 93536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!PN) 93636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 93736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 93836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 93936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PN && PN->getParent() != BB) 94036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 94136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 94236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It's not safe to eliminate the sign / zero extension of the return value. 94336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // See llvm::isInTailCallPosition(). 94436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Function *F = BB->getParent(); 94536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AttributeSet CallerAttrs = F->getAttributes(); 94636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt) || 94736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt)) 94836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 94936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 95036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure there are no instructions between the PHI and return, or that the 95136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // return is the first instruction in the block. 95236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PN) { 95336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator BI = BB->begin(); 95436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines do { ++BI; } while (isa<DbgInfoIntrinsic>(BI)); 95536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (&*BI == BCI) 95636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Also skip over the bitcast. 95736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++BI; 95836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (&*BI != RI) 95936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 96036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 96136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator BI = BB->begin(); 96236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (isa<DbgInfoIntrinsic>(BI)) ++BI; 96336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (&*BI != RI) 96436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 96536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 96636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 96736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail 96836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// call. 96936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<CallInst*, 4> TailCalls; 97036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PN) { 97136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) { 97236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I)); 97336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure the phi value is indeed produced by the tail call. 97436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) && 97536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->mayBeEmittedAsTailCall(CI)) 97636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TailCalls.push_back(CI); 97736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 97836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 97936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<BasicBlock*, 4> VisitedBBs; 98036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) { 98136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!VisitedBBs.insert(*PI)) 98236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 98336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 98436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::InstListType &InstList = (*PI)->getInstList(); 98536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::InstListType::reverse_iterator RI = InstList.rbegin(); 98636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::InstListType::reverse_iterator RE = InstList.rend(); 98736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines do { ++RI; } while (RI != RE && isa<DbgInfoIntrinsic>(&*RI)); 98836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (RI == RE) 98936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 99036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 99136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallInst *CI = dyn_cast<CallInst>(&*RI); 99236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI)) 99336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TailCalls.push_back(CI); 99436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 99536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 99636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 99736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Changed = false; 99836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = TailCalls.size(); i != e; ++i) { 99936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallInst *CI = TailCalls[i]; 100036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallSite CS(CI); 100136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 100236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Conservatively require the attributes of the call to match those of the 100336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // return. Ignore noalias because it doesn't affect the call sequence. 100436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AttributeSet CalleeAttrs = CS.getAttributes(); 100536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex). 100636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines removeAttribute(Attribute::NoAlias) != 100736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex). 100836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines removeAttribute(Attribute::NoAlias)) 100936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 101036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 101136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure the call instruction is followed by an unconditional branch to 101236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the return block. 101336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *CallBB = CI->getParent(); 101436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator()); 101536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB) 101636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 101736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 101836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Duplicate the return into CallBB. 101936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (void)FoldReturnIntoUncondBranch(RI, BB, CallBB); 102036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ModifiedDT = Changed = true; 102136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumRetsDup; 102236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 102336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 102436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we eliminated all predecessors of the block, delete the block now. 102536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB)) 102636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->eraseFromParent(); 102736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 102836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Changed; 102936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 103036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 103136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 103236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Memory Optimization 103336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 103436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 103536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesnamespace { 103636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 103736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ExtAddrMode - This is an extended version of TargetLowering::AddrMode 103836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// which holds actual Value*'s for register values. 103936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstruct ExtAddrMode : public TargetLowering::AddrMode { 104036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *BaseReg; 104136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *ScaledReg; 1042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtAddrMode() : BaseReg(nullptr), ScaledReg(nullptr) {} 104336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void print(raw_ostream &OS) const; 104436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void dump() const; 104536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 104636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool operator==(const ExtAddrMode& O) const { 104736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return (BaseReg == O.BaseReg) && (ScaledReg == O.ScaledReg) && 104836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (BaseGV == O.BaseGV) && (BaseOffs == O.BaseOffs) && 104936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (HasBaseReg == O.HasBaseReg) && (Scale == O.Scale); 105036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 105136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 105236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 105336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#ifndef NDEBUG 105436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) { 105536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AM.print(OS); 105636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OS; 105736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 105836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#endif 105936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 106036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid ExtAddrMode::print(raw_ostream &OS) const { 106136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool NeedPlus = false; 106236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << "["; 106336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BaseGV) { 106436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << (NeedPlus ? " + " : "") 106536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "GV:"; 106636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BaseGV->printAsOperand(OS, /*PrintType=*/false); 106736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NeedPlus = true; 106836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 106936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1070cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (BaseOffs) { 1071cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines OS << (NeedPlus ? " + " : "") 1072cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines << BaseOffs; 1073cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines NeedPlus = true; 1074cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines } 107536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 107636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BaseReg) { 107736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << (NeedPlus ? " + " : "") 107836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "Base:"; 107936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BaseReg->printAsOperand(OS, /*PrintType=*/false); 108036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NeedPlus = true; 108136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 108236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Scale) { 108336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << (NeedPlus ? " + " : "") 108436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << Scale << "*"; 108536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ScaledReg->printAsOperand(OS, /*PrintType=*/false); 108636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 108736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 108836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << ']'; 108936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 109036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 109136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 109236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid ExtAddrMode::dump() const { 109336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print(dbgs()); 109436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines dbgs() << '\n'; 109536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 109636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#endif 109736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 109836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \brief This class provides transaction based operation on the IR. 109936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Every change made through this class is recorded in the internal state and 110036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// can be undone (rollback) until commit is called. 110136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass TypePromotionTransaction { 110236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 110336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief This represents the common interface of the individual transaction. 110436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Each class implements the logic for doing one specific modification on 110536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// the IR via the TypePromotionTransaction. 110636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class TypePromotionAction { 110736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines protected: 110836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The Instruction modified. 110936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Inst; 111036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 111136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 111236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Constructor of the action. 111336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The constructor performs the related action on the IR. 111436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionAction(Instruction *Inst) : Inst(Inst) {} 111536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 111636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines virtual ~TypePromotionAction() {} 111736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 111836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Undo the modification done by this action. 111936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// When this method is called, the IR must be in the same state as it was 112036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// before this action was applied. 112136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \pre Undoing the action works if and only if the IR is in the exact same 112236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// state as it was directly after this action was applied. 112336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines virtual void undo() = 0; 112436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 112536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Advocate every change made by this action. 112636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// When the results on the IR of the action are to be kept, it is important 112736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// to call this function, otherwise hidden information may be kept forever. 112836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines virtual void commit() { 112936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Nothing to be done, this action is not doing anything. 113036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 113136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 113236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 113336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Utility to remember the position of an instruction. 113436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class InsertionHandler { 113536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Position of an instruction. 113636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Either an instruction: 113736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// - Is the first in a basic block: BB is used. 113836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// - Has a previous instructon: PrevInst is used. 113936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines union { 114036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *PrevInst; 114136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB; 114236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } Point; 114336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Remember whether or not the instruction had a previous instruction. 114436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool HasPrevInstruction; 114536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 114636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 114736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Record the position of \p Inst. 114836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertionHandler(Instruction *Inst) { 114936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator It = Inst; 115036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines HasPrevInstruction = (It != (Inst->getParent()->begin())); 115136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (HasPrevInstruction) 115236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Point.PrevInst = --It; 115336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 115436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Point.BB = Inst->getParent(); 115536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 115636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 115736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Insert \p Inst at the recorded position. 115836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void insert(Instruction *Inst) { 115936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (HasPrevInstruction) { 116036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Inst->getParent()) 116136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->removeFromParent(); 116236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->insertAfter(Point.PrevInst); 116336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 116436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Position = Point.BB->getFirstInsertionPt(); 116536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Inst->getParent()) 116636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->moveBefore(Position); 116736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 116836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->insertBefore(Position); 116936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 117036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 117136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 117236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 117336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Move an instruction before another. 117436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class InstructionMoveBefore : public TypePromotionAction { 117536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Original position of the instruction. 117636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertionHandler Position; 117736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 117836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 117936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Move \p Inst before \p Before. 118036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstructionMoveBefore(Instruction *Inst, Instruction *Before) 118136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), Position(Inst) { 118236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before << "\n"); 118336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->moveBefore(Before); 118436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 118536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 118636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Move the instruction back to its original position. 118736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 118836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: moveBefore: " << *Inst << "\n"); 118936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Position.insert(Inst); 119036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 119136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 119236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 119336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Set the operand of an instruction with a new value. 119436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class OperandSetter : public TypePromotionAction { 119536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Original operand of the instruction. 119636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Origin; 119736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Index of the modified instruction. 119836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx; 119936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 120036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 120136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Set \p Idx operand of \p Inst with \p NewVal. 120236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal) 120336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), Idx(Idx) { 120436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n" 120536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "for:" << *Inst << "\n" 120636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "with:" << *NewVal << "\n"); 120736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Origin = Inst->getOperand(Idx); 120836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(Idx, NewVal); 120936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 121036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 121136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Restore the original value of the instruction. 121236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 121336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n" 121436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "for: " << *Inst << "\n" 121536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "with: " << *Origin << "\n"); 121636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(Idx, Origin); 121736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 121836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 121936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 122036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Hide the operands of an instruction. 122136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Do as if this instruction was not using any of its operands. 122236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class OperandsHider : public TypePromotionAction { 122336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The list of original operands. 122436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Value *, 4> OriginalValues; 122536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 122636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 122736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove \p Inst from the uses of the operands of \p Inst. 122836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) { 122936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n"); 123036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumOpnds = Inst->getNumOperands(); 123136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OriginalValues.reserve(NumOpnds); 123236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned It = 0; It < NumOpnds; ++It) { 123336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Save the current operand. 123436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Val = Inst->getOperand(It); 123536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OriginalValues.push_back(Val); 123636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Set a dummy one. 123736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We could use OperandSetter here, but that would implied an overhead 123836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // that we are not willing to pay. 123936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(It, UndefValue::get(Val->getType())); 124036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 124136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 124236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 124336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Restore the original list of uses. 124436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 124536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n"); 124636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It) 124736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(It, OriginalValues[It]); 124836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 124936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 125036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 125136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a truncate instruction. 125236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class TruncBuilder : public TypePromotionAction { 125336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 125436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a truncate instruction of \p Opnd producing a \p Ty 125536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// result. 125636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// trunc Opnd to Ty. 125736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) { 125836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(Opnd); 125936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst = cast<Instruction>(Builder.CreateTrunc(Opnd, Ty, "promoted")); 126036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: TruncBuilder: " << *Inst << "\n"); 126136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 126236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 126336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Get the built instruction. 126436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *getBuiltInstruction() { return Inst; } 126536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 126636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove the built instruction. 126736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 126836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: TruncBuilder: " << *Inst << "\n"); 126936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->eraseFromParent(); 127036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 127136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 127236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 127336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a sign extension instruction. 127436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class SExtBuilder : public TypePromotionAction { 127536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 127636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a sign extension instruction of \p Opnd producing a \p Ty 127736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// result. 127836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// sext Opnd to Ty. 127936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty) 128036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst) { 128136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(InsertPt); 128236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst = cast<Instruction>(Builder.CreateSExt(Opnd, Ty, "promoted")); 128336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: SExtBuilder: " << *Inst << "\n"); 128436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 128536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 128636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Get the built instruction. 128736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *getBuiltInstruction() { return Inst; } 128836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 128936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove the built instruction. 129036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 129136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: SExtBuilder: " << *Inst << "\n"); 129236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->eraseFromParent(); 129336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 129436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 129536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 129636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Mutate an instruction to another type. 129736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class TypeMutator : public TypePromotionAction { 129836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Record the original type. 129936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *OrigTy; 130036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 130136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 130236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Mutate the type of \p Inst into \p NewTy. 130336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypeMutator(Instruction *Inst, Type *NewTy) 130436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), OrigTy(Inst->getType()) { 130536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy 130636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "\n"); 130736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->mutateType(NewTy); 130836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 130936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 131036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Mutate the instruction back to its original type. 131136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 131236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy 131336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "\n"); 131436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->mutateType(OrigTy); 131536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 131636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 131736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 131836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Replace the uses of an instruction by another instruction. 131936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class UsesReplacer : public TypePromotionAction { 132036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Helper structure to keep track of the replaced uses. 132136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines struct InstructionAndIdx { 132236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The instruction using the instruction. 132336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Inst; 132436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The index where this instruction is used for Inst. 132536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx; 132636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstructionAndIdx(Instruction *Inst, unsigned Idx) 132736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : Inst(Inst), Idx(Idx) {} 132836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 132936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 133036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keep track of the original uses (pair Instruction, Index). 133136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<InstructionAndIdx, 4> OriginalUses; 133236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef SmallVectorImpl<InstructionAndIdx>::iterator use_iterator; 133336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 133436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 133536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Replace all the use of \p Inst by \p New. 133636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UsesReplacer(Instruction *Inst, Value *New) : TypePromotionAction(Inst) { 133736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New 133836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "\n"); 133936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Record the original uses. 134036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Use &U : Inst->uses()) { 134136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UserI = cast<Instruction>(U.getUser()); 134236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo())); 134336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 134436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Now, we can replace the uses. 134536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->replaceAllUsesWith(New); 134636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 134736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 134836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Reassign the original uses of Inst to Inst. 134936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 135036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n"); 135136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (use_iterator UseIt = OriginalUses.begin(), 135236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EndIt = OriginalUses.end(); 135336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UseIt != EndIt; ++UseIt) { 135436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UseIt->Inst->setOperand(UseIt->Idx, Inst); 135536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 135636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 135736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 135836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 135936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove an instruction from the IR. 136036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class InstructionRemover : public TypePromotionAction { 136136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Original position of the instruction. 136236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertionHandler Inserter; 136336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Helper structure to hide all the link to the instruction. In other 136436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// words, this helps to do as if the instruction was removed. 136536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OperandsHider Hider; 136636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keep track of the uses replaced, if any. 136736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UsesReplacer *Replacer; 136836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 136936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 137036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove all reference of \p Inst and optinally replace all its 137136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// uses with New. 1372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines /// \pre If !Inst->use_empty(), then New != nullptr 1373dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InstructionRemover(Instruction *Inst, Value *New = nullptr) 137436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst), 1375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Replacer(nullptr) { 137636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (New) 137736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Replacer = new UsesReplacer(Inst, New); 137836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n"); 137936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->removeFromParent(); 138036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 138136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 138236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ~InstructionRemover() { delete Replacer; } 138336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 138436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Really remove the instruction. 138536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void commit() override { delete Inst; } 138636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 138736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Resurrect the instruction and reassign it to the proper uses if 138836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// new value was provided when build this action. 138936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 139036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n"); 139136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inserter.insert(Inst); 139236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Replacer) 139336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Replacer->undo(); 139436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Hider.undo(); 139536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 139636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 139736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 139836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 139936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Restoration point. 140036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The restoration point is a pointer to an action instead of an iterator 140136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// because the iterator may be invalidated but not the pointer. 140236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef const TypePromotionAction *ConstRestorationPt; 140336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Advocate every changes made in that transaction. 140436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void commit(); 140536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Undo all the changes made after the given point. 140636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void rollback(ConstRestorationPt Point); 140736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Get the current restoration point. 140836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstRestorationPt getRestorationPoint() const; 140936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 141036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \name API for IR modification with state keeping to support rollback. 141136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// @{ 141236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Instruction::setOperand. 141336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal); 141436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Instruction::eraseFromParent. 1415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr); 141636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Value::replaceAllUsesWith. 141736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void replaceAllUsesWith(Instruction *Inst, Value *New); 141836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Value::mutateType. 141936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void mutateType(Instruction *Inst, Type *NewTy); 142036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as IRBuilder::createTrunc. 142136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *createTrunc(Instruction *Opnd, Type *Ty); 142236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as IRBuilder::createSExt. 142336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *createSExt(Instruction *Inst, Value *Opnd, Type *Ty); 142436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Instruction::moveBefore. 142536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void moveBefore(Instruction *Inst, Instruction *Before); 142636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// @} 142736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 142836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesprivate: 142936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The ordered list of actions made so far. 1430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<std::unique_ptr<TypePromotionAction>, 16> Actions; 1431dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines typedef SmallVectorImpl<std::unique_ptr<TypePromotionAction>>::iterator CommitPt; 143236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 143336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 143436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx, 143536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *NewVal) { 143636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.push_back( 1437dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines make_unique<TypePromotionTransaction::OperandSetter>(Inst, Idx, NewVal)); 143836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 143936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 144036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::eraseInstruction(Instruction *Inst, 144136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *NewVal) { 144236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.push_back( 1443dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines make_unique<TypePromotionTransaction::InstructionRemover>(Inst, NewVal)); 144436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 144536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 144636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst, 144736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *New) { 1448dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New)); 144936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 145036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 145136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) { 1452dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy)); 145336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 145436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 145536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesInstruction *TypePromotionTransaction::createTrunc(Instruction *Opnd, 145636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *Ty) { 1457dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty)); 1458dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *I = Ptr->getBuiltInstruction(); 1459dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(std::move(Ptr)); 1460dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return I; 146136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 146236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 146336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesInstruction *TypePromotionTransaction::createSExt(Instruction *Inst, 146436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Opnd, Type *Ty) { 1465dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty)); 1466dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *I = Ptr->getBuiltInstruction(); 1467dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(std::move(Ptr)); 1468dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return I; 146936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 147036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 147136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::moveBefore(Instruction *Inst, 147236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Before) { 147336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.push_back( 1474dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines make_unique<TypePromotionTransaction::InstructionMoveBefore>(Inst, Before)); 147536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 147636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 147736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesTypePromotionTransaction::ConstRestorationPt 147836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesTypePromotionTransaction::getRestorationPoint() const { 1479dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return !Actions.empty() ? Actions.back().get() : nullptr; 148036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 148136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 148236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::commit() { 148336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt; 1484dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++It) 148536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (*It)->commit(); 148636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.clear(); 148736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 148836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 148936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::rollback( 149036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt Point) { 1491dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines while (!Actions.empty() && Point != Actions.back().get()) { 1492dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val(); 149336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Curr->undo(); 149436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 149536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 149636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 149736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \brief A helper class for matching addressing modes. 149836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 149936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// This encapsulates the logic for matching the target-legal addressing modes. 150036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass AddressingModeMatcher { 150136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<Instruction*> &AddrModeInsts; 150236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI; 150336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 150436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and 150536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// the memory instruction that we're computing this address for. 150636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *AccessTy; 150736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *MemoryInst; 150836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 150936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// AddrMode - This is the addressing mode that we're building up. This is 151036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// part of the return value of this addressing mode matching stuff. 151136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AddrMode; 151236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 151336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The truncate instruction inserted by other CodeGenPrepare optimizations. 151436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SetOfInstrs &InsertedTruncs; 151536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// A map from the instructions to their type before promotion. 151636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts; 151736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The ongoing transaction where every action should be registered. 151836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT; 151936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 152036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// IgnoreProfitability - This is set to true when we should not do 152136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// profitability checks. When true, IsProfitableToFoldIntoAddressingMode 152236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// always returns true. 152336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IgnoreProfitability; 152436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 152536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddressingModeMatcher(SmallVectorImpl<Instruction*> &AMI, 152636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &T, Type *AT, 152736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *MI, ExtAddrMode &AM, 152836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SetOfInstrs &InsertedTruncs, 152936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 153036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT) 153136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : AddrModeInsts(AMI), TLI(T), AccessTy(AT), MemoryInst(MI), AddrMode(AM), 153236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTruncs(InsertedTruncs), PromotedInsts(PromotedInsts), TPT(TPT) { 153336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IgnoreProfitability = false; 153436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 153536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 153636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 153736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Match - Find the maximal addressing mode that a load/store of V can fold, 153836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// give an access type of AccessTy. This returns a list of involved 153936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// instructions in AddrModeInsts. 154036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p InsertedTruncs The truncate instruction inserted by other 154136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// CodeGenPrepare 154236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// optimizations. 154336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 154436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p The ongoing transaction where every action should be registered. 154536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static ExtAddrMode Match(Value *V, Type *AccessTy, 154636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *MemoryInst, 154736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<Instruction*> &AddrModeInsts, 154836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI, 154936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SetOfInstrs &InsertedTruncs, 155036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 155136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT) { 155236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode Result; 155336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 155436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Success = AddressingModeMatcher(AddrModeInsts, TLI, AccessTy, 155536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryInst, Result, InsertedTruncs, 155636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts, TPT).MatchAddr(V, 0); 155736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (void)Success; assert(Success && "Couldn't select *anything*?"); 155836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Result; 155936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 156036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesprivate: 156136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MatchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth); 156236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MatchAddr(Value *V, unsigned Depth); 156336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MatchOperationAddr(User *Operation, unsigned Opcode, unsigned Depth, 1564dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool *MovedAway = nullptr); 156536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IsProfitableToFoldIntoAddressingMode(Instruction *I, 156636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AMBefore, 156736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AMAfter); 156836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool ValueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2); 156936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IsPromotionProfitable(unsigned MatchedSize, unsigned SizeWithPromotion, 157036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *PromotedOperand) const; 157136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 157236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 157336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MatchScaledValue - Try adding ScaleReg*Scale to the current addressing mode. 157436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Return true and update AddrMode if this addr mode is legal for the target, 157536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// false if not. 157636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::MatchScaledValue(Value *ScaleReg, int64_t Scale, 157736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth) { 157836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If Scale is 1, then this is the same as adding ScaleReg to the addressing 157936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // mode. Just process that directly. 158036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Scale == 1) 158136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(ScaleReg, Depth); 158236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 158336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the scale is 0, it takes nothing to add this. 158436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Scale == 0) 158536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 158636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 158736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we already have a scale of this value, we can add to it, otherwise, we 158836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // need an available scale field. 158936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg) 159036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 159136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 159236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode TestAddrMode = AddrMode; 159336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 159436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add scale to turn X*4+X*3 -> X*7. This could also do things like 159536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // [A+B + A*7] -> [B+A*8]. 159636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.Scale += Scale; 159736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.ScaledReg = ScaleReg; 159836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 159936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the new address isn't legal, bail out. 160036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI.isLegalAddressingMode(TestAddrMode, AccessTy)) 160136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 160236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 160336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It was legal, so commit it. 160436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = TestAddrMode; 160536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 160636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Okay, we decided that we can add ScaleReg+Scale to AddrMode. Check now 160736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to see if ScaleReg is actually X+C. If so, we can turn this into adding 160836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // X*Scale + C*Scale to addr mode. 1609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantInt *CI = nullptr; Value *AddLHS = nullptr; 161036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<Instruction>(ScaleReg) && // not a constant expr. 161136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI)))) { 161236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.ScaledReg = AddLHS; 161336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale; 161436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 161536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this addressing mode is legal, commit it and remember that we folded 161636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // this instruction. 161736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(TestAddrMode, AccessTy)) { 161836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.push_back(cast<Instruction>(ScaleReg)); 161936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = TestAddrMode; 162036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 162136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 162236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 162336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 162436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, not (x+c)*scale, just return what we have. 162536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 162636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 162736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 162836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MightBeFoldableInst - This is a little filter, which returns true if an 162936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// addressing computation involving I might be folded into a load/store 163036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// accessing it. This doesn't need to be perfect, but needs to accept at least 163136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the set of instructions that MatchOperationAddr can. 163236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool MightBeFoldableInst(Instruction *I) { 163336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (I->getOpcode()) { 163436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::BitCast: 1635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Instruction::AddrSpaceCast: 163636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't touch identity bitcasts. 163736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (I->getType() == I->getOperand(0)->getType()) 163836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 163936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return I->getType()->isPointerTy() || I->getType()->isIntegerTy(); 164036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::PtrToInt: 164136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // PtrToInt is always a noop, as we know that the int type is pointer sized. 164236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 164336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::IntToPtr: 164436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We know the input is intptr_t, so this is foldable. 164536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 164636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Add: 164736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 164836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Mul: 164936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Shl: 165036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Can only handle X*C and X << C. 165136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return isa<ConstantInt>(I->getOperand(1)); 165236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::GetElementPtr: 165336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 165436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines default: 165536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 165636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 165736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 165836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 165936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \brief Hepler class to perform type promotion. 166036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass TypePromotionHelper { 166136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Utility function to check whether or not a sign extension of 166236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p Inst with \p ConsideredSExtType can be moved through \p Inst by either 166336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// using the operands of \p Inst or promoting \p Inst. 166436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// In other words, check if: 166536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// sext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredSExtType. 166636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// #1 Promotion applies: 166736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// ConsideredSExtType Inst (sext opnd1 to ConsideredSExtType, ...). 166836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// #2 Operand reuses: 166936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// sext opnd1 to ConsideredSExtType. 167036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 167136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static bool canGetThrough(const Instruction *Inst, Type *ConsideredSExtType, 167236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const InstrToOrigTy &PromotedInsts); 167336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 167436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Utility function to determine if \p OpIdx should be promoted when 167536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// promoting \p Inst. 167636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static bool shouldSExtOperand(const Instruction *Inst, int OpIdx) { 167736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<SelectInst>(Inst) && OpIdx == 0) 167836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 167936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 168036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 168136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 168236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Utility function to promote the operand of \p SExt when this 168336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// operand is a promotable trunc or sext. 168436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 168536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p CreatedInsts[out] contains how many non-free instructions have been 168636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// created to promote the operand of SExt. 168736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Should never be called directly. 168836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \return The promoted value which is used instead of SExt. 168936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static Value *promoteOperandForTruncAndSExt(Instruction *SExt, 169036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT, 169136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 169236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned &CreatedInsts); 169336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 169436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Utility function to promote the operand of \p SExt when this 169536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// operand is promotable and is not a supported trunc or sext. 169636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 169736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p CreatedInsts[out] contains how many non-free instructions have been 169836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// created to promote the operand of SExt. 169936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Should never be called directly. 170036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \return The promoted value which is used instead of SExt. 170136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static Value *promoteOperandForOther(Instruction *SExt, 170236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT, 170336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 170436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned &CreatedInsts); 170536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 170636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 170736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Type for the utility function that promotes the operand of SExt. 170836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef Value *(*Action)(Instruction *SExt, TypePromotionTransaction &TPT, 170936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 171036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned &CreatedInsts); 171136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Given a sign extend instruction \p SExt, return the approriate 171236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// action to promote the operand of \p SExt instead of using SExt. 171336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \return NULL if no promotable action is possible with the current 171436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// sign extension. 171536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p InsertedTruncs keeps track of all the truncate instructions inserted by 171636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// the others CodeGenPrepare optimizations. This information is important 171736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// because we do not want to promote these instructions as CodeGenPrepare 171836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// will reinsert them later. Thus creating an infinite loop: create/remove. 171936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 172036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static Action getAction(Instruction *SExt, const SetOfInstrs &InsertedTruncs, 172136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI, 172236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const InstrToOrigTy &PromotedInsts); 172336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 172436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 172536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool TypePromotionHelper::canGetThrough(const Instruction *Inst, 172636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *ConsideredSExtType, 172736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const InstrToOrigTy &PromotedInsts) { 172836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We can always get through sext. 172936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<SExtInst>(Inst)) 173036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 173136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 173236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We can get through binary operator, if it is legal. In other words, the 173336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // binary operator must have a nuw or nsw flag. 173436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Inst); 173536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BinOp && isa<OverflowingBinaryOperator>(BinOp) && 173636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (BinOp->hasNoUnsignedWrap() || BinOp->hasNoSignedWrap())) 173736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 173836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 173936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if we can do the following simplification. 174036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // sext(trunc(sext)) --> sext 174136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<TruncInst>(Inst)) 174236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 174336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 174436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *OpndVal = Inst->getOperand(0); 174536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if we can use this operand in the sext. 174636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the type is larger than the result type of the sign extension, 174736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // we cannot. 174836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OpndVal->getType()->getIntegerBitWidth() > 174936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConsideredSExtType->getIntegerBitWidth()) 175036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 175136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 175236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the operand of the truncate is not an instruction, we will not have 175336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // any information on the dropped bits. 175436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // (Actually we could for constant but it is not worth the extra logic). 175536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Opnd = dyn_cast<Instruction>(OpndVal); 175636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Opnd) 175736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 175836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 175936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if the source of the type is narrow enough. 176036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // I.e., check that trunc just drops sign extended bits. 176136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // #1 get the type of the operand. 176236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Type *OpndType; 176336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd); 176436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (It != PromotedInsts.end()) 176536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OpndType = It->second; 176636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else if (isa<SExtInst>(Opnd)) 176736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OpndType = cast<Instruction>(Opnd)->getOperand(0)->getType(); 176836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 176936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 177036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 177136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // #2 check that the truncate just drop sign extended bits. 177236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Inst->getType()->getIntegerBitWidth() >= OpndType->getIntegerBitWidth()) 177336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 177436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 177536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 177636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 177736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 177836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesTypePromotionHelper::Action TypePromotionHelper::getAction( 177936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *SExt, const SetOfInstrs &InsertedTruncs, 178036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) { 178136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *SExtOpnd = dyn_cast<Instruction>(SExt->getOperand(0)); 178236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *SExtTy = SExt->getType(); 178336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the operand of the sign extension is not an instruction, we cannot 178436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // get through. 178536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If it, check we can get through. 178636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SExtOpnd || !canGetThrough(SExtOpnd, SExtTy, PromotedInsts)) 1787dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 178836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 178936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do not promote if the operand has been added by codegenprepare. 179036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, it means we are undoing an optimization that is likely to be 179136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // redone, thus causing potential infinite loop. 179236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<TruncInst>(SExtOpnd) && InsertedTruncs.count(SExtOpnd)) 1793dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 179436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 179536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // SExt or Trunc instructions. 179636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Return the related handler. 179736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<SExtInst>(SExtOpnd) || isa<TruncInst>(SExtOpnd)) 179836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return promoteOperandForTruncAndSExt; 179936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 180036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Regular instruction. 180136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Abort early if we will have to insert non-free instructions. 180236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SExtOpnd->hasOneUse() && 180336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !TLI.isTruncateFree(SExtTy, SExtOpnd->getType())) 1804dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 180536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return promoteOperandForOther; 180636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 180736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 180836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesValue *TypePromotionHelper::promoteOperandForTruncAndSExt( 180936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm::Instruction *SExt, TypePromotionTransaction &TPT, 181036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, unsigned &CreatedInsts) { 181136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // By construction, the operand of SExt is an instruction. Otherwise we cannot 181236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // get through it and this method should not be called. 181336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0)); 181436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Replace sext(trunc(opnd)) or sext(sext(opnd)) 181536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // => sext(opnd). 181636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0)); 181736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CreatedInsts = 0; 181836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 181936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remove dead code. 182036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SExtOpnd->use_empty()) 182136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.eraseInstruction(SExtOpnd); 182236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 182336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if the sext is still needed. 182436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SExt->getType() != SExt->getOperand(0)->getType()) 182536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SExt; 182636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 182736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // At this point we have: sext ty opnd to ty. 182836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Reassign the uses of SExt to the opnd and remove SExt. 182936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *NextVal = SExt->getOperand(0); 183036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.eraseInstruction(SExt, NextVal); 183136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return NextVal; 183236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 183336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 183436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesValue * 183536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesTypePromotionHelper::promoteOperandForOther(Instruction *SExt, 183636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT, 183736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 183836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned &CreatedInsts) { 183936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // By construction, the operand of SExt is an instruction. Otherwise we cannot 184036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // get through it and this method should not be called. 184136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0)); 184236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CreatedInsts = 0; 184336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SExtOpnd->hasOneUse()) { 184436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // SExtOpnd will be promoted. 184536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // All its uses, but SExt, will need to use a truncated value of the 184636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // promoted version. 184736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Create the truncate now. 184836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Trunc = TPT.createTrunc(SExt, SExtOpnd->getType()); 184936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Trunc->removeFromParent(); 185036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Insert it just after the definition. 185136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Trunc->insertAfter(SExtOpnd); 185236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 185336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.replaceAllUsesWith(SExtOpnd, Trunc); 185436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Restore the operand of SExt (which has been replace by the previous call 185536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext. 185636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.setOperand(SExt, 0, SExtOpnd); 185736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 185836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 185936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Get through the Instruction: 186036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // 1. Update its type. 186136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // 2. Replace the uses of SExt by Inst. 186236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // 3. Sign extend each operand that needs to be sign extended. 186336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 186436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember the original type of the instruction before promotion. 186536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This is useful to know that the high bits are sign extended bits. 186636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts.insert( 186736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines std::pair<Instruction *, Type *>(SExtOpnd, SExtOpnd->getType())); 186836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Step #1. 186936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.mutateType(SExtOpnd, SExt->getType()); 187036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Step #2. 187136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.replaceAllUsesWith(SExt, SExtOpnd); 187236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Step #3. 187336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *SExtForOpnd = SExt; 187436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 187536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Propagate SExt to operands\n"); 187636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (int OpIdx = 0, EndOpIdx = SExtOpnd->getNumOperands(); OpIdx != EndOpIdx; 187736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++OpIdx) { 187836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Operand:\n" << *(SExtOpnd->getOperand(OpIdx)) << '\n'); 187936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SExtOpnd->getOperand(OpIdx)->getType() == SExt->getType() || 188036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !shouldSExtOperand(SExtOpnd, OpIdx)) { 188136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "No need to propagate\n"); 188236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 188336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 188436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if we can statically sign extend the operand. 188536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Opnd = SExtOpnd->getOperand(OpIdx); 188636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) { 188736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Statically sign extend\n"); 188836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.setOperand( 188936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SExtOpnd, OpIdx, 189036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt::getSigned(SExt->getType(), Cst->getSExtValue())); 189136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 189236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 189336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // UndefValue are typed, so we have to statically sign extend them. 189436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<UndefValue>(Opnd)) { 189536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Statically sign extend\n"); 189636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.setOperand(SExtOpnd, OpIdx, UndefValue::get(SExt->getType())); 189736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 189836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 189936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 190036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise we have to explicity sign extend the operand. 190136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if SExt was reused to sign extend an operand. 190236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SExtForOpnd) { 190336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If yes, create a new one. 190436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "More operands to sext\n"); 190536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SExtForOpnd = TPT.createSExt(SExt, Opnd, SExt->getType()); 190636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++CreatedInsts; 190736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 190836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 190936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.setOperand(SExtForOpnd, 0, Opnd); 191036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 191136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Move the sign extension before the insertion point. 191236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.moveBefore(SExtForOpnd, SExtOpnd); 191336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.setOperand(SExtOpnd, OpIdx, SExtForOpnd); 191436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If more sext are required, new instructions will have to be created. 1915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SExtForOpnd = nullptr; 191636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 191736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SExtForOpnd == SExt) { 191836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Sign extension is useless now\n"); 191936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.eraseInstruction(SExt); 192036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 192136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SExtOpnd; 192236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 192336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 192436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsPromotionProfitable - Check whether or not promoting an instruction 192536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// to a wider type was profitable. 192636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p MatchedSize gives the number of instructions that have been matched 192736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// in the addressing mode after the promotion was applied. 192836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p SizeWithPromotion gives the number of created instructions for 192936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the promotion plus the number of instructions that have been 193036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// matched in the addressing mode before the promotion. 193136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p PromotedOperand is the value that has been promoted. 193236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \return True if the promotion is profitable, false otherwise. 193336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool 193436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesAddressingModeMatcher::IsPromotionProfitable(unsigned MatchedSize, 193536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned SizeWithPromotion, 193636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *PromotedOperand) const { 193736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We folded less instructions than what we created to promote the operand. 193836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This is not profitable. 193936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchedSize < SizeWithPromotion) 194036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 194136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchedSize > SizeWithPromotion) 194236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 194336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The promotion is neutral but it may help folding the sign extension in 194436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // loads for instance. 194536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check that we did not create an illegal instruction. 194636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *PromotedInst = dyn_cast<Instruction>(PromotedOperand); 194736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!PromotedInst) 194836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 194936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode()); 195036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the ISDOpcode is undefined, it was undefined before the promotion. 195136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ISDOpcode) 195236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 195336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, check if the promoted instruction is legal or not. 195436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return TLI.isOperationLegalOrCustom(ISDOpcode, 195536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT::getEVT(PromotedInst->getType())); 195636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 195736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 195836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MatchOperationAddr - Given an instruction or constant expr, see if we can 195936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// fold the operation into the addressing mode. If so, update the addressing 196036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// mode and return true, otherwise return false without modifying AddrMode. 196136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// If \p MovedAway is not NULL, it contains the information of whether or 196236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// not AddrInst has to be folded into the addressing mode on success. 196336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// If \p MovedAway == true, \p AddrInst will not be part of the addressing 196436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// because it has been moved away. 196536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Thus AddrInst must not be added in the matched instructions. 196636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// This state can happen when AddrInst is a sext, since it may be moved away. 196736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Therefore, AddrInst may not be valid when MovedAway is true and it must 196836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// not be referenced anymore. 196936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode, 197036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth, 197136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool *MovedAway) { 197236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Avoid exponential behavior on extremely deep expression trees. 197336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Depth >= 5) return false; 197436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 197536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // By default, all matched instructions stay in place. 197636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MovedAway) 197736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines *MovedAway = false; 197836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 197936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (Opcode) { 198036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::PtrToInt: 198136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // PtrToInt is always a noop, as we know that the int type is pointer sized. 198236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(AddrInst->getOperand(0), Depth); 198336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::IntToPtr: 198436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This inttoptr is a no-op if the integer type is pointer sized. 198536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.getValueType(AddrInst->getOperand(0)->getType()) == 198636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI.getPointerTy(AddrInst->getType()->getPointerAddressSpace())) 198736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(AddrInst->getOperand(0), Depth); 198836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 198936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::BitCast: 1990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Instruction::AddrSpaceCast: 199136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // BitCast is always a noop, and we can handle it as long as it is 199236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // int->int or pointer->pointer (we don't want int<->fp or something). 199336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if ((AddrInst->getOperand(0)->getType()->isPointerTy() || 199436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrInst->getOperand(0)->getType()->isIntegerTy()) && 199536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't touch identity bitcasts. These were probably put here by LSR, 199636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // and we don't want to mess around with them. Assume it knows what it 199736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // is doing. 199836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrInst->getOperand(0)->getType() != AddrInst->getType()) 199936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(AddrInst->getOperand(0), Depth); 200036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 200136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Add: { 200236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if we can merge in the RHS then the LHS. If so, we win. 200336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 200436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 200536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Start a transaction at this point. 200636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The LHS may match but not the RHS. 200736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Therefore, we need a higher level restoration point to undo partially 200836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // matched operation. 200936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 201036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 201136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 201236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchAddr(AddrInst->getOperand(1), Depth+1) && 201336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MatchAddr(AddrInst->getOperand(0), Depth+1)) 201436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 201536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 201636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Restore the old addr mode info. 201736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 201836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 201936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 202036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 202136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise this was over-aggressive. Try merging in the LHS then the RHS. 202236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchAddr(AddrInst->getOperand(0), Depth+1) && 202336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MatchAddr(AddrInst->getOperand(1), Depth+1)) 202436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 202536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 202636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise we definitely can't merge the ADD in. 202736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 202836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 202936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 203036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 203136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 203236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines //case Instruction::Or: 203336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD. 203436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines //break; 203536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Mul: 203636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Shl: { 203736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Can only handle X*C and X << C. 203836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1)); 203936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!RHS) return false; 204036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t Scale = RHS->getSExtValue(); 204136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Opcode == Instruction::Shl) 204236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scale = 1LL << Scale; 204336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 204436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchScaledValue(AddrInst->getOperand(0), Scale, Depth); 204536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 204636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::GetElementPtr: { 204736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Scan the GEP. We check it if it contains constant offsets and at most 204836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // one variable offset. 204936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int VariableOperand = -1; 205036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned VariableScale = 0; 205136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 205236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t ConstantOffset = 0; 205336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const DataLayout *TD = TLI.getDataLayout(); 205436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gep_type_iterator GTI = gep_type_begin(AddrInst); 205536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) { 205636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StructType *STy = dyn_cast<StructType>(*GTI)) { 205736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const StructLayout *SL = TD->getStructLayout(STy); 205836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx = 205936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue(); 206036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantOffset += SL->getElementOffset(Idx); 206136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 206236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t TypeSize = TD->getTypeAllocSize(GTI.getIndexedType()); 206336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) { 206436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantOffset += CI->getSExtValue()*TypeSize; 206536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (TypeSize) { // Scales of zero don't do anything. 206636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We only allow one variable index at the moment. 206736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VariableOperand != -1) 206836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 206936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 207036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember the variable index. 207136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VariableOperand = i; 207236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VariableScale = TypeSize; 207336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 207436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 207536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 207636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 207736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // A common case is for the GEP to only do a constant offset. In this case, 207836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // just add it to the disp field and check validity. 207936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VariableOperand == -1) { 208036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += ConstantOffset; 208136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstantOffset == 0 || TLI.isLegalAddressingMode(AddrMode, AccessTy)){ 208236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if we can fold the base pointer in too. 208336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchAddr(AddrInst->getOperand(0), Depth+1)) 208436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 208536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 208636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs -= ConstantOffset; 208736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 208836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 208936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 209036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Save the valid addressing mode in case we can't match. 209136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 209236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 209336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 209436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // See if the scale and offset amount is valid for this target. 209536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += ConstantOffset; 209636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 209736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Match the base operand of the GEP. 209836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchAddr(AddrInst->getOperand(0), Depth+1)) { 209936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If it couldn't be matched, just stuff the value in a register. 210036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.HasBaseReg) { 210136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 210236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 210336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 210436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 210536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = true; 210636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseReg = AddrInst->getOperand(0); 210736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 210836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 210936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Match the remaining variable portion of the GEP. 211036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale, 211136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Depth)) { 211236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If it couldn't be matched, try stuffing the base into a register 211336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // instead of matching it, and retrying the match of the scale. 211436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 211536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 211636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.HasBaseReg) 211736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 211836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = true; 211936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseReg = AddrInst->getOperand(0); 212036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += ConstantOffset; 212136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchScaledValue(AddrInst->getOperand(VariableOperand), 212236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VariableScale, Depth)) { 212336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If even that didn't work, bail. 212436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 212536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 212636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 212736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 212836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 212936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 213036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 213136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 213236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::SExt: { 213336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Try to move this sext out of the way of the addressing mode. 213436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *SExt = cast<Instruction>(AddrInst); 213536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Ask for a method for doing so. 213636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionHelper::Action TPH = TypePromotionHelper::getAction( 213736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SExt, InsertedTruncs, TLI, PromotedInsts); 213836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TPH) 213936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 214036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 214136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 214236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 214336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned CreatedInsts = 0; 214436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *PromotedOperand = TPH(SExt, TPT, PromotedInsts, CreatedInsts); 214536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // SExt has been moved away. 214636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Thus either it will be rematched later in the recursive calls or it is 214736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // gone. Anyway, we must not fold it into the addressing mode at this point. 214836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // E.g., 214936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // op = add opnd, 1 215036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // idx = sext op 215136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // addr = gep base, idx 215236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // is now: 215336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // promotedOpnd = sext opnd <- no match here 215436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls) 215536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // addr = gep base, op <- match 215636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MovedAway) 215736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines *MovedAway = true; 215836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 215936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(PromotedOperand && 216036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "TypePromotionHelper should have filtered out those cases"); 216136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 216236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 216336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 216436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 216536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchAddr(PromotedOperand, Depth) || 216636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !IsPromotionProfitable(AddrModeInsts.size(), OldSize + CreatedInsts, 216736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedOperand)) { 216836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 216936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 217036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Sign extension does not pay off: rollback\n"); 217136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 217236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 217336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 217436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 217536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 217636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 217736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 217836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 217936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 218036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MatchAddr - If we can, try to add the value of 'Addr' into the current 218136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// addressing mode. If Addr can't be added to AddrMode this returns false and 218236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// leaves AddrMode unmodified. This assumes that Addr is either a pointer type 218336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// or intptr_t for the target. 218436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 218536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::MatchAddr(Value *Addr, unsigned Depth) { 218636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Start a transaction at this point that we will rollback if the matching 218736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // fails. 218836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 218936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 219036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) { 219136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Fold in immediates if legal for the target. 219236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += CI->getSExtValue(); 219336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 219436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 219536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs -= CI->getSExtValue(); 219636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) { 219736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is a global variable, try to fold it into the addressing mode. 2198dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!AddrMode.BaseGV) { 219936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseGV = GV; 220036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 220136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 2202dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseGV = nullptr; 220336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 220436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (Instruction *I = dyn_cast<Instruction>(Addr)) { 220536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 220636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 220736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 220836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if it is possible to fold this operation. 220936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MovedAway = false; 221036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) { 221136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This instruction may have been move away. If so, there is nothing 221236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to check here. 221336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MovedAway) 221436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 221536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Okay, it's possible to fold this. Check to see if it is actually 221636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // *profitable* to do so. We use a simple cost model to avoid increasing 221736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // register pressure too much. 221836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (I->hasOneUse() || 221936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IsProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) { 222036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.push_back(I); 222136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 222236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 222336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 222436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It isn't profitable to do this, roll back. 222536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines //cerr << "NOT FOLDING: " << *I; 222636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 222736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 222836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 222936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 223036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) { 223136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchOperationAddr(CE, CE->getOpcode(), Depth)) 223236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 223336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 223436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (isa<ConstantPointerNull>(Addr)) { 223536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Null pointer gets folded without affecting the addressing mode. 223636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 223736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 223836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 223936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Worse case, the target should support [reg] addressing modes. :) 224036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!AddrMode.HasBaseReg) { 224136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = true; 224236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseReg = Addr; 224336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Still check for legality in case the target supports [imm] but not [i+r]. 224436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 224536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 224636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = false; 2247dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseReg = nullptr; 224836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 224936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 225036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the base register is already taken, see if we can do [r+r]. 225136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale == 0) { 225236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.Scale = 1; 225336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.ScaledReg = Addr; 225436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 225536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 225636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.Scale = 0; 2257dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.ScaledReg = nullptr; 225836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 225936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Couldn't match. 226036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 226136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 226236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 226336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 226436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsOperandAMemoryOperand - Check to see if all uses of OpVal by the specified 226536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// inline asm call are due to memory operands. If so, return true, otherwise 226636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// return false. 226736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal, 226836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI) { 226936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(ImmutableCallSite(CI)); 227036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 227136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 227236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 227336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Compute the constraint code and ConstraintType to use. 227436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI.ComputeConstraintToUse(OpInfo, SDValue()); 227536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 227636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this asm operand is our Value*, and if it isn't an indirect memory 227736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // operand, we can't fold it! 227836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OpInfo.CallOperandVal == OpVal && 227936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (OpInfo.ConstraintType != TargetLowering::C_Memory || 228036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !OpInfo.isIndirect)) 228136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 228236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 228336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 228436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 228536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 228636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 228736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// FindAllMemoryUses - Recursively walk all the uses of I until we find a 228836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// memory use. If we find an obviously non-foldable instruction, return true. 228936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Add the ultimately found memory instructions to MemoryUses. 229036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool FindAllMemoryUses(Instruction *I, 229136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<std::pair<Instruction*,unsigned> > &MemoryUses, 229236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<Instruction*, 16> &ConsideredInsts, 229336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI) { 229436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we already considered this instruction, we're done. 229536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!ConsideredInsts.insert(I)) 229636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 229736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 229836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is an obviously unfoldable instruction, bail out. 229936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MightBeFoldableInst(I)) 230036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 230136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 230236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Loop over all the uses, recursively processing them. 230336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Use &U : I->uses()) { 230436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UserI = cast<Instruction>(U.getUser()); 230536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 230636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) { 230736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryUses.push_back(std::make_pair(LI, U.getOperandNo())); 230836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 230936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 231036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 231136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) { 231236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned opNo = U.getOperandNo(); 231336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (opNo == 0) return true; // Storing addr, not into addr. 231436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryUses.push_back(std::make_pair(SI, opNo)); 231536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 231636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 231736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 231836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CallInst *CI = dyn_cast<CallInst>(UserI)) { 231936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue()); 232036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!IA) return true; 232136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 232236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is a memory operand, we're cool, otherwise bail out. 232336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!IsOperandAMemoryOperand(CI, IA, I, TLI)) 232436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 232536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 232636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 232736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 232836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TLI)) 232936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 233036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 233136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 233236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 233336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 233436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 233536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ValueAlreadyLiveAtInst - Retrn true if Val is already known to be live at 233636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the use site that we're folding it into. If so, there is no cost to 233736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// include it in the addressing mode. KnownLive1 and KnownLive2 are two values 233836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// that we know are live at the instruction already. 233936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::ValueAlreadyLiveAtInst(Value *Val,Value *KnownLive1, 234036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *KnownLive2) { 234136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If Val is either of the known-live values, we know it is live! 2342dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2) 234336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 234436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 234536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // All values other than instructions and arguments (e.g. constants) are live. 234636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<Instruction>(Val) && !isa<Argument>(Val)) return true; 234736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 234836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If Val is a constant sized alloca in the entry block, it is live, this is 234936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // true because it is just a reference to the stack/frame pointer, which is 235036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // live for the whole function. 235136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AllocaInst *AI = dyn_cast<AllocaInst>(Val)) 235236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AI->isStaticAlloca()) 235336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 235436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 235536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if this value is already used in the memory instruction's 235636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // block. If so, it's already live into the block at the very least, so we 235736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // can reasonably fold it. 235836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Val->isUsedInBasicBlock(MemoryInst->getParent()); 235936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 236036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 236136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsProfitableToFoldIntoAddressingMode - It is possible for the addressing 236236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// mode of the machine to fold the specified instruction into a load or store 236336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// that ultimately uses it. However, the specified instruction has multiple 236436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// uses. Given this, it may actually increase register pressure to fold it 236536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// into the load. For example, consider this code: 236636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 236736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// X = ... 236836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Y = X+1 236936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// use(Y) -> nonload/store 237036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Z = Y+1 237136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// load Z 237236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 237336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// In this case, Y has multiple uses, and can be folded into the load of Z 237436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (yielding load [X+2]). However, doing this will cause both "X" and "X+1" to 237536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// be live at the use(Y) line. If we don't fold Y into load Z, we use one 237636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// fewer register. Since Y can't be folded into "use(Y)" we don't increase the 237736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// number of computations either. 237836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 237936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Note that this (like most of CodeGenPrepare) is just a rough heuristic. If 238036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// X was live across 'load Z' for other reasons, we actually *would* want to 238136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// fold the addressing mode in the Z case. This would make Y die earlier. 238236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher:: 238336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesIsProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore, 238436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AMAfter) { 238536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IgnoreProfitability) return true; 238636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 238736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // AMBefore is the addressing mode before this instruction was folded into it, 238836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // and AMAfter is the addressing mode after the instruction was folded. Get 238936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the set of registers referenced by AMAfter and subtract out those 239036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // referenced by AMBefore: this is the set of values which folding in this 239136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // address extends the lifetime of. 239236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // 239336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Note that there are only two potential values being referenced here, 239436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // BaseReg and ScaleReg (global addresses are always available, as are any 239536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // folded immediates). 239636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg; 239736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 239836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the BaseReg or ScaledReg was referenced by the previous addrmode, their 239936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // lifetime wasn't extended by adding this instruction. 240036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ValueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg)) 2401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BaseReg = nullptr; 240236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ValueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg)) 2403dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ScaledReg = nullptr; 240436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 240536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If folding this instruction (and it's subexprs) didn't extend any live 240636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // ranges, we're ok with it. 2407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BaseReg && !ScaledReg) 240836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 240936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 241036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If all uses of this instruction are ultimately load/store/inlineasm's, 241136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // check to see if their addressing modes will include this instruction. If 241236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // so, we can fold it into all uses, so it doesn't matter if it has multiple 241336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // uses. 241436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<std::pair<Instruction*,unsigned>, 16> MemoryUses; 241536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<Instruction*, 16> ConsideredInsts; 241636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TLI)) 241736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; // Has a non-memory, non-foldable use! 241836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 241936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Now that we know that all uses of this instruction are part of a chain of 242036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // computation involving only operations that could theoretically be folded 242136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // into a memory use, loop over each of these uses and see if they could 242236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // *actually* fold the instruction. 242336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Instruction*, 32> MatchedAddrModeInsts; 242436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = MemoryUses.size(); i != e; ++i) { 242536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = MemoryUses[i].first; 242636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OpNo = MemoryUses[i].second; 242736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 242836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Get the access type of this use. If the use isn't a pointer, we don't 242936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // know what it accesses. 243036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Address = User->getOperand(OpNo); 243136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Address->getType()->isPointerTy()) 243236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 243336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *AddressAccessTy = Address->getType()->getPointerElementType(); 243436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 243536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do a match against the root of this address, ignoring profitability. This 243636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // will tell us if the addressing mode for the memory operation will 243736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // *actually* cover the shared instruction. 243836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode Result; 243936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 244036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 244136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddressingModeMatcher Matcher(MatchedAddrModeInsts, TLI, AddressAccessTy, 244236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryInst, Result, InsertedTruncs, 244336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts, TPT); 244436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matcher.IgnoreProfitability = true; 244536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Success = Matcher.MatchAddr(Address, 0); 244636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (void)Success; assert(Success && "Couldn't select *anything*?"); 244736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 244836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The match was to check the profitability, the changes made are not 244936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // part of the original matcher. Therefore, they should be dropped 245036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // otherwise the original matcher will not present the right state. 245136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 245236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 245336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the match didn't cover I, then it won't be shared by it. 245436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (std::find(MatchedAddrModeInsts.begin(), MatchedAddrModeInsts.end(), 245536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I) == MatchedAddrModeInsts.end()) 245636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 245736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 245836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MatchedAddrModeInsts.clear(); 245936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 246036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 246136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 246236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 246336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 246436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} // end anonymous namespace 246536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 246636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsNonLocalValue - Return true if the specified values are defined in a 246736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// different basic block than BB. 246836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool IsNonLocalValue(Value *V, BasicBlock *BB) { 246936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Instruction *I = dyn_cast<Instruction>(V)) 247036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return I->getParent() != BB; 247136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 247236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 247336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 247436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeMemoryInst - Load and Store Instructions often have 247536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// addressing modes that can do significant amounts of computation. As such, 247636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// instruction selection will try to get the load or store to do as much 247736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// computation as possible for the program. The problem is that isel can only 247836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// see within a single block. As such, we sink as much legal addressing mode 247936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// stuff into the block as possible. 248036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 248136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// This method is used to optimize both load/store and inline asms with memory 248236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// operands. 248336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr, 248436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *AccessTy) { 248536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Repl = Addr; 248636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 248736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Try to collapse single-value PHI nodes. This is necessary to undo 248836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // unprofitable PRE transformations. 248936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Value*, 8> worklist; 249036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<Value*, 16> Visited; 249136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines worklist.push_back(Addr); 249236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 249336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use a worklist to iteratively look through PHI nodes, and ensure that 249436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the addressing mode obtained from the non-PHI roots of the graph 249536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // are equivalent. 2496dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Consensus = nullptr; 249736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumUsesConsensus = 0; 249836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IsNumUsesConsensusValid = false; 249936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Instruction*, 16> AddrModeInsts; 250036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode AddrMode; 250136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction TPT; 250236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 250336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 250436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (!worklist.empty()) { 250536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = worklist.back(); 250636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines worklist.pop_back(); 250736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 250836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Break use-def graph loops. 250936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Visited.insert(V)) { 2510dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Consensus = nullptr; 251136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 251236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 251336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 251436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // For a PHI node, push all of its incoming values. 251536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *P = dyn_cast<PHINode>(V)) { 251636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = P->getNumIncomingValues(); i != e; ++i) 251736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines worklist.push_back(P->getIncomingValue(i)); 251836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 251936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 252036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 252136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // For non-PHIs, determine the addressing mode being computed. 252236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Instruction*, 16> NewAddrModeInsts; 252336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode NewAddrMode = AddressingModeMatcher::Match( 252436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V, AccessTy, MemoryInst, NewAddrModeInsts, *TLI, InsertedTruncsSet, 252536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts, TPT); 252636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 252736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This check is broken into two cases with very similar code to avoid using 252836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // getNumUses() as much as possible. Some values have a lot of uses, so 252936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // calling getNumUses() unconditionally caused a significant compile-time 253036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // regression. 253136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Consensus) { 253236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Consensus = V; 253336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = NewAddrMode; 253436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts = NewAddrModeInsts; 253536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 253636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (NewAddrMode == AddrMode) { 253736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!IsNumUsesConsensusValid) { 253836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NumUsesConsensus = Consensus->getNumUses(); 253936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IsNumUsesConsensusValid = true; 254036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 254136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 254236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Ensure that the obtained addressing mode is equivalent to that obtained 254336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // for all other roots of the PHI traversal. Also, when choosing one 254436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // such root as representative, select the one with the most uses in order 254536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to keep the cost modeling heuristics in AddressingModeMatcher 254636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // applicable. 254736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumUses = V->getNumUses(); 254836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NumUses > NumUsesConsensus) { 254936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Consensus = V; 255036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NumUsesConsensus = NumUses; 255136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts = NewAddrModeInsts; 255236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 255336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 255436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 255536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2556dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Consensus = nullptr; 255736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 255836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 255936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 256036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the addressing mode couldn't be determined, or if multiple different 256136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // ones were determined, bail out now. 256236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Consensus) { 256336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 256436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 256536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 256636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.commit(); 256736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 256836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if any of the instructions supersumed by this addr mode are 256936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // non-local to I's BB. 257036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool AnyNonLocal = false; 257136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = AddrModeInsts.size(); i != e; ++i) { 257236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IsNonLocalValue(AddrModeInsts[i], MemoryInst->getParent())) { 257336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AnyNonLocal = true; 257436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 257536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 257636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 257736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 257836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If all the instructions matched are already in this BB, don't do anything. 257936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!AnyNonLocal) { 258036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode << "\n"); 258136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 258236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 258336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 258436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Insert this computation right after this user. Since our caller is 258536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // scanning from the top of the BB to the bottom, reuse of the expr are 258636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // guaranteed to happen later. 258736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(MemoryInst); 258836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 258936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Now that we determined the addressing expression we want to use and know 259036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // that we have to sink it into this block. Check to see if we have already 259136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // done this for some other load/store instr in this block. If so, reuse the 259236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // computation. 259336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *&SunkAddr = SunkAddrs[Addr]; 259436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SunkAddr) { 259536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for " 2596dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines << *MemoryInst << "\n"); 259736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SunkAddr->getType() != Addr->getType()) 259836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType()); 2599dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (AddrSinkUsingGEPs || (!AddrSinkUsingGEPs.getNumOccurrences() && 2600dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TM && TM->getSubtarget<TargetSubtargetInfo>().useAA())) { 2601dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // By default, we use the GEP-based method when AA is used later. This 2602dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities. 2603dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for " 2604dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines << *MemoryInst << "\n"); 2605dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(Addr->getType()); 2606dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *ResultPtr = nullptr, *ResultIndex = nullptr; 2607dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2608dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // First, find the pointer. 2609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) { 2610dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = AddrMode.BaseReg; 2611dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseReg = nullptr; 2612dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2613dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2614dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) { 2615dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We can't add more than one pointer together, nor can we scale a 2616dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // pointer (both of which seem meaningless). 2617dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr || AddrMode.Scale != 1) 2618dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 2619dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2620dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = AddrMode.ScaledReg; 2621dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.Scale = 0; 2622dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2623dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2624dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseGV) { 2625dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr) 2626dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 2627dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2628dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = AddrMode.BaseGV; 2629dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2631dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the real base value actually came from an inttoptr, then the matcher 2632dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // will look through it and provide only the integer value. In that case, 2633dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // use it here. 2634dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ResultPtr && AddrMode.BaseReg) { 2635dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = 2636dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(), "sunkaddr"); 2637dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseReg = nullptr; 2638dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (!ResultPtr && AddrMode.Scale == 1) { 2639dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = 2640dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(), "sunkaddr"); 2641dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.Scale = 0; 2642dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2643dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2644dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ResultPtr && 2645dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) { 2646dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SunkAddr = Constant::getNullValue(Addr->getType()); 2647dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (!ResultPtr) { 2648dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 2649dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 2650dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *I8PtrTy = 2651dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace()); 2652dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2653dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Start with the base register. Do this first so that subsequent address 2654dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // matching finds it last, which will prevent it from trying to match it 2655dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // as the scaled value in case it happens to be a mul. That would be 2656dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // problematic if we've sunk a different mul for the scale, because then 2657dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we'd end up sinking both muls. 2658dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseReg) { 2659dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *V = AddrMode.BaseReg; 2660dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (V->getType() != IntPtrTy) 2661dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr"); 2662dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2663dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = V; 2664dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2665dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2666dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add the scale value. 2667dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.Scale) { 2668dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *V = AddrMode.ScaledReg; 2669dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (V->getType() == IntPtrTy) { 2670dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // done. 2671dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < 2672dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cast<IntegerType>(V->getType())->getBitWidth()) { 2673dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); 2674dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 2675dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // It is only safe to sign extend the BaseReg if we know that the math 2676dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // required to create it did not overflow before we extend it. Since 2677dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the original IR value was tossed in favor of a constant back when 2678dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the AddrMode was created we need to bail out gracefully if widths 2679dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // do not match instead of extending it. 2680dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *I = dyn_cast_or_null<Instruction>(ResultIndex); 2681dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (I && (ResultIndex != AddrMode.BaseReg)) 2682dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines I->eraseFromParent(); 2683dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 2684dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2685dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2686dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.Scale != 1) 2687dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale), 2688dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "sunkaddr"); 2689dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultIndex) 2690dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr"); 2691dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 2692dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = V; 2693dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2694dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2695dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add in the Base Offset if present. 2696dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseOffs) { 2697dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); 2698dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultIndex) { 2699dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We need to add this separately from the scale above to help with 2700dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // SDAG consecutive load/store merging. 2701dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr->getType() != I8PtrTy) 2702dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy); 2703dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = Builder.CreateGEP(ResultPtr, ResultIndex, "sunkaddr"); 2704dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2705dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2706dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = V; 2707dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2708dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2709dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ResultIndex) { 2710dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SunkAddr = ResultPtr; 2711dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 2712dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr->getType() != I8PtrTy) 2713dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy); 2714dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SunkAddr = Builder.CreateGEP(ResultPtr, ResultIndex, "sunkaddr"); 2715dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 2716dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2717dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SunkAddr->getType() != Addr->getType()) 2718dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType()); 2719dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 272036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 272136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for " 2722dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines << *MemoryInst << "\n"); 272336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(Addr->getType()); 2724dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Result = nullptr; 272536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 272636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Start with the base register. Do this first so that subsequent address 272736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // matching finds it last, which will prevent it from trying to match it 272836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // as the scaled value in case it happens to be a mul. That would be 272936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // problematic if we've sunk a different mul for the scale, because then 273036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // we'd end up sinking both muls. 273136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.BaseReg) { 273236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = AddrMode.BaseReg; 273336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V->getType()->isPointerTy()) 273436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr"); 273536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V->getType() != IntPtrTy) 273636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr"); 273736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 273836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 273936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 274036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add the scale value. 274136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale) { 274236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = AddrMode.ScaledReg; 274336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V->getType() == IntPtrTy) { 274436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // done. 274536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (V->getType()->isPointerTy()) { 274636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr"); 274736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < 274836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<IntegerType>(V->getType())->getBitWidth()) { 274936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); 275036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 275136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It is only safe to sign extend the BaseReg if we know that the math 275236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // required to create it did not overflow before we extend it. Since 275336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the original IR value was tossed in favor of a constant back when 275436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the AddrMode was created we need to bail out gracefully if widths 275536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // do not match instead of extending it. 2756dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *I = dyn_cast_or_null<Instruction>(Result); 2757dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (I && (Result != AddrMode.BaseReg)) 2758dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines I->eraseFromParent(); 275936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 276036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 276136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale != 1) 276236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale), 276336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "sunkaddr"); 276436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Result) 276536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = Builder.CreateAdd(Result, V, "sunkaddr"); 276636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 276736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 276836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 276936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 277036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add in the BaseGV if present. 277136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.BaseGV) { 277236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr"); 277336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Result) 277436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = Builder.CreateAdd(Result, V, "sunkaddr"); 277536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 277636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 277736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 277836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 277936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add in the Base Offset if present. 278036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.BaseOffs) { 278136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); 278236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Result) 278336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = Builder.CreateAdd(Result, V, "sunkaddr"); 278436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 278536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 278636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 278736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2788dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Result) 278936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddr = Constant::getNullValue(Addr->getType()); 279036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 279136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr"); 279236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 279336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 279436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryInst->replaceUsesOfWith(Repl, SunkAddr); 279536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 279636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we have no uses, recursively delete the value and all dead instructions 279736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // using it. 279836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Repl->use_empty()) { 279936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This can cause recursive deletion, which can invalidate our iterator. 280036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use a WeakVH to hold onto it in case this happens. 280136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WeakVH IterHandle(CurInstIterator); 280236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = CurInstIterator->getParent(); 280336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 280436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines RecursivelyDeleteTriviallyDeadInstructions(Repl, TLInfo); 280536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 280636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IterHandle != CurInstIterator) { 280736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the iterator instruction was recursively deleted, start over at the 280836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // start of the block. 280936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = BB->begin(); 281036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 281136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 281236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 281336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumMemoryInsts; 281436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 281536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 281636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 281736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeInlineAsmInst - If there are any memory operands, use 281836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeMemoryInst to sink their address computing into the block when 281936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// possible / profitable. 282036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeInlineAsmInst(CallInst *CS) { 282136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 282236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 282336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::AsmOperandInfoVector 282436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetConstraints = TLI->ParseConstraints(CS); 282536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned ArgNo = 0; 282636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 282736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 282836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 282936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Compute the constraint code and ConstraintType to use. 283036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->ComputeConstraintToUse(OpInfo, SDValue()); 283136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 283236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OpInfo.ConstraintType == TargetLowering::C_Memory && 283336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OpInfo.isIndirect) { 283436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *OpVal = CS->getArgOperand(ArgNo++); 283536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= OptimizeMemoryInst(CS, OpVal, OpVal->getType()); 283636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (OpInfo.Type == InlineAsm::isInput) 283736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ArgNo++; 283836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 283936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 284036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 284136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 284236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 284336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MoveExtToFormExtLoad - Move a zext or sext fed by a load into the same 284436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// basic block as the load, unless conditions are unfavorable. This allows 284536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// SelectionDAG to fold the extend into the load. 284636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 284736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::MoveExtToFormExtLoad(Instruction *I) { 284836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Look for a load being extended. 284936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LoadInst *LI = dyn_cast<LoadInst>(I->getOperand(0)); 285036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!LI) return false; 285136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 285236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If they're already in the same block, there's nothing to do. 285336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (LI->getParent() == I->getParent()) 285436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 285536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 285636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the load has other users and the truncate is not free, this probably 285736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // isn't worthwhile. 285836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!LI->hasOneUse() && 285936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI && (TLI->isTypeLegal(TLI->getValueType(LI->getType())) || 286036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !TLI->isTypeLegal(TLI->getValueType(I->getType()))) && 286136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !TLI->isTruncateFree(I->getType(), LI->getType())) 286236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 286336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 286436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check whether the target supports casts folded into loads. 286536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned LType; 286636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<ZExtInst>(I)) 286736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LType = ISD::ZEXTLOAD; 286836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else { 286936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(isa<SExtInst>(I) && "Unexpected ext type!"); 287036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LType = ISD::SEXTLOAD; 287136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 287236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && !TLI->isLoadExtLegal(LType, TLI->getValueType(LI->getType()))) 287336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 287436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 287536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Move the extend into the same block as the load, so that SelectionDAG 287636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // can fold it. 287736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I->removeFromParent(); 287836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I->insertAfter(LI); 287936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumExtsMoved; 288036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 288136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 288236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 288336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeExtUses(Instruction *I) { 288436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = I->getParent(); 288536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 288636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the result of a {s|z}ext and its source are both live out, rewrite all 288736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // other uses of the source with result of extension. 288836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Src = I->getOperand(0); 288936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Src->hasOneUse()) 289036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 289136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 289236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Only do this xform if truncating is free. 289336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && !TLI->isTruncateFree(I->getType(), Src->getType())) 289436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 289536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 289636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Only safe to perform the optimization if the source is also defined in 289736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // this block. 289836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent()) 289936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 290036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 290136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool DefIsLiveOut = false; 290236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (User *U : I->users()) { 290336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UI = cast<Instruction>(U); 290436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 290536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this ext is used in. 290636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = UI->getParent(); 290736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 290836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DefIsLiveOut = true; 290936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 291036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 291136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DefIsLiveOut) 291236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 291336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 291436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure none of the uses are PHI nodes. 291536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (User *U : Src->users()) { 291636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UI = cast<Instruction>(U); 291736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = UI->getParent(); 291836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 291936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Be conservative. We don't want this xform to end up introducing 292036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // reloads just before load / store instructions. 292136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI)) 292236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 292336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 292436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 292536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // InsertedTruncs - Only insert one trunc in each block once. 292636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, Instruction*> InsertedTruncs; 292736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 292836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 292936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Use &U : Src->uses()) { 293036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = cast<Instruction>(U.getUser()); 293136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 293236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this ext is used in. 293336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = User->getParent(); 293436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 293536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 293636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Both src and def are live in this block. Rewrite the use. 293736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *&InsertedTrunc = InsertedTruncs[UserBB]; 293836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 293936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedTrunc) { 294036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 294136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTrunc = new TruncInst(I, Src->getType(), "", InsertPt); 294236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTruncsSet.insert(InsertedTrunc); 294336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 294436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 294536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Replace a use of the {s|z}ext source with a use of the result. 294636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines U = InsertedTrunc; 294736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumExtUses; 294836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 294936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 295036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 295136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 295236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 295336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 295436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// isFormingBranchFromSelectProfitable - Returns true if a SelectInst should be 295536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// turned into an explicit branch. 295636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool isFormingBranchFromSelectProfitable(SelectInst *SI) { 295736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // FIXME: This should use the same heuristics as IfConversion to determine 295836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // whether a select is better represented as a branch. This requires that 295936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // branch probability metadata is preserved for the select, which is not the 296036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // case currently. 296136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 296236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition()); 296336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 296436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the branch is predicted right, an out of order CPU can avoid blocking on 296536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the compare. Emit cmovs on compares with a memory operand as branches to 296636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // avoid stalls on the load from memory. If the compare has more than one use 296736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // there's probably another cmov or setcc around so it's not worth emitting a 296836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // branch. 296936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Cmp) 297036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 297136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 297236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *CmpOp0 = Cmp->getOperand(0); 297336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *CmpOp1 = Cmp->getOperand(1); 297436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 297536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We check that the memory operand has one use to avoid uses of the loaded 297636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // value directly after the compare, making branches unprofitable. 297736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Cmp->hasOneUse() && 297836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ((isa<LoadInst>(CmpOp0) && CmpOp0->hasOneUse()) || 297936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (isa<LoadInst>(CmpOp1) && CmpOp1->hasOneUse())); 298036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 298136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 298236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 298336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// If we have a SelectInst that will likely profit from branch prediction, 298436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// turn it into a branch. 298536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeSelectInst(SelectInst *SI) { 298636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1); 298736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 298836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Can we convert the 'select' to CF ? 298936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (DisableSelectToBranch || OptSize || !TLI || VectorCond) 299036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 299136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 299236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::SelectSupportKind SelectKind; 299336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VectorCond) 299436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectKind = TargetLowering::VectorMaskSelect; 299536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else if (SI->getType()->isVectorTy()) 299636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectKind = TargetLowering::ScalarCondVectorVal; 299736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 299836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectKind = TargetLowering::ScalarValSelect; 299936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 300036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do we have efficient codegen support for this kind of 'selects' ? 300136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI->isSelectSupported(SelectKind)) { 300236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We have efficient codegen support for the select instruction. 300336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if it is profitable to keep this 'select'. 300436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI->isPredictableSelectExpensive() || 300536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !isFormingBranchFromSelectProfitable(SI)) 300636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 300736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 300836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 300936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ModifiedDT = true; 301036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 301136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // First, we split the block containing the select into 2 blocks. 301236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *StartBlock = SI->getParent(); 301336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(SI)); 301436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *NextBlock = StartBlock->splitBasicBlock(SplitPt, "select.end"); 301536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 301636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Create a new block serving as the landing pad for the branch. 301736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *SmallBlock = BasicBlock::Create(SI->getContext(), "select.mid", 301836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NextBlock->getParent(), NextBlock); 301936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 302036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Move the unconditional branch from the block with the select in it into our 302136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // landing pad block. 302236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines StartBlock->getTerminator()->eraseFromParent(); 302336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst::Create(NextBlock, SmallBlock); 302436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 302536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Insert the real conditional branch based on the original condition. 302636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst::Create(NextBlock, SmallBlock, SI->getCondition(), SI); 302736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 302836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The select itself is replaced with a PHI Node. 302936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PHINode *PN = PHINode::Create(SI->getType(), 2, "", NextBlock->begin()); 303036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->takeName(SI); 303136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(SI->getTrueValue(), StartBlock); 303236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(SI->getFalseValue(), SmallBlock); 303336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI->replaceAllUsesWith(PN); 303436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI->eraseFromParent(); 303536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 303636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Instruct OptimizeBlock to skip to the next block. 303736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = StartBlock->end(); 303836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumSelectsExpanded; 303936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 304036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 304136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 304236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool isBroadcastShuffle(ShuffleVectorInst *SVI) { 304336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<int, 16> Mask(SVI->getShuffleMask()); 304436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int SplatElem = -1; 304536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0; i < Mask.size(); ++i) { 304636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem) 304736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 304836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SplatElem = Mask[i]; 304936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 305036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 305136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 305236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 305336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 305436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Some targets have expensive vector shifts if the lanes aren't all the same 305536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (e.g. x86 only introduced "vpsllvd" and friends with AVX2). In these cases 305636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// it's often worth sinking a shufflevector splat down to its use so that 305736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// codegen can spot all lanes are identical. 305836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeShuffleVectorInst(ShuffleVectorInst *SVI) { 305936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = SVI->getParent(); 306036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 306136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Only do this xform if variable vector shifts are particularly expensive. 306236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI || !TLI->isVectorShiftByScalarCheap(SVI->getType())) 306336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 306436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 306536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We only expect better codegen by sinking a shuffle if we can recognise a 306636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // constant splat. 306736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isBroadcastShuffle(SVI)) 306836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 306936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 307036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // InsertedShuffles - Only insert a shuffle in each block once. 307136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, Instruction*> InsertedShuffles; 307236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 307336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 307436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (User *U : SVI->users()) { 307536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UI = cast<Instruction>(U); 307636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 307736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this ext is used in. 307836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = UI->getParent(); 307936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 308036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 308136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // For now only apply this when the splat is used by a shift instruction. 308236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!UI->isShift()) continue; 308336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 308436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Everything checks out, sink the shuffle if the user's block doesn't 308536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // already have a copy. 308636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *&InsertedShuffle = InsertedShuffles[UserBB]; 308736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 308836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedShuffle) { 308936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 309036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedShuffle = new ShuffleVectorInst(SVI->getOperand(0), 309136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SVI->getOperand(1), 309236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SVI->getOperand(2), "", InsertPt); 309336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 309436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 309536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI->replaceUsesOfWith(SVI, InsertedShuffle); 309636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 309736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 309836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 309936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we removed all uses, nuke the shuffle. 310036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SVI->use_empty()) { 310136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SVI->eraseFromParent(); 310236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 310336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 310436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 310536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 310636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 310736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 310836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeInst(Instruction *I) { 310936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *P = dyn_cast<PHINode>(I)) { 311036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It is possible for very late stage optimizations (such as SimplifyCFG) 311136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to introduce PHI nodes too late to be cleaned up. If we detect such a 311236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // trivial PHI, go ahead and zap it here. 3113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Value *V = SimplifyInstruction(P, TLI ? TLI->getDataLayout() : nullptr, 311436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLInfo, DT)) { 311536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines P->replaceAllUsesWith(V); 311636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines P->eraseFromParent(); 311736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumPHIsElim; 311836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 311936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 312036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 312136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 312236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 312336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CastInst *CI = dyn_cast<CastInst>(I)) { 312436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the source of the cast is a constant, then this should have 312536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // already been constant folded. The only reason NOT to constant fold 312636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // it is if something (e.g. LSR) was careful to place the constant 312736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // evaluation in a block other than then one that uses it (e.g. to hoist 312836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the address of globals out of a loop). If this is the case, we don't 312936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // want to forward-subst the cast. 313036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<Constant>(CI->getOperand(0))) 313136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 313236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 313336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && OptimizeNoopCopyExpression(CI, *TLI)) 313436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 313536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 313636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<ZExtInst>(I) || isa<SExtInst>(I)) { 313736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Sink a zext or sext into its user blocks if the target type doesn't 313836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// fit in one register 313936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && TLI->getTypeAction(CI->getContext(), 314036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->getValueType(CI->getType())) == 314136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::TypeExpandInteger) { 314236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SinkCast(CI); 314336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 314436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = MoveExtToFormExtLoad(I); 314536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange | OptimizeExtUses(I); 314636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 314736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 314836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 314936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 315036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 315136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CmpInst *CI = dyn_cast<CmpInst>(I)) 315236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI || !TLI->hasMultipleConditionRegisters()) 315336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeCmpExpression(CI); 315436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 315536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 315636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI) 315736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeMemoryInst(I, I->getOperand(0), LI->getType()); 315836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 315936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 316036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 316136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 316236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI) 316336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeMemoryInst(I, SI->getOperand(1), 316436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI->getOperand(0)->getType()); 316536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 316636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 316736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3168dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I); 3169dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (BinOp && (BinOp->getOpcode() == Instruction::AShr || 3171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinOp->getOpcode() == Instruction::LShr)) { 3172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)); 3173dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TLI && CI && TLI->hasExtractBitsInsn()) 3174dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return OptimizeExtractBits(BinOp, CI, *TLI); 3175dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3176dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 3177dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 317936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) { 318036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (GEPI->hasAllZeroIndices()) { 318136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The GEP operand must be a pointer, so must its result -> BitCast 318236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(), 318336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GEPI->getName(), GEPI); 318436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GEPI->replaceAllUsesWith(NC); 318536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GEPI->eraseFromParent(); 318636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumGEPsElim; 318736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OptimizeInst(NC); 318836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 318936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 319036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 319136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 319236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 319336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CallInst *CI = dyn_cast<CallInst>(I)) 319436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeCallInst(CI); 319536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 319636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SelectInst *SI = dyn_cast<SelectInst>(I)) 319736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeSelectInst(SI); 319836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 319936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I)) 320036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeShuffleVectorInst(SVI); 320136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 320236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 320336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 320436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 320536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// In this pass we look for GEP and cast instructions that are used 320636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// across basic blocks and rewrite them to improve basic-block-at-a-time 320736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// selection. 320836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeBlock(BasicBlock &BB) { 320936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 321036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 321136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 321236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = BB.begin(); 321336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (CurInstIterator != BB.end()) 321436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= OptimizeInst(CurInstIterator++); 321536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 321636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= DupRetToEnableTailCallOpts(&BB); 321736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 321836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 321936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 322036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 322136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// llvm.dbg.value is far away from the value then iSel may not be able 322236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// handle it properly. iSel will drop llvm.dbg.value if it can not 322336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// find a node corresponding to the value. 322436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::PlaceDbgValues(Function &F) { 322536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 322636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) { 3227dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *PrevNonDbgInst = nullptr; 322836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (BasicBlock::iterator BI = I->begin(), BE = I->end(); BI != BE;) { 322936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Insn = BI; ++BI; 323036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DbgValueInst *DVI = dyn_cast<DbgValueInst>(Insn); 3231dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Leave dbg.values that refer to an alloca alone. These 3232dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instrinsics describe the address of a variable (= the alloca) 3233dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // being taken. They should not be moved next to the alloca 3234dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (and to the beginning of the scope), but rather stay close to 3235dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // where said address is used. 3236dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DVI || (DVI->getValue() && isa<AllocaInst>(DVI->getValue()))) { 323736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PrevNonDbgInst = Insn; 323836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 323936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 324036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 324136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue()); 324236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) { 324336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Moving Debug Value before :\n" << *DVI << ' ' << *VI); 324436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DVI->removeFromParent(); 324536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<PHINode>(VI)) 324636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DVI->insertBefore(VI->getParent()->getFirstInsertionPt()); 324736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 324836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DVI->insertAfter(VI); 324936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 325036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumDbgValueMoved; 325136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 325236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 325336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 325436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 325536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 325636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 325736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// If there is a sequence that branches based on comparing a single bit 325836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// against zero that can be combined into a single instruction, and the 325936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// target supports folding these into a single instruction, sink the 326036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// mask and compare into the branch uses. Do this before OptimizeBlock -> 326136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// OptimizeInst -> OptimizeCmpExpression, which perturbs the pattern being 326236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// searched for. 326336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::sinkAndCmp(Function &F) { 326436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!EnableAndCmpSinking) 326536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 326636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI || !TLI->isMaskAndBranchFoldingLegal()) 326736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 326836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 326936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = F.begin(), E = F.end(); I != E; ) { 327036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 327136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 327236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Does this BB end with the following? 327336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // %andVal = and %val, #single-bit-set 327436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // %icmpVal = icmp %andResult, 0 327536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // br i1 %cmpVal label %dest1, label %dest2" 327636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *Brcc = dyn_cast<BranchInst>(BB->getTerminator()); 327736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Brcc || !Brcc->isConditional()) 327836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 327936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ICmpInst *Cmp = dyn_cast<ICmpInst>(Brcc->getOperand(0)); 328036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Cmp || Cmp->getParent() != BB) 328136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 328236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt *Zero = dyn_cast<ConstantInt>(Cmp->getOperand(1)); 328336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Zero || !Zero->isZero()) 328436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 328536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *And = dyn_cast<Instruction>(Cmp->getOperand(0)); 328636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!And || And->getOpcode() != Instruction::And || And->getParent() != BB) 328736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 328836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt* Mask = dyn_cast<ConstantInt>(And->getOperand(1)); 328936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Mask || !Mask->getUniqueInteger().isPowerOf2()) 329036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 329136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "found and; icmp ?,0; brcc\n"); DEBUG(BB->dump()); 329236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 329336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Push the "and; icmp" for any users that are conditional branches. 329436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Since there can only be one branch use per BB, we don't need to keep 329536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // track of which BBs we insert into. 329636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Value::use_iterator UI = Cmp->use_begin(), E = Cmp->use_end(); 329736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI != E; ) { 329836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Use &TheUse = *UI; 329936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Find brcc use. 330036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BrccUser = dyn_cast<BranchInst>(*UI); 330136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++UI; 330236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!BrccUser || !BrccUser->isConditional()) 330336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 330436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = BrccUser->getParent(); 330536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == BB) continue; 330636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "found Brcc use\n"); 330736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 330836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Sink the "and; icmp" to use. 330936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 331036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BinaryOperator *NewAnd = 331136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BinaryOperator::CreateAnd(And->getOperand(0), And->getOperand(1), "", 331236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BrccUser); 331336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst *NewCmp = 331436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(), NewAnd, Zero, 331536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "", BrccUser); 331636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TheUse = NewCmp; 331736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumAndCmpsMoved; 331836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(BrccUser->getParent()->dump()); 331936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 332036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 332136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 332236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 3323