136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===- CodeGenPrepare.cpp - Prepare a function for code generation --------===// 236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// The LLVM Compiler Infrastructure 436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This file is distributed under the University of Illinois Open Source 636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// License. See LICENSE.TXT for details. 736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 1036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// This pass munges the code in the input function to better prepare it for 1136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// SelectionDAG-based code generation. This works around limitations in it's 1236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// basic-block-at-a-time approach. It should eventually be removed. 1336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// 1436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 1536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/CodeGen/Passes.h" 1736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/ADT/DenseMap.h" 1836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/ADT/SmallSet.h" 1936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/ADT/Statistic.h" 2036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Analysis/InstructionSimplify.h" 21ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines#include "llvm/Analysis/TargetLibraryInfo.h" 2237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "llvm/Analysis/TargetTransformInfo.h" 2336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/CallSite.h" 2436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Constants.h" 2536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/DataLayout.h" 2636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/DerivedTypes.h" 2736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Dominators.h" 2836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Function.h" 2936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/GetElementPtrTypeIterator.h" 3036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/IRBuilder.h" 3136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/InlineAsm.h" 3236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/Instructions.h" 3336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/IntrinsicInst.h" 34ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines#include "llvm/IR/MDBuilder.h" 3536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/PatternMatch.h" 36ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines#include "llvm/IR/Statepoint.h" 3736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/ValueHandle.h" 3836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/IR/ValueMap.h" 3936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Pass.h" 4036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Support/CommandLine.h" 4136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Support/Debug.h" 4236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Support/raw_ostream.h" 4336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Target/TargetLowering.h" 44dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Target/TargetSubtargetInfo.h" 4536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/BasicBlockUtils.h" 4636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/BuildLibCalls.h" 4736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/BypassSlowDivision.h" 4836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#include "llvm/Transforms/Utils/Local.h" 49ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines#include "llvm/Transforms/Utils/SimplifyLibCalls.h" 5036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesusing namespace llvm; 5136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesusing namespace llvm::PatternMatch; 5236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 53dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "codegenprepare" 54dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumBlocksElim, "Number of blocks eliminated"); 5636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumPHIsElim, "Number of trivial PHIs eliminated"); 5736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumGEPsElim, "Number of GEPs converted to casts"); 5836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of " 5936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "sunken Cmps"); 6036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses " 6136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "of sunken Casts"); 6236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumMemoryInsts, "Number of memory instructions whose address " 6336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "computations were sunk"); 6436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumExtsMoved, "Number of [s|z]ext instructions combined with loads"); 6536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumExtUses, "Number of uses of [s|z]ext instructions optimized"); 6636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumRetsDup, "Number of return instructions duplicated"); 6736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumDbgValueMoved, "Number of debug value instructions moved"); 6836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumSelectsExpanded, "Number of selects turned into branches"); 6936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesSTATISTIC(NumAndCmpsMoved, "Number of and/cmp's pushed into branches"); 7037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesSTATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed"); 7136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 7236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<bool> DisableBranchOpts( 7336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "disable-cgp-branch-opts", cl::Hidden, cl::init(false), 7436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Disable branch optimizations in CodeGenPrepare")); 7536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 76ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic cl::opt<bool> 77ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false), 78ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines cl::desc("Disable GC optimizations in CodeGenPrepare")); 79ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 8036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<bool> DisableSelectToBranch( 8136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "disable-cgp-select2branch", cl::Hidden, cl::init(false), 8236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Disable select to branch conversion.")); 8336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 84dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic cl::opt<bool> AddrSinkUsingGEPs( 85dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "addr-sink-using-gep", cl::Hidden, cl::init(false), 86dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cl::desc("Address sinking in CGP using GEPs.")); 87dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic cl::opt<bool> EnableAndCmpSinking( 8936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "enable-andcmp-sinking", cl::Hidden, cl::init(true), 9036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cl::desc("Enable sinkinig and/cmp into branches.")); 9136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 9237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic cl::opt<bool> DisableStoreExtract( 9337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "disable-cgp-store-extract", cl::Hidden, cl::init(false), 9437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines cl::desc("Disable store(extract) optimizations in CodeGenPrepare")); 9537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 9637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstatic cl::opt<bool> StressStoreExtract( 9737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "stress-cgp-store-extract", cl::Hidden, cl::init(false), 9837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines cl::desc("Stress test store(extract) optimizations in CodeGenPrepare")); 9937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 100ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic cl::opt<bool> DisableExtLdPromotion( 101ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false), 102ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in " 103ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "CodeGenPrepare")); 104ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 105ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic cl::opt<bool> StressExtLdPromotion( 106ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false), 107ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) " 108ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "optimization in CodeGenPrepare")); 109ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 11036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesnamespace { 11136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinestypedef SmallPtrSet<Instruction *, 16> SetOfInstrs; 11237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesstruct TypeIsSExt { 11337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *Ty; 11437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsSExt; 11537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TypeIsSExt(Type *Ty, bool IsSExt) : Ty(Ty), IsSExt(IsSExt) {} 11637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines}; 11737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinestypedef DenseMap<Instruction *, TypeIsSExt> InstrToOrigTy; 118ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesclass TypePromotionTransaction; 11936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class CodeGenPrepare : public FunctionPass { 12136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// TLI - Keep a pointer of a TargetLowering to consult for determining 12236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// transformation profitability. 12336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetMachine *TM; 12436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering *TLI; 12537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const TargetTransformInfo *TTI; 12636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLibraryInfo *TLInfo; 12736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// CurInstIterator - As we scan instructions optimizing them, this is the 12936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// next instruction to optimize. Xforms that can invalidate this should 13036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// update it. 13136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator CurInstIterator; 13236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 13336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keeps track of non-local addresses that have been sunk into a block. 13436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// This allows us to avoid inserting duplicate code for blocks with 13536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// multiple load/stores of the same address. 13636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ValueMap<Value*, Value*> SunkAddrs; 13736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 13836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keeps track of all truncates inserted for the current function. 13936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SetOfInstrs InsertedTruncsSet; 14036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keeps track of the type of the related instruction before their 14136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// promotion for the current function. 14236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy PromotedInsts; 14336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1444c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar /// ModifiedDT - If CFG is modified in anyway. 14536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool ModifiedDT; 14636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 14736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// OptSize - True if optimizing for size. 14836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptSize; 14936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 15036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 15136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static char ID; // Pass identification, replacement for typeid 152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines explicit CodeGenPrepare(const TargetMachine *TM = nullptr) 15337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : FunctionPass(ID), TM(TM), TLI(nullptr), TTI(nullptr) { 15436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines initializeCodeGenPreparePass(*PassRegistry::getPassRegistry()); 15536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 15636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool runOnFunction(Function &F) override; 15736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 15836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const char *getPassName() const override { return "CodeGen Prepare"; } 15936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 16036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void getAnalysisUsage(AnalysisUsage &AU) const override { 16136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AU.addPreserved<DominatorTreeWrapperPass>(); 162ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines AU.addRequired<TargetLibraryInfoWrapperPass>(); 163ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines AU.addRequired<TargetTransformInfoWrapperPass>(); 16436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 16536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 16636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines private: 16736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool EliminateFallThrough(Function &F); 16836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool EliminateMostlyEmptyBlocks(Function &F); 16936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool CanMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const; 17036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void EliminateMostlyEmptyBlock(BasicBlock *BB); 171ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool OptimizeBlock(BasicBlock &BB, bool& ModifiedDT); 172ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool OptimizeInst(Instruction *I, bool& ModifiedDT); 17336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeMemoryInst(Instruction *I, Value *Addr, Type *AccessTy); 17436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeInlineAsmInst(CallInst *CS); 175ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool OptimizeCallInst(CallInst *CI, bool& ModifiedDT); 176ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool MoveExtToFormExtLoad(Instruction *&I); 17736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeExtUses(Instruction *I); 17836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeSelectInst(SelectInst *SI); 17936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool OptimizeShuffleVectorInst(ShuffleVectorInst *SI); 18037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool OptimizeExtractElementInst(Instruction *Inst); 18136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool DupRetToEnableTailCallOpts(BasicBlock *BB); 18236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool PlaceDbgValues(Function &F); 18336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool sinkAndCmp(Function &F); 184ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool ExtLdPromotion(TypePromotionTransaction &TPT, LoadInst *&LI, 185ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *&Inst, 186ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const SmallVectorImpl<Instruction *> &Exts, 1874c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned CreatedInstCost); 188ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool splitBranchCondition(Function &F); 189ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool simplifyOffsetableRelocate(Instruction &I); 19036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 19136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 19236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 19336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hineschar CodeGenPrepare::ID = 0; 194c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesINITIALIZE_TM_PASS(CodeGenPrepare, "codegenprepare", 195c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines "Optimize for code generation", false, false) 19636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 19736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesFunctionPass *llvm::createCodeGenPreparePass(const TargetMachine *TM) { 19836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return new CodeGenPrepare(TM); 19936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 20036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 20136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::runOnFunction(Function &F) { 20236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (skipOptnoneFunction(F)) 20336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 20436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 20536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool EverMadeChange = false; 20636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Clear per function information. 20736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTruncsSet.clear(); 20836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts.clear(); 20936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 21036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ModifiedDT = false; 21137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (TM) 212ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TLI = TM->getSubtargetImpl(F)->getTargetLowering(); 213ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); 214ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 215ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OptSize = F.hasFnAttribute(Attribute::OptimizeForSize); 21636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 21736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// This optimization identifies DIV instructions that can be 21836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// profitably bypassed and carried out with a shorter, faster divide. 21936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!OptSize && TLI && TLI->isSlowDivBypassed()) { 22036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const DenseMap<unsigned int, unsigned int> &BypassWidths = 22136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->getBypassSlowDivWidths(); 22236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = F.begin(); I != F.end(); I++) 22336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= bypassSlowDivision(F, I, BypassWidths); 22436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 22536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 22636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Eliminate blocks that contain only PHI nodes and an 22736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // unconditional branch. 22836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= EliminateMostlyEmptyBlocks(F); 22936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // llvm.dbg.value is far away from the value then iSel may not be able 23136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // handle it properly. iSel will drop llvm.dbg.value if it can not 23236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // find a node corresponding to the value. 23336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= PlaceDbgValues(F); 23436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 23536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If there is a mask, compare against zero, and branch that can be combined 23636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // into a single target instruction, push the mask and compare into branch 23736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // users. Do this before OptimizeBlock -> OptimizeInst -> 23836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // OptimizeCmpExpression, which perturbs the pattern being searched for. 239ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!DisableBranchOpts) { 24036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= sinkAndCmp(F); 241ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EverMadeChange |= splitBranchCondition(F); 242ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 24336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 24436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = true; 24536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (MadeChange) { 24636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = false; 24736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = F.begin(); I != F.end(); ) { 24836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 249ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool ModifiedDTOnIteration = false; 250ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MadeChange |= OptimizeBlock(*BB, ModifiedDTOnIteration); 251ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 252ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Restart BB iteration if the dominator tree of the Function was changed 253ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (ModifiedDTOnIteration) 254ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines break; 25536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 25636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= MadeChange; 25736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 25836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 25936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 26036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 26136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DisableBranchOpts) { 26236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = false; 26336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<BasicBlock*, 8> WorkList; 264ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (BasicBlock &BB : F) { 265ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<BasicBlock *, 2> Successors(succ_begin(&BB), succ_end(&BB)); 266ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MadeChange |= ConstantFoldTerminator(&BB, true); 26736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MadeChange) continue; 26836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 26936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallVectorImpl<BasicBlock*>::iterator 27036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines II = Successors.begin(), IE = Successors.end(); II != IE; ++II) 27136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (pred_begin(*II) == pred_end(*II)) 27236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WorkList.insert(*II); 27336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 27436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 27536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Delete the dead blocks and any of their dead successors. 27636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= !WorkList.empty(); 27736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (!WorkList.empty()) { 27836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = *WorkList.begin(); 27936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WorkList.erase(BB); 28036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<BasicBlock*, 2> Successors(succ_begin(BB), succ_end(BB)); 28136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 28236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DeleteDeadBlock(BB); 28336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 28436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (SmallVectorImpl<BasicBlock*>::iterator 28536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines II = Successors.begin(), IE = Successors.end(); II != IE; ++II) 28636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (pred_begin(*II) == pred_end(*II)) 28736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WorkList.insert(*II); 28836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 28936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 29036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Merge pairs of basic blocks with unconditional branches, connected by 29136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // a single edge. 29236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (EverMadeChange || MadeChange) 29336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= EliminateFallThrough(F); 29436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 29536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EverMadeChange |= MadeChange; 29636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 29736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 298ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!DisableGCOpts) { 299ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<Instruction *, 2> Statepoints; 300ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (BasicBlock &BB : F) 301ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (Instruction &I : BB) 302ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (isStatepoint(I)) 303ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Statepoints.push_back(&I); 304ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &I : Statepoints) 305ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EverMadeChange |= simplifyOffsetableRelocate(*I); 306ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 307ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 30836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return EverMadeChange; 30936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 31036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 31136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// EliminateFallThrough - Merge basic blocks which are connected 31236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// by a single edge, where one of the basic blocks has a single successor 31336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// pointing to the other basic block, which has a single predecessor. 31436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::EliminateFallThrough(Function &F) { 31536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Changed = false; 31636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Scan all of the blocks in the function, except for the entry block. 31736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) { 31836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 31936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the destination block has a single pred, then this is a trivial 32036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // edge, just collapse it. 32136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *SinglePred = BB->getSinglePredecessor(); 32236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 32336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't merge if BB's address is taken. 32436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue; 32536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 32636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator()); 32736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Term && !Term->isConditional()) { 32836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Changed = true; 32936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "To merge:\n"<< *SinglePred << "\n\n\n"); 33036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember if SinglePred was the entry block of the function. 33136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If so, we will need to move BB back to the entry position. 33236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); 3334c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MergeBasicBlockIntoOnlyPred(BB, nullptr); 33436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 33536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isEntry && BB != &BB->getParent()->getEntryBlock()) 33636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->moveBefore(&BB->getParent()->getEntryBlock()); 33736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 33836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We have erased a block. Update the iterator. 33936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I = BB; 34036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 34136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 34236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Changed; 34336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 34436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 34536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// EliminateMostlyEmptyBlocks - eliminate blocks that contain only PHI nodes, 34636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// debug info directives, and an unconditional branch. Passes before isel 34736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (e.g. LSR/loopsimplify) often split edges in ways that are non-optimal for 34836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// isel. Start by eliminating these blocks so we can split them the way we 34936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// want them. 35036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::EliminateMostlyEmptyBlocks(Function &F) { 35136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 35236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Note that this intentionally skips the entry block. 35336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) { 35436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 35536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 35636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this block doesn't end with an uncond branch, ignore it. 35736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); 35836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!BI || !BI->isUnconditional()) 35936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 36036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 36136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the instruction before the branch (skipping debug info) isn't a phi 36236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // node, then other stuff is happening here. 36336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator BBI = BI; 36436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BBI != BB->begin()) { 36536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines --BBI; 36636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (isa<DbgInfoIntrinsic>(BBI)) { 36736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BBI == BB->begin()) 36836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 36936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines --BBI; 37036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 37136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI)) 37236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 37336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 37436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 37536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do not break infinite loops. 37636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DestBB = BI->getSuccessor(0); 37736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (DestBB == BB) 37836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 37936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 38036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!CanMergeBlocks(BB, DestBB)) 38136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 38236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 38336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EliminateMostlyEmptyBlock(BB); 38436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 38536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 38636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 38736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 38836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 38936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// CanMergeBlocks - Return true if we can merge BB into DestBB if there is a 39036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// single uncond branch between them, and BB contains no other non-phi 39136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// instructions. 39236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::CanMergeBlocks(const BasicBlock *BB, 39336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const BasicBlock *DestBB) const { 39436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We only want to eliminate blocks whose phi nodes are used by phi nodes in 39536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the successor. If there are more complex condition (e.g. preheaders), 39636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // don't mess around with them. 39736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::const_iterator BBI = BB->begin(); 39836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) { 39936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (const User *U : PN->users()) { 40036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Instruction *UI = cast<Instruction>(U); 40136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UI->getParent() != DestBB || !isa<PHINode>(UI)) 40236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 40336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If User is inside DestBB block and it is a PHINode then check 40436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // incoming value. If incoming value is not from BB then this is 40536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // a complex condition (e.g. preheaders) we want to avoid here. 40636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UI->getParent() == DestBB) { 40736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const PHINode *UPN = dyn_cast<PHINode>(UI)) 40836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) { 40936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I)); 41036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Insn && Insn->getParent() == BB && 41136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Insn->getParent() != UPN->getIncomingBlock(I)) 41236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 41336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 41436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 41536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 41636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 41736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 41836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If BB and DestBB contain any common predecessors, then the phi nodes in BB 41936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // and DestBB may have conflicting incoming values for the block. If so, we 42036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // can't merge the block. 42136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin()); 42236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DestBBPN) return true; // no conflict. 42336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 42436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Collect the preds of BB. 42536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<const BasicBlock*, 16> BBPreds; 42636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { 42736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It is faster to get preds from a PHI than with pred_iterator. 42836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) 42936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BBPreds.insert(BBPN->getIncomingBlock(i)); 43036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 43136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BBPreds.insert(pred_begin(BB), pred_end(BB)); 43236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 43336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 43436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Walk the preds of DestBB. 43536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) { 43636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *Pred = DestBBPN->getIncomingBlock(i); 43736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BBPreds.count(Pred)) { // Common predecessor? 43836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BBI = DestBB->begin(); 43936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) { 44036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Value *V1 = PN->getIncomingValueForBlock(Pred); 44136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Value *V2 = PN->getIncomingValueForBlock(BB); 44236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 44336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If V2 is a phi node in BB, look up what the mapped value will be. 44436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const PHINode *V2PN = dyn_cast<PHINode>(V2)) 44536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V2PN->getParent() == BB) 44636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V2 = V2PN->getIncomingValueForBlock(Pred); 44736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 44836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If there is a conflict, bail out. 44936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V1 != V2) return false; 45036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 45136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 45236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 45336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 45436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 45536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 45636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 45736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 45836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// EliminateMostlyEmptyBlock - Eliminate a basic block that have only phi's and 45936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// an unconditional branch in it. 46036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid CodeGenPrepare::EliminateMostlyEmptyBlock(BasicBlock *BB) { 46136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BI = cast<BranchInst>(BB->getTerminator()); 46236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DestBB = BI->getSuccessor(0); 46336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 46436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" << *BB << *DestBB); 46536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 46636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the destination block has a single pred, then this is a trivial edge, 46736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // just collapse it. 46836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) { 46936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SinglePred != DestBB) { 47036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember if SinglePred was the entry block of the function. If so, we 47136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // will need to move BB back to the entry position. 47236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); 4734c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MergeBasicBlockIntoOnlyPred(DestBB, nullptr); 47436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 47536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isEntry && BB != &BB->getParent()->getEntryBlock()) 47636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->moveBefore(&BB->getParent()->getEntryBlock()); 47736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 47836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n"); 47936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return; 48036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 48136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 48236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 48336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB 48436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to handle the new incoming edges it is about to have. 48536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PHINode *PN; 48636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (BasicBlock::iterator BBI = DestBB->begin(); 48736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (PN = dyn_cast<PHINode>(BBI)); ++BBI) { 48836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remove the incoming value for BB, and remember it. 48936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *InVal = PN->removeIncomingValue(BB, false); 49036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 49136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Two options: either the InVal is a phi node defined in BB or it is some 49236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // value that dominates BB. 49336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PHINode *InValPhi = dyn_cast<PHINode>(InVal); 49436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (InValPhi && InValPhi->getParent() == BB) { 49536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add all of the input values of the input PHI as inputs of this phi. 49636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i) 49736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(InValPhi->getIncomingValue(i), 49836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InValPhi->getIncomingBlock(i)); 49936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 50036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, add one instance of the dominating value for each edge that 50136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // we will be adding. 50236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { 50336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) 50436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(InVal, BBPN->getIncomingBlock(i)); 50536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 50636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) 50736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(InVal, *PI); 50836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 50936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 51036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 51136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 51236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The PHIs are now updated, change everything that refers to BB to use 51336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // DestBB and remove BB. 51436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->replaceAllUsesWith(DestBB); 51536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->eraseFromParent(); 51636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumBlocksElim; 51736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 51836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n"); 51936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 52036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 521ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// Computes a map of base pointer relocation instructions to corresponding 522ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// derived pointer relocation instructions given a vector of all relocate calls 523ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic void computeBaseDerivedRelocateMap( 524ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const SmallVectorImpl<User *> &AllRelocateCalls, 525ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DenseMap<IntrinsicInst *, SmallVector<IntrinsicInst *, 2>> & 526ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines RelocateInstMap) { 527ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Collect information in two maps: one primarily for locating the base object 528ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // while filling the second map; the second map is the final structure holding 529ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // a mapping between Base and corresponding Derived relocate calls 530ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DenseMap<std::pair<unsigned, unsigned>, IntrinsicInst *> RelocateIdxMap; 531ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &U : AllRelocateCalls) { 532ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines GCRelocateOperands ThisRelocate(U); 533ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines IntrinsicInst *I = cast<IntrinsicInst>(U); 534ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto K = std::make_pair(ThisRelocate.basePtrIndex(), 535ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ThisRelocate.derivedPtrIndex()); 536ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines RelocateIdxMap.insert(std::make_pair(K, I)); 537ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 538ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &Item : RelocateIdxMap) { 539ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines std::pair<unsigned, unsigned> Key = Item.first; 540ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Key.first == Key.second) 541ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Base relocation: nothing to insert 542ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 543ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 544ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines IntrinsicInst *I = Item.second; 545ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto BaseKey = std::make_pair(Key.first, Key.first); 5464c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 5474c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // We're iterating over RelocateIdxMap so we cannot modify it. 5484c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar auto MaybeBase = RelocateIdxMap.find(BaseKey); 5494c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (MaybeBase == RelocateIdxMap.end()) 550ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // TODO: We might want to insert a new base object relocate and gep off 551ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // that, if there are enough derived object relocates. 552ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 5534c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 5544c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar RelocateInstMap[MaybeBase->second].push_back(I); 555ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 556ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 557ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 558ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// Accepts a GEP and extracts the operands into a vector provided they're all 559ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// small integer constants 560ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP, 561ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVectorImpl<Value *> &OffsetV) { 562ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (unsigned i = 1; i < GEP->getNumOperands(); i++) { 563ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Only accept small constant integer operands 564ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto Op = dyn_cast<ConstantInt>(GEP->getOperand(i)); 565ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!Op || Op->getZExtValue() > 20) 566ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 567ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 568ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 569ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (unsigned i = 1; i < GEP->getNumOperands(); i++) 570ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OffsetV.push_back(GEP->getOperand(i)); 571ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 572ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 573ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 574ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// Takes a RelocatedBase (base pointer relocation instruction) and Targets to 575ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// replace, computes a replacement, and affects it. 576ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic bool 577ebe69fe11e48d322045d5949c83283927a0d790bStephen HinessimplifyRelocatesOffABase(IntrinsicInst *RelocatedBase, 578ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const SmallVectorImpl<IntrinsicInst *> &Targets) { 579ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool MadeChange = false; 580ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &ToReplace : Targets) { 581ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines GCRelocateOperands MasterRelocate(RelocatedBase); 582ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines GCRelocateOperands ThisRelocate(ToReplace); 583ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 584ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(ThisRelocate.basePtrIndex() == MasterRelocate.basePtrIndex() && 585ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "Not relocating a derived object of the original base object"); 586ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (ThisRelocate.basePtrIndex() == ThisRelocate.derivedPtrIndex()) { 587ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // A duplicate relocate call. TODO: coalesce duplicates. 588ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 589ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 590ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 591ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Base = ThisRelocate.basePtr(); 592ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto Derived = dyn_cast<GetElementPtrInst>(ThisRelocate.derivedPtr()); 593ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!Derived || Derived->getPointerOperand() != Base) 594ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 595ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 596ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<Value *, 2> OffsetV; 597ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!getGEPSmallConstantIntOffsetV(Derived, OffsetV)) 598ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 599ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 600ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Create a Builder and replace the target callsite with a gep 601ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines IRBuilder<> Builder(ToReplace); 602ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); 6034c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Value *Replacement = Builder.CreateGEP( 6044c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Derived->getSourceElementType(), RelocatedBase, makeArrayRef(OffsetV)); 605ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *ReplacementInst = cast<Instruction>(Replacement); 606ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ReplacementInst->removeFromParent(); 607ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ReplacementInst->insertAfter(RelocatedBase); 608ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Replacement->takeName(ToReplace); 609ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ToReplace->replaceAllUsesWith(Replacement); 610ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ToReplace->eraseFromParent(); 611ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 612ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MadeChange = true; 613ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 614ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return MadeChange; 615ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 616ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 617ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// Turns this: 618ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 619ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %base = ... 620ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %ptr = gep %base + 15 621ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr) 622ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %base' = relocate(%tok, i32 4, i32 4) 623ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %ptr' = relocate(%tok, i32 4, i32 5) 624ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %val = load %ptr' 625ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 626ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// into this: 627ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 628ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %base = ... 629ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %ptr = gep %base + 15 630ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr) 631ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %base' = gc.relocate(%tok, i32 4, i32 4) 632ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %ptr' = gep %base' + 15 633ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %val = load %ptr' 634ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool CodeGenPrepare::simplifyOffsetableRelocate(Instruction &I) { 635ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool MadeChange = false; 636ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<User *, 2> AllRelocateCalls; 637ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 638ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto *U : I.users()) 639ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (isGCRelocate(dyn_cast<Instruction>(U))) 640ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Collect all the relocate calls associated with a statepoint 641ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines AllRelocateCalls.push_back(U); 642ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 643ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // We need atleast one base pointer relocation + one derived pointer 644ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // relocation to mangle 645ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (AllRelocateCalls.size() < 2) 646ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 647ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 648ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // RelocateInstMap is a mapping from the base relocate instruction to the 649ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // corresponding derived relocate instructions 650ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DenseMap<IntrinsicInst *, SmallVector<IntrinsicInst *, 2>> RelocateInstMap; 651ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines computeBaseDerivedRelocateMap(AllRelocateCalls, RelocateInstMap); 652ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (RelocateInstMap.empty()) 653ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 654ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 655ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &Item : RelocateInstMap) 656ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Item.first is the RelocatedBase to offset against 657ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Item.second is the vector of Targets to replace 658ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MadeChange = simplifyRelocatesOffABase(Item.first, Item.second); 659ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return MadeChange; 660ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 661ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 66236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// SinkCast - Sink the specified cast instruction into its user blocks 66336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool SinkCast(CastInst *CI) { 66436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = CI->getParent(); 66536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 66636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// InsertedCasts - Only insert a cast in each block once. 66736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, CastInst*> InsertedCasts; 66836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 66936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 67036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); 67136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI != E; ) { 67236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Use &TheUse = UI.getUse(); 67336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = cast<Instruction>(*UI); 67436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 67536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this cast is used in. For PHI's this is the 67636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // appropriate predecessor block. 67736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = User->getParent(); 67836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *PN = dyn_cast<PHINode>(User)) { 67936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UserBB = PN->getIncomingBlock(TheUse); 68036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 68136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 68236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Preincrement use iterator so we don't invalidate it. 68336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++UI; 68436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 68536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this user is in the same block as the cast, don't change the cast. 68636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 68736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 68836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we have already inserted a cast into this block, use it. 68936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CastInst *&InsertedCast = InsertedCasts[UserBB]; 69036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 69136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedCast) { 69236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 69336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedCast = 69436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CastInst::Create(CI->getOpcode(), CI->getOperand(0), CI->getType(), "", 69536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertPt); 69636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 69736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 69836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Replace a use of the cast with a use of the new cast. 69936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TheUse = InsertedCast; 7002c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar MadeChange = true; 70136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumCastUses; 70236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 70336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 70436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we removed all uses, nuke the cast. 70536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CI->use_empty()) { 70636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->eraseFromParent(); 70736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 70836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 70936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 71036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 71136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 71236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 71336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeNoopCopyExpression - If the specified cast instruction is a noop 71436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// copy (e.g. it's casting from one pointer type to another, i32->i8 on PPC), 71536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// sink it into user blocks to reduce the number of virtual 71636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// registers that must be created and coalesced. 71736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 71836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Return true if any changes are made. 71936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 72036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI){ 72136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is a noop copy, 72236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT SrcVT = TLI.getValueType(CI->getOperand(0)->getType()); 72336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EVT DstVT = TLI.getValueType(CI->getType()); 72436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 72536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This is an fp<->int conversion? 72636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SrcVT.isInteger() != DstVT.isInteger()) 72736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 72836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 72936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is an extension, it will be a zero or sign extension, which 73036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // isn't a noop. 73136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SrcVT.bitsLT(DstVT)) return false; 73236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 73336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If these values will be promoted, find out what they will be promoted 73436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to. This helps us consider truncates on PPC as noop copies when they 73536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // are. 73636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.getTypeAction(CI->getContext(), SrcVT) == 73736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::TypePromoteInteger) 73836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT); 73936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.getTypeAction(CI->getContext(), DstVT) == 74036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::TypePromoteInteger) 74136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT); 74236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 74336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If, after promotion, these are the same types, this is a noop copy. 74436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SrcVT != DstVT) 74536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 74636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 74736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SinkCast(CI); 74836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 74936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 7502c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar/// CombineUAddWithOverflow - try to combine CI into a call to the 7512c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar/// llvm.uadd.with.overflow intrinsic if possible. 7522c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar/// 7532c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar/// Return true if any changes were made. 7542c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainarstatic bool CombineUAddWithOverflow(CmpInst *CI) { 7552c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Value *A, *B; 7562c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Instruction *AddI; 7572c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if (!match(CI, 7582c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI)))) 7592c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar return false; 7602c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7612c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Type *Ty = AddI->getType(); 7622c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if (!isa<IntegerType>(Ty)) 7632c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar return false; 7642c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7652c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // We don't want to move around uses of condition values this late, so we we 7662c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // check if it is legal to create the call to the intrinsic in the basic 7672c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // block containing the icmp: 7682c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7692c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse()) 7702c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar return false; 7712c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7722c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar#ifndef NDEBUG 7732c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption 7742c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // for now: 7752c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if (AddI->hasOneUse()) 7762c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar assert(*AddI->user_begin() == CI && "expected!"); 7772c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar#endif 7782c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7792c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Module *M = CI->getParent()->getParent()->getParent(); 7802c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty); 7812c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7822c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar auto *InsertPt = AddI->hasOneUse() ? CI : AddI; 7832c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7842c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar auto *UAddWithOverflow = 7852c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt); 7862c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt); 7872c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar auto *Overflow = 7882c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt); 7892c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7902c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar CI->replaceAllUsesWith(Overflow); 7912c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar AddI->replaceAllUsesWith(UAdd); 7922c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar CI->eraseFromParent(); 7932c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar AddI->eraseFromParent(); 7942c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar return true; 7952c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar} 7962c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 7972c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar/// SinkCmpExpression - Sink the given CmpInst into user blocks to reduce 79836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the number of virtual registers that must be created and coalesced. This is 79936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// a clear win except on targets with multiple condition code registers 80036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (PowerPC), where it might lose; some adjustment may be wanted there. 80136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 80236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Return true if any changes are made. 8032c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainarstatic bool SinkCmpExpression(CmpInst *CI) { 80436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = CI->getParent(); 80536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 80636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// InsertedCmp - Only insert a cmp in each block once. 80736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, CmpInst*> InsertedCmps; 80836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 80936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 81036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); 81136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI != E; ) { 81236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Use &TheUse = UI.getUse(); 81336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = cast<Instruction>(*UI); 81436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 81536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Preincrement use iterator so we don't invalidate it. 81636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++UI; 81736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 81836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't bother for PHI nodes. 81936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<PHINode>(User)) 82036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 82136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 82236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this cmp is used in. 82336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = User->getParent(); 82436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 82536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this user is in the same block as the cmp, don't change the cmp. 82636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 82736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 82836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we have already inserted a cmp into this block, use it. 82936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst *&InsertedCmp = InsertedCmps[UserBB]; 83036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 83136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedCmp) { 83236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 83336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedCmp = 83436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst::Create(CI->getOpcode(), 83536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->getPredicate(), CI->getOperand(0), 83636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->getOperand(1), "", InsertPt); 83736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 83836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 83936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Replace a use of the cmp with a use of the new cmp. 84036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TheUse = InsertedCmp; 8412c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar MadeChange = true; 84236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumCmpUses; 84336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 84436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 84536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we removed all uses, nuke the cmp. 8462c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if (CI->use_empty()) { 84736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CI->eraseFromParent(); 8482c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar MadeChange = true; 8492c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar } 85036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 85136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 85236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 85336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8542c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainarstatic bool OptimizeCmpExpression(CmpInst *CI) { 8552c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if (SinkCmpExpression(CI)) 8562c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar return true; 8572c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 8582c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if (CombineUAddWithOverflow(CI)) 8592c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar return true; 8602c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 8612c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar return false; 8622c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar} 8632c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 864dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// isExtractBitsCandidateUse - Check if the candidates could 865dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// be combined with shift instruction, which includes: 866dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 1. Truncate instruction 867dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 2. And instruction and the imm is a mask of the low bits: 868dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// imm & (imm+1) == 0 869dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool isExtractBitsCandidateUse(Instruction *User) { 870dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isa<TruncInst>(User)) { 871dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (User->getOpcode() != Instruction::And || 872dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !isa<ConstantInt>(User->getOperand(1))) 873dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 874dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 875dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue(); 876dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 877dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if ((Cimm & (Cimm + 1)).getBoolValue()) 878dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 879dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 880dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 881dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 882dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 883dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// SinkShiftAndTruncate - sink both shift and truncate instruction 884dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// to the use of truncate's BB. 885dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool 886dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesSinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI, 887dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts, 888dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLowering &TLI) { 889dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *UserBB = User->getParent(); 890dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<BasicBlock *, CastInst *> InsertedTruncs; 891dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncInst *TruncI = dyn_cast<TruncInst>(User); 892dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool MadeChange = false; 893dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 894dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (Value::user_iterator TruncUI = TruncI->user_begin(), 895dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncE = TruncI->user_end(); 896dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncUI != TruncE;) { 897dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 898dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Use &TruncTheUse = TruncUI.getUse(); 899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *TruncUser = cast<Instruction>(*TruncUI); 900dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Preincrement use iterator so we don't invalidate it. 901dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 902dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++TruncUI; 903dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 904dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode()); 905dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ISDOpcode) 906dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 90837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If the use is actually a legal node, there will not be an 90937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // implicit truncate. 91037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // FIXME: always querying the result type is just an 91137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // approximation; some nodes' legality is determined by the 91237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // operand or other means. There's no good way to find out though. 91337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (TLI.isOperationLegalOrCustom( 91437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ISDOpcode, TLI.getValueType(TruncUser->getType(), true))) 915dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 916dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 917dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't bother for PHI nodes. 918dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<PHINode>(TruncUser)) 919dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 920dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 921dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *TruncUserBB = TruncUser->getParent(); 922dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 923dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (UserBB == TruncUserBB) 924dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 925dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 926dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB]; 927dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB]; 928dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 929dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!InsertedShift && !InsertedTrunc) { 930dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt(); 931dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Sink the shift 932dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShiftI->getOpcode() == Instruction::AShr) 933dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 934dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "", InsertPt); 935dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 936dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 937dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "", InsertPt); 938dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 939dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Sink the trunc 940dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt(); 941dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncInsertPt++; 942dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 943dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift, 944dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncI->getType(), "", TruncInsertPt); 945dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 946dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MadeChange = true; 947dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 948dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TruncTheUse = InsertedTrunc; 949dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 951dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MadeChange; 952dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 953dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 954dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// OptimizeExtractBits - sink the shift *right* instruction into user blocks if 955dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// the uses could potentially be combined with this shift instruction and 956dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// generate BitExtract instruction. It will only be applied if the architecture 957dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// supports BitExtract instruction. Here is an example: 958dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// BB1: 959dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.shift = lshr i64 %arg1, 32 960dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// BB2: 961dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.trunc = trunc i64 %x.extract.shift to i16 962dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// ==> 963dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 964dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// BB2: 965dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.shift.1 = lshr i64 %arg1, 32 966dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16 967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// 968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// CodeGen will recoginze the pattern in BB2 and generate BitExtract 969dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// instruction. 970dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines/// Return true if any changes are made. 971dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesstatic bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI, 972dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const TargetLowering &TLI) { 973dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *DefBB = ShiftI->getParent(); 974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 975dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines /// Only insert instructions in each block once. 976dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts; 977dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 978dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(ShiftI->getType())); 979dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool MadeChange = false; 981dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end(); 982dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines UI != E;) { 983dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Use &TheUse = UI.getUse(); 984dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *User = cast<Instruction>(*UI); 985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Preincrement use iterator so we don't invalidate it. 986dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++UI; 987dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 988dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Don't bother for PHI nodes. 989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<PHINode>(User)) 990dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 991dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 992dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!isExtractBitsCandidateUse(User)) 993dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 995dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock *UserBB = User->getParent(); 996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 997dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (UserBB == DefBB) { 998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the shift and truncate instruction are in the same BB. The use of 999dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the truncate(TruncUse) may still introduce another truncate if not 1000dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // legal. In this case, we would like to sink both shift and truncate 1001dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instruction to the BB of TruncUse. 1002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // for example: 1003dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // BB1: 1004dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // i64 shift.result = lshr i64 opnd, imm 1005dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // trunc.result = trunc shift.result to i16 1006dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 1007dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // BB2: 1008dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // ----> We will have an implicit truncate here if the architecture does 1009dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // not have i16 compare. 1010dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // cmp i16 trunc.result, opnd2 1011dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // 1012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (isa<TruncInst>(User) && shiftIsLegal 1013dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the type of the truncate is legal, no trucate will be 1014dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // introduced in other basic blocks. 1015dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines && (!TLI.isTypeLegal(TLI.getValueType(User->getType())))) 1016dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MadeChange = 1017dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI); 1018dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines continue; 1020dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1021dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we have already inserted a shift into this block, use it. 1022dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator *&InsertedShift = InsertedShifts[UserBB]; 1023dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!InsertedShift) { 1025dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 1026dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1027dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShiftI->getOpcode() == Instruction::AShr) 1028dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 1029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "", InsertPt); 1030dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 1031dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InsertedShift = 1032dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "", InsertPt); 1033dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1034dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MadeChange = true; 1035dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Replace a use of the shift with a use of the new shift. 1038dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TheUse = InsertedShift; 1039dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 1040dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1041dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If we removed all uses, nuke the shift. 1042dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ShiftI->use_empty()) 1043dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ShiftI->eraseFromParent(); 1044dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1045dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return MadeChange; 1046dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 1047dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 1048ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// ScalarizeMaskedLoad() translates masked load intrinsic, like 1049ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// <16 x i32 > @llvm.masked.load( <16 x i32>* %addr, i32 align, 1050ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// <16 x i1> %mask, <16 x i32> %passthru) 1051ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// to a chain of basic blocks, whith loading element one-by-one if 1052ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// the appropriate mask bit is set 1053ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1054ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %1 = bitcast i8* %addr to i32* 1055ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %2 = extractelement <16 x i1> %mask, i32 0 1056ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %3 = icmp eq i1 %2, true 1057ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br i1 %3, label %cond.load, label %else 1058ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1059ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines//cond.load: ; preds = %0 1060ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %4 = getelementptr i32* %1, i32 0 1061ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %5 = load i32* %4 1062ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %6 = insertelement <16 x i32> undef, i32 %5, i32 0 1063ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br label %else 1064ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1065ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines//else: ; preds = %0, %cond.load 1066ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %res.phi.else = phi <16 x i32> [ %6, %cond.load ], [ undef, %0 ] 1067ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %7 = extractelement <16 x i1> %mask, i32 1 1068ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %8 = icmp eq i1 %7, true 1069ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br i1 %8, label %cond.load1, label %else2 1070ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1071ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines//cond.load1: ; preds = %else 1072ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %9 = getelementptr i32* %1, i32 1 1073ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %10 = load i32* %9 1074ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %11 = insertelement <16 x i32> %res.phi.else, i32 %10, i32 1 1075ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br label %else2 1076ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1077ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines//else2: ; preds = %else, %cond.load1 1078ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ] 1079ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %12 = extractelement <16 x i1> %mask, i32 2 1080ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %13 = icmp eq i1 %12, true 1081ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br i1 %13, label %cond.load4, label %else5 1082ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1083ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic void ScalarizeMaskedLoad(CallInst *CI) { 1084ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Ptr = CI->getArgOperand(0); 1085ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Src0 = CI->getArgOperand(3); 1086ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Mask = CI->getArgOperand(2); 1087ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines VectorType *VecType = dyn_cast<VectorType>(CI->getType()); 1088ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *EltTy = VecType->getElementType(); 1089ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1090ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(VecType && "Unexpected return type of masked load intrinsic"); 1091ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1092ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines IRBuilder<> Builder(CI->getContext()); 1093ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *InsertPt = CI; 1094ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *IfBlock = CI->getParent(); 1095ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *CondBlock = nullptr; 1096ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *PrevIfBlock = CI->getParent(); 1097ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetInsertPoint(InsertPt); 1098ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1099ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetCurrentDebugLocation(CI->getDebugLoc()); 1100ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1101ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Bitcast %addr fron i8* to EltTy* 1102ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *NewPtrType = 1103ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace()); 1104ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType); 1105ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *UndefVal = UndefValue::get(VecType); 1106ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1107ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The result vector 1108ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *VResult = UndefVal; 1109ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1110ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines PHINode *Phi = nullptr; 1111ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *PrevPhi = UndefVal; 1112ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1113ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned VectorWidth = VecType->getNumElements(); 1114ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { 1115ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1116ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Fill the "else" block, created in the previous iteration 1117ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1118ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ] 1119ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %mask_1 = extractelement <16 x i1> %mask, i32 Idx 1120ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %to_load = icmp eq i1 %mask_1, true 1121ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // br i1 %to_load, label %cond.load, label %else 1122ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1123ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Idx > 0) { 1124ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Phi = Builder.CreatePHI(VecType, 2, "res.phi.else"); 1125ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Phi->addIncoming(VResult, CondBlock); 1126ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Phi->addIncoming(PrevPhi, PrevIfBlock); 1127ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines PrevPhi = Phi; 1128ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines VResult = Phi; 1129ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1130ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1131ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx)); 1132ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate, 1133ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ConstantInt::get(Predicate->getType(), 1)); 1134ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1135ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Create "cond" block 1136ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1137ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %EltAddr = getelementptr i32* %1, i32 0 1138ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %Elt = load i32* %EltAddr 1139ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx 1140ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1141ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.load"); 1142ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetInsertPoint(InsertPt); 11432c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar 11442c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Value *Gep = 11452c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); 1146ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LoadInst* Load = Builder.CreateLoad(Gep, false); 1147ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx)); 1148ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1149ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Create "else" block, fill it in the next iteration 1150ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else"); 1151ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetInsertPoint(InsertPt); 1152ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *OldBr = IfBlock->getTerminator(); 1153ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr); 1154ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OldBr->eraseFromParent(); 1155ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines PrevIfBlock = IfBlock; 1156ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines IfBlock = NewIfBlock; 115736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 1158ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1159ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Phi = Builder.CreatePHI(VecType, 2, "res.phi.select"); 1160ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Phi->addIncoming(VResult, CondBlock); 1161ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Phi->addIncoming(PrevPhi, PrevIfBlock); 1162ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *NewI = Builder.CreateSelect(Mask, Phi, Src0); 1163ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CI->replaceAllUsesWith(NewI); 1164ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CI->eraseFromParent(); 1165ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 1166ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1167ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// ScalarizeMaskedStore() translates masked store intrinsic, like 1168ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// void @llvm.masked.store(<16 x i32> %src, <16 x i32>* %addr, i32 align, 1169ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// <16 x i1> %mask) 1170ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// to a chain of basic blocks, that stores element one-by-one if 1171ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// the appropriate mask bit is set 1172ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1173ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %1 = bitcast i8* %addr to i32* 1174ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %2 = extractelement <16 x i1> %mask, i32 0 1175ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %3 = icmp eq i1 %2, true 1176ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br i1 %3, label %cond.store, label %else 1177ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1178ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// cond.store: ; preds = %0 1179ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %4 = extractelement <16 x i32> %val, i32 0 1180ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %5 = getelementptr i32* %1, i32 0 1181ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// store i32 %4, i32* %5 1182ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br label %else 1183ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1184ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// else: ; preds = %0, %cond.store 1185ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %6 = extractelement <16 x i1> %mask, i32 1 1186ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %7 = icmp eq i1 %6, true 1187ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br i1 %7, label %cond.store1, label %else2 1188ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// 1189ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// cond.store1: ; preds = %else 1190ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %8 = extractelement <16 x i32> %val, i32 1 1191ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// %9 = getelementptr i32* %1, i32 1 1192ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// store i32 %8, i32* %9 1193ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// br label %else2 1194ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// . . . 1195ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic void ScalarizeMaskedStore(CallInst *CI) { 1196ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Ptr = CI->getArgOperand(1); 1197ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Src = CI->getArgOperand(0); 1198ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Mask = CI->getArgOperand(3); 1199ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1200ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines VectorType *VecType = dyn_cast<VectorType>(Src->getType()); 1201ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *EltTy = VecType->getElementType(); 1202ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1203ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(VecType && "Unexpected data type in masked store intrinsic"); 1204ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1205ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines IRBuilder<> Builder(CI->getContext()); 1206ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *InsertPt = CI; 1207ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *IfBlock = CI->getParent(); 1208ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetInsertPoint(InsertPt); 1209ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetCurrentDebugLocation(CI->getDebugLoc()); 1210ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1211ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Bitcast %addr fron i8* to EltTy* 1212ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *NewPtrType = 1213ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace()); 1214ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType); 1215ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1216ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned VectorWidth = VecType->getNumElements(); 1217ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { 1218ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1219ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Fill the "else" block, created in the previous iteration 1220ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1221ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %mask_1 = extractelement <16 x i1> %mask, i32 Idx 1222ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %to_store = icmp eq i1 %mask_1, true 1223ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // br i1 %to_load, label %cond.store, label %else 1224ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1225ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx)); 1226ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Predicate, 1227ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ConstantInt::get(Predicate->getType(), 1)); 1228ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1229ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Create "cond" block 1230ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1231ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %OneElt = extractelement <16 x i32> %Src, i32 Idx 1232ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %EltAddr = getelementptr i32* %1, i32 0 1233ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %store i32 %OneElt, i32* %EltAddr 1234ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 1235ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.store"); 1236ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetInsertPoint(InsertPt); 1237ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1238ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx)); 12392c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Value *Gep = 12402c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); 1241ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.CreateStore(OneElt, Gep); 1242ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1243ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Create "else" block, fill it in the next iteration 1244ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else"); 1245ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Builder.SetInsertPoint(InsertPt); 1246ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *OldBr = IfBlock->getTerminator(); 1247ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BranchInst::Create(CondBlock, NewIfBlock, Cmp, OldBr); 1248ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OldBr->eraseFromParent(); 1249ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines IfBlock = NewIfBlock; 125036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 1251ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CI->eraseFromParent(); 1252ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 125336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1254ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool CodeGenPrepare::OptimizeCallInst(CallInst *CI, bool& ModifiedDT) { 125536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = CI->getParent(); 125636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 125736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower inline assembly if we can. 125836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we found an inline asm expession, and if the target knows how to 125936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // lower it to normal LLVM code, do so now. 126036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && isa<InlineAsm>(CI->getCalledValue())) { 126136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI->ExpandInlineAsm(CI)) { 126236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Avoid invalidating the iterator. 126336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = BB->begin(); 126436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Avoid processing instructions out of order, which could cause 126536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // reuse before a value is defined. 126636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 126736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 126836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 126936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Sink address computing for memory operands into the block. 127036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OptimizeInlineAsmInst(CI)) 127136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 127236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 127336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12744c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout *TD = TLI ? TLI->getDataLayout() : nullptr; 12754c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 12764c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // Align the pointer arguments to this call if the target thinks it's a good 12774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // idea 12784c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned MinSize, PrefAlign; 12794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (TLI && TD && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) { 12804c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar for (auto &Arg : CI->arg_operands()) { 12814c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // We want to align both objects whose address is used directly and 12824c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // objects whose address is used in casts and GEPs, though it only makes 12834c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // sense for GEPs if the offset is a multiple of the desired alignment and 12844c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // if size - offset meets the size threshold. 12854c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (!Arg->getType()->isPointerTy()) 12864c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar continue; 12874c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar APInt Offset(TD->getPointerSizeInBits( 12884c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar cast<PointerType>(Arg->getType())->getAddressSpace()), 0); 12894c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*TD, Offset); 12904c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar uint64_t Offset2 = Offset.getLimitedValue(); 12912c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if ((Offset2 & (PrefAlign-1)) != 0) 12922c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar continue; 12934c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar AllocaInst *AI; 12942c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if ((AI = dyn_cast<AllocaInst>(Val)) && 12954c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar AI->getAlignment() < PrefAlign && 12964c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TD->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2) 12974c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar AI->setAlignment(PrefAlign); 12982c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // Global variables can only be aligned if they are defined in this 12992c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // object (i.e. they are uniquely initialized in this object), and 13002c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // over-aligning global variables that have an explicit section is 13012c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar // forbidden. 13022c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar GlobalVariable *GV; 13032c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar if ((GV = dyn_cast<GlobalVariable>(Val)) && 13042c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar GV->hasUniqueInitializer() && 13052c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar !GV->hasSection() && 13062c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar GV->getAlignment() < PrefAlign && 13072c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TD->getTypeAllocSize( 13082c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar GV->getType()->getElementType()) >= MinSize + Offset2) 13092c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar GV->setAlignment(PrefAlign); 13104c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 13114c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // If this is a memcpy (or similar) then we may be able to improve the 13124c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // alignment 13134c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) { 13144c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned Align = getKnownAlignment(MI->getDest(), *TD); 13154c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) 13164c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Align = std::min(Align, getKnownAlignment(MTI->getSource(), *TD)); 13174c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (Align > MI->getAlignment()) 13184c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), Align)); 13194c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 13204c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 13214c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar 132236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI); 1323ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (II) { 1324ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines switch (II->getIntrinsicID()) { 1325ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines default: break; 1326ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case Intrinsic::objectsize: { 1327ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Lower all uses of llvm.objectsize.* 1328ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool Min = (cast<ConstantInt>(II->getArgOperand(1))->getZExtValue() == 1); 1329ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *ReturnTy = CI->getType(); 1330ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Constant *RetVal = ConstantInt::get(ReturnTy, Min ? 0 : -1ULL); 1331ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1332ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Substituting this can cause recursive simplifications, which can 1333ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // invalidate our iterator. Use a WeakVH to hold onto it in case this 1334ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // happens. 1335ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines WeakVH IterHandle(CurInstIterator); 1336ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 1337ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines replaceAndRecursivelySimplify(CI, RetVal, 13384c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TLInfo, nullptr); 133936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1340ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If the iterator instruction was recursively deleted, start over at the 1341ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // start of the block. 1342ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (IterHandle != CurInstIterator) { 1343ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CurInstIterator = BB->begin(); 1344ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SunkAddrs.clear(); 1345ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1346ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 1347ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1348ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case Intrinsic::masked_load: { 1349ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Scalarize unsupported vector masked load 1350ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!TTI->isLegalMaskedLoad(CI->getType(), 1)) { 1351ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ScalarizeMaskedLoad(CI); 1352ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ModifiedDT = true; 1353ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 1354ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1355ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 1356ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1357ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case Intrinsic::masked_store: { 1358ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType(), 1)) { 1359ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ScalarizeMaskedStore(CI); 1360ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ModifiedDT = true; 1361ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 1362ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1363ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 1364ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 136536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 136636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1367ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (TLI) { 1368ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<Value*, 2> PtrOps; 1369ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *AccessTy; 1370ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy)) 1371ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines while (!PtrOps.empty()) 1372ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (OptimizeMemoryInst(II, PtrOps.pop_back_val(), AccessTy)) 1373ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 1374ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 137536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 137636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 137736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // From here on out we're working with named functions. 1378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!CI->getCalledFunction()) return false; 137936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 138036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Lower all default uses of _chk calls. This is very similar 138136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to what InstCombineCalls does, but here we are only lowering calls 1382ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // to fortified library functions (e.g. __memcpy_chk) that have the default 1383ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // "don't know" as the objectsize. Anything else should be left alone. 13844c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar FortifiedLibCallSimplifier Simplifier(TLInfo, true); 1385ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Value *V = Simplifier.optimizeCall(CI)) { 1386ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CI->replaceAllUsesWith(V); 1387ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CI->eraseFromParent(); 1388ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 1389ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 1390ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 139136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 139236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 139336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// DupRetToEnableTailCallOpts - Look for opportunities to duplicate return 139436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// instructions to the predecessor to enable tail call optimizations. The 139536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// case it is currently looking for is: 139636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @code 139736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb0: 139836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp0 = tail call i32 @f0() 139936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// br label %return 140036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb1: 140136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp1 = tail call i32 @f1() 140236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// br label %return 140336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb2: 140436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp2 = tail call i32 @f2() 140536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// br label %return 140636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// return: 140736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ] 140836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %retval 140936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @endcode 141036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 141136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// => 141236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 141336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @code 141436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb0: 141536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp0 = tail call i32 @f0() 141636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %tmp0 141736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb1: 141836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp1 = tail call i32 @f1() 141936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %tmp1 142036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// bb2: 142136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// %tmp2 = tail call i32 @f2() 142236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ret i32 %tmp2 142336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// @endcode 142436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::DupRetToEnableTailCallOpts(BasicBlock *BB) { 142536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI) 142636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 142736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 142836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator()); 142936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!RI) 143036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 143136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1432dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines PHINode *PN = nullptr; 1433dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BitCastInst *BCI = nullptr; 143436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = RI->getReturnValue(); 143536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V) { 143636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BCI = dyn_cast<BitCastInst>(V); 143736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BCI) 143836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = BCI->getOperand(0); 143936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 144036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN = dyn_cast<PHINode>(V); 144136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!PN) 144236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 144336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 144436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 144536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PN && PN->getParent() != BB) 144636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 144736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 144836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It's not safe to eliminate the sign / zero extension of the return value. 144936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // See llvm::isInTailCallPosition(). 145036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Function *F = BB->getParent(); 145136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AttributeSet CallerAttrs = F->getAttributes(); 145236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt) || 145336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt)) 145436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 145536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 145636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure there are no instructions between the PHI and return, or that the 145736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // return is the first instruction in the block. 145836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PN) { 145936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator BI = BB->begin(); 146036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines do { ++BI; } while (isa<DbgInfoIntrinsic>(BI)); 146136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (&*BI == BCI) 146236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Also skip over the bitcast. 146336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++BI; 146436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (&*BI != RI) 146536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 146636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 146736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator BI = BB->begin(); 146836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (isa<DbgInfoIntrinsic>(BI)) ++BI; 146936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (&*BI != RI) 147036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 147136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 147236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 147336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail 147436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// call. 147536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<CallInst*, 4> TailCalls; 147636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PN) { 147736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) { 147836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I)); 147936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure the phi value is indeed produced by the tail call. 148036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) && 148136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->mayBeEmittedAsTailCall(CI)) 148236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TailCalls.push_back(CI); 148336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 148436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 148536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<BasicBlock*, 4> VisitedBBs; 148636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) { 148737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!VisitedBBs.insert(*PI).second) 148836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 148936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 149036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::InstListType &InstList = (*PI)->getInstList(); 149136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::InstListType::reverse_iterator RI = InstList.rbegin(); 149236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::InstListType::reverse_iterator RE = InstList.rend(); 149336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines do { ++RI; } while (RI != RE && isa<DbgInfoIntrinsic>(&*RI)); 149436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (RI == RE) 149536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 149636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 149736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallInst *CI = dyn_cast<CallInst>(&*RI); 149836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI)) 149936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TailCalls.push_back(CI); 150036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 150136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 150236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 150336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Changed = false; 150436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = TailCalls.size(); i != e; ++i) { 150536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallInst *CI = TailCalls[i]; 150636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CallSite CS(CI); 150736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 150836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Conservatively require the attributes of the call to match those of the 150936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // return. Ignore noalias because it doesn't affect the call sequence. 151036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AttributeSet CalleeAttrs = CS.getAttributes(); 151136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex). 151236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines removeAttribute(Attribute::NoAlias) != 151336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AttrBuilder(CalleeAttrs, AttributeSet::ReturnIndex). 151436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines removeAttribute(Attribute::NoAlias)) 151536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 151636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 151736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure the call instruction is followed by an unconditional branch to 151836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the return block. 151936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *CallBB = CI->getParent(); 152036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator()); 152136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB) 152236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 152336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 152436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Duplicate the return into CallBB. 152536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (void)FoldReturnIntoUncondBranch(RI, BB, CallBB); 152636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ModifiedDT = Changed = true; 152736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumRetsDup; 152836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 152936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 153036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we eliminated all predecessors of the block, delete the block now. 153136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB)) 153236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BB->eraseFromParent(); 153336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 153436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Changed; 153536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 153636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 153736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 153836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// Memory Optimization 153936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines//===----------------------------------------------------------------------===// 154036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 154136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesnamespace { 154236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 154336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ExtAddrMode - This is an extended version of TargetLowering::AddrMode 154436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// which holds actual Value*'s for register values. 154536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstruct ExtAddrMode : public TargetLowering::AddrMode { 154636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *BaseReg; 154736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *ScaledReg; 1548dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ExtAddrMode() : BaseReg(nullptr), ScaledReg(nullptr) {} 154936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void print(raw_ostream &OS) const; 155036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void dump() const; 155136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 155236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool operator==(const ExtAddrMode& O) const { 155336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return (BaseReg == O.BaseReg) && (ScaledReg == O.ScaledReg) && 155436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (BaseGV == O.BaseGV) && (BaseOffs == O.BaseOffs) && 155536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (HasBaseReg == O.HasBaseReg) && (Scale == O.Scale); 155636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 155736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 155836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 155936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#ifndef NDEBUG 156036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) { 156136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AM.print(OS); 156236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OS; 156336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 156436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#endif 156536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 156636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid ExtAddrMode::print(raw_ostream &OS) const { 156736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool NeedPlus = false; 156836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << "["; 156936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BaseGV) { 157036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << (NeedPlus ? " + " : "") 157136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "GV:"; 157236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BaseGV->printAsOperand(OS, /*PrintType=*/false); 157336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NeedPlus = true; 157436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 157536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1576c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (BaseOffs) { 1577c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines OS << (NeedPlus ? " + " : "") 1578c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines << BaseOffs; 1579c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines NeedPlus = true; 1580c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines } 158136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 158236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BaseReg) { 158336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << (NeedPlus ? " + " : "") 158436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "Base:"; 158536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BaseReg->printAsOperand(OS, /*PrintType=*/false); 158636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NeedPlus = true; 158736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 158836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Scale) { 158936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << (NeedPlus ? " + " : "") 159036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << Scale << "*"; 159136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ScaledReg->printAsOperand(OS, /*PrintType=*/false); 159236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 159336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 159436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OS << ']'; 159536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 159636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 159736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 159836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid ExtAddrMode::dump() const { 159936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print(dbgs()); 160036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines dbgs() << '\n'; 160136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 160236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines#endif 160336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 160436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \brief This class provides transaction based operation on the IR. 160536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Every change made through this class is recorded in the internal state and 160636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// can be undone (rollback) until commit is called. 160736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass TypePromotionTransaction { 160836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 160936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief This represents the common interface of the individual transaction. 161036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Each class implements the logic for doing one specific modification on 161136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// the IR via the TypePromotionTransaction. 161236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class TypePromotionAction { 161336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines protected: 161436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The Instruction modified. 161536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Inst; 161636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 161736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 161836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Constructor of the action. 161936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The constructor performs the related action on the IR. 162036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionAction(Instruction *Inst) : Inst(Inst) {} 162136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 162236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines virtual ~TypePromotionAction() {} 162336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 162436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Undo the modification done by this action. 162536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// When this method is called, the IR must be in the same state as it was 162636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// before this action was applied. 162736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \pre Undoing the action works if and only if the IR is in the exact same 162836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// state as it was directly after this action was applied. 162936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines virtual void undo() = 0; 163036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 163136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Advocate every change made by this action. 163236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// When the results on the IR of the action are to be kept, it is important 163336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// to call this function, otherwise hidden information may be kept forever. 163436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines virtual void commit() { 163536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Nothing to be done, this action is not doing anything. 163636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 163736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 163836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 163936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Utility to remember the position of an instruction. 164036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class InsertionHandler { 164136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Position of an instruction. 164236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Either an instruction: 164336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// - Is the first in a basic block: BB is used. 164436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// - Has a previous instructon: PrevInst is used. 164536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines union { 164636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *PrevInst; 164736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB; 164836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } Point; 164936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Remember whether or not the instruction had a previous instruction. 165036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool HasPrevInstruction; 165136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 165236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 165336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Record the position of \p Inst. 165436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertionHandler(Instruction *Inst) { 165536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator It = Inst; 165636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines HasPrevInstruction = (It != (Inst->getParent()->begin())); 165736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (HasPrevInstruction) 165836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Point.PrevInst = --It; 165936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 166036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Point.BB = Inst->getParent(); 166136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 166236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 166336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Insert \p Inst at the recorded position. 166436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void insert(Instruction *Inst) { 166536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (HasPrevInstruction) { 166636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Inst->getParent()) 166736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->removeFromParent(); 166836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->insertAfter(Point.PrevInst); 166936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 167036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Position = Point.BB->getFirstInsertionPt(); 167136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Inst->getParent()) 167236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->moveBefore(Position); 167336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 167436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->insertBefore(Position); 167536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 167636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 167736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 167836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 167936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Move an instruction before another. 168036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class InstructionMoveBefore : public TypePromotionAction { 168136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Original position of the instruction. 168236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertionHandler Position; 168336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 168436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 168536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Move \p Inst before \p Before. 168636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstructionMoveBefore(Instruction *Inst, Instruction *Before) 168736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), Position(Inst) { 168836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before << "\n"); 168936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->moveBefore(Before); 169036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 169136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 169236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Move the instruction back to its original position. 169336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 169436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: moveBefore: " << *Inst << "\n"); 169536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Position.insert(Inst); 169636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 169736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 169836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 169936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Set the operand of an instruction with a new value. 170036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class OperandSetter : public TypePromotionAction { 170136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Original operand of the instruction. 170236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Origin; 170336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Index of the modified instruction. 170436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx; 170536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 170636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 170736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Set \p Idx operand of \p Inst with \p NewVal. 170836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal) 170936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), Idx(Idx) { 171036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n" 171136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "for:" << *Inst << "\n" 171236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "with:" << *NewVal << "\n"); 171336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Origin = Inst->getOperand(Idx); 171436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(Idx, NewVal); 171536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 171636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 171736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Restore the original value of the instruction. 171836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 171936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n" 172036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "for: " << *Inst << "\n" 172136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "with: " << *Origin << "\n"); 172236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(Idx, Origin); 172336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 172436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 172536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 172636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Hide the operands of an instruction. 172736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Do as if this instruction was not using any of its operands. 172836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class OperandsHider : public TypePromotionAction { 172936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The list of original operands. 173036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Value *, 4> OriginalValues; 173136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 173236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 173336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove \p Inst from the uses of the operands of \p Inst. 173436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) { 173536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n"); 173636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumOpnds = Inst->getNumOperands(); 173736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OriginalValues.reserve(NumOpnds); 173836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned It = 0; It < NumOpnds; ++It) { 173936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Save the current operand. 174036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Val = Inst->getOperand(It); 174136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OriginalValues.push_back(Val); 174236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Set a dummy one. 174336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We could use OperandSetter here, but that would implied an overhead 174436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // that we are not willing to pay. 174536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(It, UndefValue::get(Val->getType())); 174636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 174736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 174836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 174936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Restore the original list of uses. 175036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 175136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n"); 175236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It) 175336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->setOperand(It, OriginalValues[It]); 175436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 175536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 175636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 175736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a truncate instruction. 175836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class TruncBuilder : public TypePromotionAction { 175937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val; 176036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 176136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a truncate instruction of \p Opnd producing a \p Ty 176236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// result. 176336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// trunc Opnd to Ty. 176436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) { 176536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(Opnd); 176637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Val = Builder.CreateTrunc(Opnd, Ty, "promoted"); 176737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Do: TruncBuilder: " << *Val << "\n"); 176836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 176936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 177037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Get the built value. 177137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *getBuiltValue() { return Val; } 177236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 177336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove the built instruction. 177436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 177537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val << "\n"); 177637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Instruction *IVal = dyn_cast<Instruction>(Val)) 177737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IVal->eraseFromParent(); 177836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 177936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 178036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 178136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a sign extension instruction. 178236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class SExtBuilder : public TypePromotionAction { 178337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val; 178436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 178536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Build a sign extension instruction of \p Opnd producing a \p Ty 178636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// result. 178736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// sext Opnd to Ty. 178836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty) 178937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : TypePromotionAction(InsertPt) { 179037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRBuilder<> Builder(InsertPt); 179137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Val = Builder.CreateSExt(Opnd, Ty, "promoted"); 179237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Do: SExtBuilder: " << *Val << "\n"); 179337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 179437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 179537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Get the built value. 179637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *getBuiltValue() { return Val; } 179737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 179837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Remove the built instruction. 179937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines void undo() override { 180037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val << "\n"); 180137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Instruction *IVal = dyn_cast<Instruction>(Val)) 180237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IVal->eraseFromParent(); 180337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 180437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines }; 180537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 180637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Build a zero extension instruction. 180737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines class ZExtBuilder : public TypePromotionAction { 180837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val; 180937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines public: 181037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Build a zero extension instruction of \p Opnd producing a \p Ty 181137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// result. 181237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// zext Opnd to Ty. 181337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ZExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty) 181437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : TypePromotionAction(InsertPt) { 181536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(InsertPt); 181637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Val = Builder.CreateZExt(Opnd, Ty, "promoted"); 181737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val << "\n"); 181836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 181936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 182037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Get the built value. 182137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *getBuiltValue() { return Val; } 182236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 182336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove the built instruction. 182436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 182537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val << "\n"); 182637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Instruction *IVal = dyn_cast<Instruction>(Val)) 182737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IVal->eraseFromParent(); 182836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 182936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 183036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 183136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Mutate an instruction to another type. 183236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class TypeMutator : public TypePromotionAction { 183336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Record the original type. 183436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *OrigTy; 183536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 183636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 183736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Mutate the type of \p Inst into \p NewTy. 183836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypeMutator(Instruction *Inst, Type *NewTy) 183936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), OrigTy(Inst->getType()) { 184036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy 184136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "\n"); 184236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->mutateType(NewTy); 184336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 184436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 184536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Mutate the instruction back to its original type. 184636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 184736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy 184836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "\n"); 184936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->mutateType(OrigTy); 185036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 185136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 185236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 185336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Replace the uses of an instruction by another instruction. 185436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class UsesReplacer : public TypePromotionAction { 185536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Helper structure to keep track of the replaced uses. 185636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines struct InstructionAndIdx { 185736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The instruction using the instruction. 185836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Inst; 185936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The index where this instruction is used for Inst. 186036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx; 186136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstructionAndIdx(Instruction *Inst, unsigned Idx) 186236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : Inst(Inst), Idx(Idx) {} 186336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 186436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 186536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keep track of the original uses (pair Instruction, Index). 186636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<InstructionAndIdx, 4> OriginalUses; 186736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef SmallVectorImpl<InstructionAndIdx>::iterator use_iterator; 186836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 186936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 187036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Replace all the use of \p Inst by \p New. 187136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UsesReplacer(Instruction *Inst, Value *New) : TypePromotionAction(Inst) { 187236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New 187336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines << "\n"); 187436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Record the original uses. 187536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Use &U : Inst->uses()) { 187636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UserI = cast<Instruction>(U.getUser()); 187736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo())); 187836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 187936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Now, we can replace the uses. 188036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->replaceAllUsesWith(New); 188136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 188236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 188336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Reassign the original uses of Inst to Inst. 188436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 188536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n"); 188636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (use_iterator UseIt = OriginalUses.begin(), 188736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines EndIt = OriginalUses.end(); 188836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UseIt != EndIt; ++UseIt) { 188936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UseIt->Inst->setOperand(UseIt->Idx, Inst); 189036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 189136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 189236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 189336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 189436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove an instruction from the IR. 189536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines class InstructionRemover : public TypePromotionAction { 189636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Original position of the instruction. 189736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertionHandler Inserter; 189836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Helper structure to hide all the link to the instruction. In other 189936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// words, this helps to do as if the instruction was removed. 190036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OperandsHider Hider; 190136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Keep track of the uses replaced, if any. 190236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UsesReplacer *Replacer; 190336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 190436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines public: 190536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Remove all reference of \p Inst and optinally replace all its 190636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// uses with New. 1907dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines /// \pre If !Inst->use_empty(), then New != nullptr 1908dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InstructionRemover(Instruction *Inst, Value *New = nullptr) 190936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst), 1910dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Replacer(nullptr) { 191136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (New) 191236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Replacer = new UsesReplacer(Inst, New); 191336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n"); 191436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inst->removeFromParent(); 191536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 191636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 19172c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar ~InstructionRemover() override { delete Replacer; } 191836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 191936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Really remove the instruction. 192036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void commit() override { delete Inst; } 192136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 192236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Resurrect the instruction and reassign it to the proper uses if 192336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// new value was provided when build this action. 192436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void undo() override { 192536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n"); 192636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Inserter.insert(Inst); 192736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Replacer) 192836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Replacer->undo(); 192936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Hider.undo(); 193036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 193136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines }; 193236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 193336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 193436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Restoration point. 193536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The restoration point is a pointer to an action instead of an iterator 193636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// because the iterator may be invalidated but not the pointer. 193736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines typedef const TypePromotionAction *ConstRestorationPt; 193836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Advocate every changes made in that transaction. 193936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void commit(); 194036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Undo all the changes made after the given point. 194136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void rollback(ConstRestorationPt Point); 194236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Get the current restoration point. 194336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstRestorationPt getRestorationPoint() const; 194436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 194536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \name API for IR modification with state keeping to support rollback. 194636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// @{ 194736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Instruction::setOperand. 194836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal); 194936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Instruction::eraseFromParent. 1950dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr); 195136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Value::replaceAllUsesWith. 195236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void replaceAllUsesWith(Instruction *Inst, Value *New); 195336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Value::mutateType. 195436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void mutateType(Instruction *Inst, Type *NewTy); 195536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as IRBuilder::createTrunc. 195637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *createTrunc(Instruction *Opnd, Type *Ty); 195736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as IRBuilder::createSExt. 195837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *createSExt(Instruction *Inst, Value *Opnd, Type *Ty); 195937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Same as IRBuilder::createZExt. 196037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *createZExt(Instruction *Inst, Value *Opnd, Type *Ty); 196136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Same as Instruction::moveBefore. 196236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines void moveBefore(Instruction *Inst, Instruction *Before); 196336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// @} 196436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 196536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesprivate: 196636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The ordered list of actions made so far. 1967dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SmallVector<std::unique_ptr<TypePromotionAction>, 16> Actions; 1968dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines typedef SmallVectorImpl<std::unique_ptr<TypePromotionAction>>::iterator CommitPt; 196936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 197036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 197136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx, 197236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *NewVal) { 197336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.push_back( 1974dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines make_unique<TypePromotionTransaction::OperandSetter>(Inst, Idx, NewVal)); 197536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 197636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 197736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::eraseInstruction(Instruction *Inst, 197836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *NewVal) { 197936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.push_back( 1980dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines make_unique<TypePromotionTransaction::InstructionRemover>(Inst, NewVal)); 198136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 198236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 198336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst, 198436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *New) { 1985dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New)); 198636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 198736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 198836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) { 1989dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy)); 199036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 199136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 199237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesValue *TypePromotionTransaction::createTrunc(Instruction *Opnd, 199337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *Ty) { 1994dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty)); 199537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val = Ptr->getBuiltValue(); 1996dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(std::move(Ptr)); 199737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Val; 199836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 199936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 200037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesValue *TypePromotionTransaction::createSExt(Instruction *Inst, 200137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Opnd, Type *Ty) { 2002dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty)); 200337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val = Ptr->getBuiltValue(); 200437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Actions.push_back(std::move(Ptr)); 200537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Val; 200637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 200737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 200837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesValue *TypePromotionTransaction::createZExt(Instruction *Inst, 200937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Opnd, Type *Ty) { 201037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines std::unique_ptr<ZExtBuilder> Ptr(new ZExtBuilder(Inst, Opnd, Ty)); 201137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val = Ptr->getBuiltValue(); 2012dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Actions.push_back(std::move(Ptr)); 201337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Val; 201436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 201536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 201636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::moveBefore(Instruction *Inst, 201736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Before) { 201836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.push_back( 2019dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines make_unique<TypePromotionTransaction::InstructionMoveBefore>(Inst, Before)); 202036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 202136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 202236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesTypePromotionTransaction::ConstRestorationPt 202336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesTypePromotionTransaction::getRestorationPoint() const { 2024dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return !Actions.empty() ? Actions.back().get() : nullptr; 202536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 202636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 202736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::commit() { 202836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt; 2029dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++It) 203036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (*It)->commit(); 203136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Actions.clear(); 203236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 203336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 203436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesvoid TypePromotionTransaction::rollback( 203536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt Point) { 2036dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines while (!Actions.empty() && Point != Actions.back().get()) { 2037dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val(); 203836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Curr->undo(); 203936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 204036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 204136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 204236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \brief A helper class for matching addressing modes. 204336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 204436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// This encapsulates the logic for matching the target-legal addressing modes. 204536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass AddressingModeMatcher { 204636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<Instruction*> &AddrModeInsts; 2047ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetMachine &TM; 204836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI; 204936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 205036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and 205136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// the memory instruction that we're computing this address for. 205236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *AccessTy; 205336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *MemoryInst; 205436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 205536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// AddrMode - This is the addressing mode that we're building up. This is 205636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// part of the return value of this addressing mode matching stuff. 205736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AddrMode; 205836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 205936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The truncate instruction inserted by other CodeGenPrepare optimizations. 206036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SetOfInstrs &InsertedTruncs; 206136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// A map from the instructions to their type before promotion. 206236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts; 206336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The ongoing transaction where every action should be registered. 206436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT; 206536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 206636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// IgnoreProfitability - This is set to true when we should not do 206736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// profitability checks. When true, IsProfitableToFoldIntoAddressingMode 206836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// always returns true. 206936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IgnoreProfitability; 207036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2071ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines AddressingModeMatcher(SmallVectorImpl<Instruction *> &AMI, 2072ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetMachine &TM, Type *AT, Instruction *MI, 2073ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ExtAddrMode &AM, const SetOfInstrs &InsertedTruncs, 207436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 207536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT) 2076ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : AddrModeInsts(AMI), TM(TM), 2077ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TLI(*TM.getSubtargetImpl(*MI->getParent()->getParent()) 2078ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ->getTargetLowering()), 2079ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines AccessTy(AT), MemoryInst(MI), AddrMode(AM), 208036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTruncs(InsertedTruncs), PromotedInsts(PromotedInsts), TPT(TPT) { 208136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IgnoreProfitability = false; 208236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 208336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 208436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 208536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Match - Find the maximal addressing mode that a load/store of V can fold, 208636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// give an access type of AccessTy. This returns a list of involved 208736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// instructions in AddrModeInsts. 208836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p InsertedTruncs The truncate instruction inserted by other 208936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// CodeGenPrepare 209036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// optimizations. 209136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 209236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p The ongoing transaction where every action should be registered. 209336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines static ExtAddrMode Match(Value *V, Type *AccessTy, 209436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *MemoryInst, 209536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVectorImpl<Instruction*> &AddrModeInsts, 2096ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetMachine &TM, 209736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const SetOfInstrs &InsertedTruncs, 209836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy &PromotedInsts, 209936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction &TPT) { 210036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode Result; 210136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2102ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool Success = AddressingModeMatcher(AddrModeInsts, TM, AccessTy, 210336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryInst, Result, InsertedTruncs, 210436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts, TPT).MatchAddr(V, 0); 210536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (void)Success; assert(Success && "Couldn't select *anything*?"); 210636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Result; 210736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 210836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesprivate: 210936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MatchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth); 211036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MatchAddr(Value *V, unsigned Depth); 211136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MatchOperationAddr(User *Operation, unsigned Opcode, unsigned Depth, 2112dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool *MovedAway = nullptr); 211336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IsProfitableToFoldIntoAddressingMode(Instruction *I, 211436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AMBefore, 211536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AMAfter); 211636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool ValueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2); 21174c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool IsPromotionProfitable(unsigned NewCost, unsigned OldCost, 211836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *PromotedOperand) const; 211936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 212036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 212136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MatchScaledValue - Try adding ScaleReg*Scale to the current addressing mode. 212236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Return true and update AddrMode if this addr mode is legal for the target, 212336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// false if not. 212436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::MatchScaledValue(Value *ScaleReg, int64_t Scale, 212536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth) { 212636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If Scale is 1, then this is the same as adding ScaleReg to the addressing 212736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // mode. Just process that directly. 212836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Scale == 1) 212936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(ScaleReg, Depth); 213036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 213136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the scale is 0, it takes nothing to add this. 213236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Scale == 0) 213336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 213436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 213536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we already have a scale of this value, we can add to it, otherwise, we 213636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // need an available scale field. 213736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg) 213836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 213936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 214036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode TestAddrMode = AddrMode; 214136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 214236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add scale to turn X*4+X*3 -> X*7. This could also do things like 214336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // [A+B + A*7] -> [B+A*8]. 214436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.Scale += Scale; 214536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.ScaledReg = ScaleReg; 214636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 214736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the new address isn't legal, bail out. 214836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI.isLegalAddressingMode(TestAddrMode, AccessTy)) 214936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 215036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 215136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It was legal, so commit it. 215236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = TestAddrMode; 215336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 215436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Okay, we decided that we can add ScaleReg+Scale to AddrMode. Check now 215536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to see if ScaleReg is actually X+C. If so, we can turn this into adding 215636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // X*Scale + C*Scale to addr mode. 2157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantInt *CI = nullptr; Value *AddLHS = nullptr; 215836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<Instruction>(ScaleReg) && // not a constant expr. 215936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI)))) { 216036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.ScaledReg = AddLHS; 216136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale; 216236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 216336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this addressing mode is legal, commit it and remember that we folded 216436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // this instruction. 216536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(TestAddrMode, AccessTy)) { 216636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.push_back(cast<Instruction>(ScaleReg)); 216736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = TestAddrMode; 216836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 216936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 217036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 217136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 217236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, not (x+c)*scale, just return what we have. 217336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 217436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 217536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 217636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MightBeFoldableInst - This is a little filter, which returns true if an 217736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// addressing computation involving I might be folded into a load/store 217836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// accessing it. This doesn't need to be perfect, but needs to accept at least 217936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the set of instructions that MatchOperationAddr can. 218036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool MightBeFoldableInst(Instruction *I) { 218136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (I->getOpcode()) { 218236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::BitCast: 2183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Instruction::AddrSpaceCast: 218436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't touch identity bitcasts. 218536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (I->getType() == I->getOperand(0)->getType()) 218636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 218736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return I->getType()->isPointerTy() || I->getType()->isIntegerTy(); 218836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::PtrToInt: 218936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // PtrToInt is always a noop, as we know that the int type is pointer sized. 219036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 219136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::IntToPtr: 219236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We know the input is intptr_t, so this is foldable. 219336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 219436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Add: 219536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 219636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Mul: 219736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Shl: 219836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Can only handle X*C and X << C. 219936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return isa<ConstantInt>(I->getOperand(1)); 220036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::GetElementPtr: 220136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 220236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines default: 220336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 220436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 220536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 220636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2207ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Check whether or not \p Val is a legal instruction for \p TLI. 2208ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \note \p Val is assumed to be the product of some type promotion. 2209ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Therefore if \p Val has an undefined state in \p TLI, this is assumed 2210ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// to be legal, as the non-promoted value would have had the same state. 2211ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic bool isPromotedInstructionLegal(const TargetLowering &TLI, Value *Val) { 2212ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *PromotedInst = dyn_cast<Instruction>(Val); 2213ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!PromotedInst) 2214ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 2215ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode()); 2216ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If the ISDOpcode is undefined, it was undefined before the promotion. 2217ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!ISDOpcode) 2218ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 2219ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Otherwise, check if the promoted instruction is legal or not. 2220ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return TLI.isOperationLegalOrCustom( 2221ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ISDOpcode, TLI.getValueType(PromotedInst->getType())); 2222ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 2223ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 222436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \brief Hepler class to perform type promotion. 222536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesclass TypePromotionHelper { 222637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Utility function to check whether or not a sign or zero extension 222737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by 222837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// either using the operands of \p Inst or promoting \p Inst. 222937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// The type of the extension is defined by \p IsSExt. 223036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// In other words, check if: 223137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType. 223236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// #1 Promotion applies: 223337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...). 223436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// #2 Operand reuses: 223537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// ext opnd1 to ConsideredExtType. 223636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 223737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static bool canGetThrough(const Instruction *Inst, Type *ConsideredExtType, 223837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const InstrToOrigTy &PromotedInsts, bool IsSExt); 223936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 224036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \brief Utility function to determine if \p OpIdx should be promoted when 224136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// promoting \p Inst. 224237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static bool shouldExtOperand(const Instruction *Inst, int OpIdx) { 224336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<SelectInst>(Inst) && OpIdx == 0) 224436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 224536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 224636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 224736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 224837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Utility function to promote the operand of \p Ext when this 224937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// operand is a promotable trunc or sext or zext. 225036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 22514c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar /// \p CreatedInstsCost[out] contains the cost of all instructions 225237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// created to promote the operand of Ext. 2253ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// Newly added extensions are inserted in \p Exts. 2254ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// Newly added truncates are inserted in \p Truncs. 225536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Should never be called directly. 225637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \return The promoted value which is used instead of Ext. 2257ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static Value *promoteOperandForTruncAndAnyExt( 2258ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *Ext, TypePromotionTransaction &TPT, 22594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 2260ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVectorImpl<Instruction *> *Exts, 22614c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI); 226236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 226337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Utility function to promote the operand of \p Ext when this 226436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// operand is promotable and is not a supported trunc or sext. 226536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 22664c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar /// \p CreatedInstsCost[out] contains the cost of all the instructions 226737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// created to promote the operand of Ext. 2268ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// Newly added extensions are inserted in \p Exts. 2269ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines /// Newly added truncates are inserted in \p Truncs. 227036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Should never be called directly. 227137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \return The promoted value which is used instead of Ext. 22724c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar static Value *promoteOperandForOther(Instruction *Ext, 22734c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TypePromotionTransaction &TPT, 22744c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar InstrToOrigTy &PromotedInsts, 22754c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned &CreatedInstsCost, 22764c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Exts, 22774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Truncs, 22784c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const TargetLowering &TLI, bool IsSExt); 227937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 228037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \see promoteOperandForOther. 22814c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar static Value *signExtendOperandForOther( 22824c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Instruction *Ext, TypePromotionTransaction &TPT, 22834c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 22844c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Exts, 22854c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) { 22864c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost, 22874c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Exts, Truncs, TLI, true); 228837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 228937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 229037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \see promoteOperandForOther. 22914c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar static Value *zeroExtendOperandForOther( 22924c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Instruction *Ext, TypePromotionTransaction &TPT, 22934c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 22944c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Exts, 22954c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) { 22964c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost, 22974c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Exts, Truncs, TLI, false); 229837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 229936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 230036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinespublic: 230137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Type for the utility function that promotes the operand of Ext. 230237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines typedef Value *(*Action)(Instruction *Ext, TypePromotionTransaction &TPT, 23034c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar InstrToOrigTy &PromotedInsts, 23044c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned &CreatedInstsCost, 2305ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVectorImpl<Instruction *> *Exts, 23064c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Truncs, 23074c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const TargetLowering &TLI); 230837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Given a sign/zero extend instruction \p Ext, return the approriate 230937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// action to promote the operand of \p Ext instead of using Ext. 231036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \return NULL if no promotable action is possible with the current 231136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// sign extension. 231236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p InsertedTruncs keeps track of all the truncate instructions inserted by 231336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// the others CodeGenPrepare optimizations. This information is important 231436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// because we do not want to promote these instructions as CodeGenPrepare 231536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// will reinsert them later. Thus creating an infinite loop: create/remove. 231636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// \p PromotedInsts maps the instructions to their type before promotion. 231737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedTruncs, 231836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI, 231936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const InstrToOrigTy &PromotedInsts); 232036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines}; 232136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 232236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool TypePromotionHelper::canGetThrough(const Instruction *Inst, 232337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *ConsideredExtType, 232437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const InstrToOrigTy &PromotedInsts, 232537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsSExt) { 2326ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The promotion helper does not know how to deal with vector types yet. 2327ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // To be able to fix that, we would need to fix the places where we 2328ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // statically extend, e.g., constants and such. 2329ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Inst->getType()->isVectorTy()) 2330ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 2331ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 233237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We can always get through zext. 233337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isa<ZExtInst>(Inst)) 233437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 233537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 233637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // sext(sext) is ok too. 233737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (IsSExt && isa<SExtInst>(Inst)) 233836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 233936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 234036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We can get through binary operator, if it is legal. In other words, the 234136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // binary operator must have a nuw or nsw flag. 234236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Inst); 234336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (BinOp && isa<OverflowingBinaryOperator>(BinOp) && 234437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ((!IsSExt && BinOp->hasNoUnsignedWrap()) || 234537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (IsSExt && BinOp->hasNoSignedWrap()))) 234636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 234736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 234836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if we can do the following simplification. 234937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // ext(trunc(opnd)) --> ext(opnd) 235036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<TruncInst>(Inst)) 235136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 235236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 235336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *OpndVal = Inst->getOperand(0); 235437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check if we can use this operand in the extension. 235537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If the type is larger than the result type of the extension, 235636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // we cannot. 2357ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!OpndVal->getType()->isIntegerTy() || 2358ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OpndVal->getType()->getIntegerBitWidth() > 2359ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ConsideredExtType->getIntegerBitWidth()) 236036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 236136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 236236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the operand of the truncate is not an instruction, we will not have 236336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // any information on the dropped bits. 236436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // (Actually we could for constant but it is not worth the extra logic). 236536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *Opnd = dyn_cast<Instruction>(OpndVal); 236636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Opnd) 236736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 236836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 236936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if the source of the type is narrow enough. 237037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // I.e., check that trunc just drops extended bits of the same kind of 237137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the extension. 237237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // #1 get the type of the operand and check the kind of the extended bits. 237336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const Type *OpndType; 237436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd); 237537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (It != PromotedInsts.end() && It->second.IsSExt == IsSExt) 237637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines OpndType = It->second.Ty; 237737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd))) 237837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines OpndType = Opnd->getOperand(0)->getType(); 237936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 238036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 238136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 238237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // #2 check that the truncate just drop extended bits. 238336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Inst->getType()->getIntegerBitWidth() >= OpndType->getIntegerBitWidth()) 238436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 238536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 238636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 238736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 238836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 238936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesTypePromotionHelper::Action TypePromotionHelper::getAction( 239037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *Ext, const SetOfInstrs &InsertedTruncs, 239136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) { 239237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) && 239337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "Unexpected instruction type"); 239437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0)); 239537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *ExtTy = Ext->getType(); 239637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsSExt = isa<SExtInst>(Ext); 239737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If the operand of the extension is not an instruction, we cannot 239836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // get through. 239936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If it, check we can get through. 240037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt)) 2401dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 240236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 240336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do not promote if the operand has been added by codegenprepare. 240436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise, it means we are undoing an optimization that is likely to be 240536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // redone, thus causing potential infinite loop. 240637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isa<TruncInst>(ExtOpnd) && InsertedTruncs.count(ExtOpnd)) 2407dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 240836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 240936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // SExt or Trunc instructions. 241036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Return the related handler. 241137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) || 241237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isa<ZExtInst>(ExtOpnd)) 241337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return promoteOperandForTruncAndAnyExt; 241436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 241536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Regular instruction. 241636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Abort early if we will have to insert non-free instructions. 241737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType())) 2418dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 241937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther; 242036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 242136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 242237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesValue *TypePromotionHelper::promoteOperandForTruncAndAnyExt( 242336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines llvm::Instruction *SExt, TypePromotionTransaction &TPT, 24244c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 2425ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVectorImpl<Instruction *> *Exts, 24264c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) { 242736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // By construction, the operand of SExt is an instruction. Otherwise we cannot 242836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // get through it and this method should not be called. 242936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0)); 243037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *ExtVal = SExt; 24314c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool HasMergedNonFreeExt = false; 243237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isa<ZExtInst>(SExtOpnd)) { 243337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Replace s|zext(zext(opnd)) 243437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // => zext(opnd). 24354c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd); 243637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *ZExt = 243737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType()); 243837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.replaceAllUsesWith(SExt, ZExt); 243937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.eraseInstruction(SExt); 244037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtVal = ZExt; 244137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else { 244237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Replace z|sext(trunc(opnd)) or sext(sext(opnd)) 244337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // => z|sext(opnd). 244437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0)); 244537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 24464c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar CreatedInstsCost = 0; 244736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 244836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remove dead code. 244936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SExtOpnd->use_empty()) 245036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.eraseInstruction(SExtOpnd); 245136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 245237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check if the extension is still needed. 245337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *ExtInst = dyn_cast<Instruction>(ExtVal); 2454ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) { 24554c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (ExtInst) { 24564c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (Exts) 24574c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Exts->push_back(ExtInst); 24584c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt; 24594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar } 246037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return ExtVal; 2461ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 246236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 246337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // At this point we have: ext ty opnd to ty. 246437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Reassign the uses of ExtInst to the opnd and remove ExtInst. 246537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *NextVal = ExtInst->getOperand(0); 246637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.eraseInstruction(ExtInst, NextVal); 246736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return NextVal; 246836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 246936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 247037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesValue *TypePromotionHelper::promoteOperandForOther( 247137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *Ext, TypePromotionTransaction &TPT, 24724c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 2473ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVectorImpl<Instruction *> *Exts, 24744c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI, 24754c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool IsSExt) { 247637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // By construction, the operand of Ext is an instruction. Otherwise we cannot 247736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // get through it and this method should not be called. 247837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0)); 24794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar CreatedInstsCost = 0; 248037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ExtOpnd->hasOneUse()) { 248137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // ExtOpnd will be promoted. 248237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // All its uses, but Ext, will need to use a truncated value of the 248336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // promoted version. 248436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Create the truncate now. 248537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType()); 248637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) { 248737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ITrunc->removeFromParent(); 248837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Insert it just after the definition. 248937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ITrunc->insertAfter(ExtOpnd); 2490ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Truncs) 2491ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Truncs->push_back(ITrunc); 249237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 249336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 249437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.replaceAllUsesWith(ExtOpnd, Trunc); 249537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Restore the operand of Ext (which has been replace by the previous call 249636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext. 249737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.setOperand(Ext, 0, ExtOpnd); 249836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 249936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 250036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Get through the Instruction: 250136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // 1. Update its type. 250237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 2. Replace the uses of Ext by Inst. 250337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 3. Extend each operand that needs to be extended. 250436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 250536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember the original type of the instruction before promotion. 250636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This is useful to know that the high bits are sign extended bits. 250737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines PromotedInsts.insert(std::pair<Instruction *, TypeIsSExt>( 250837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtOpnd, TypeIsSExt(ExtOpnd->getType(), IsSExt))); 250936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Step #1. 251037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.mutateType(ExtOpnd, Ext->getType()); 251136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Step #2. 251237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.replaceAllUsesWith(Ext, ExtOpnd); 251336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Step #3. 251437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *ExtForOpnd = Ext; 251536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 251637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Propagate Ext to operands\n"); 251737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx; 251836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++OpIdx) { 251937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n'); 252037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() || 252137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !shouldExtOperand(ExtOpnd, OpIdx)) { 252236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "No need to propagate\n"); 252336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 252436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 252537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check if we can statically extend the operand. 252637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Opnd = ExtOpnd->getOperand(OpIdx); 252736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) { 252837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Statically extend\n"); 252937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned BitWidth = Ext->getType()->getIntegerBitWidth(); 253037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth) 253137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : Cst->getValue().zext(BitWidth); 253237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal)); 253336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 253436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 253536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // UndefValue are typed, so we have to statically sign extend them. 253636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<UndefValue>(Opnd)) { 253737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Statically extend\n"); 253837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType())); 253936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 254036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 254136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 254236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise we have to explicity sign extend the operand. 254337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check if Ext was reused to extend an operand. 254437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ExtForOpnd) { 254536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If yes, create a new one. 254637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "More operands to ext\n"); 2547ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *ValForExtOpnd = IsSExt ? TPT.createSExt(Ext, Opnd, Ext->getType()) 2548ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : TPT.createZExt(Ext, Opnd, Ext->getType()); 2549ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!isa<Instruction>(ValForExtOpnd)) { 2550ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd); 2551ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 2552ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 2553ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ExtForOpnd = cast<Instruction>(ValForExtOpnd); 255436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 2555ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Exts) 2556ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Exts->push_back(ExtForOpnd); 255737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.setOperand(ExtForOpnd, 0, Opnd); 255836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 255936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Move the sign extension before the insertion point. 256037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.moveBefore(ExtForOpnd, ExtOpnd); 256137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd); 25624c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar CreatedInstsCost += !TLI.isExtFree(ExtForOpnd); 256336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If more sext are required, new instructions will have to be created. 256437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtForOpnd = nullptr; 256536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 256637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ExtForOpnd == Ext) { 256737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Extension is useless now\n"); 256837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TPT.eraseInstruction(Ext); 256936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 257037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return ExtOpnd; 257136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 257236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 257336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsPromotionProfitable - Check whether or not promoting an instruction 257436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// to a wider type was profitable. 25754c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// \p NewCost gives the cost of extension instructions created by the 25764c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// promotion. 25774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// \p OldCost gives the cost of extension instructions before the promotion 25784c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// plus the number of instructions that have been 25794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar/// matched in the addressing mode the promotion. 258036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \p PromotedOperand is the value that has been promoted. 258136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// \return True if the promotion is profitable, false otherwise. 25824c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarbool AddressingModeMatcher::IsPromotionProfitable( 25834c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const { 25844c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost << '\n'); 25854c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // The cost of the new extensions is greater than the cost of the 25864c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // old extension plus what we folded. 258736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This is not profitable. 25884c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (NewCost > OldCost) 258936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 25904c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (NewCost < OldCost) 259136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 259236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The promotion is neutral but it may help folding the sign extension in 259336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // loads for instance. 259436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check that we did not create an illegal instruction. 2595ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return isPromotedInstructionLegal(TLI, PromotedOperand); 259636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 259736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 259836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MatchOperationAddr - Given an instruction or constant expr, see if we can 259936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// fold the operation into the addressing mode. If so, update the addressing 260036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// mode and return true, otherwise return false without modifying AddrMode. 260136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// If \p MovedAway is not NULL, it contains the information of whether or 260236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// not AddrInst has to be folded into the addressing mode on success. 260336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// If \p MovedAway == true, \p AddrInst will not be part of the addressing 260436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// because it has been moved away. 260536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Thus AddrInst must not be added in the matched instructions. 260636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// This state can happen when AddrInst is a sext, since it may be moved away. 260736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Therefore, AddrInst may not be valid when MovedAway is true and it must 260836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// not be referenced anymore. 260936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode, 261036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Depth, 261136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool *MovedAway) { 261236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Avoid exponential behavior on extremely deep expression trees. 261336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Depth >= 5) return false; 261436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 261536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // By default, all matched instructions stay in place. 261636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MovedAway) 261736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines *MovedAway = false; 261836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 261936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines switch (Opcode) { 262036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::PtrToInt: 262136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // PtrToInt is always a noop, as we know that the int type is pointer sized. 262236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(AddrInst->getOperand(0), Depth); 262336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::IntToPtr: 262436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This inttoptr is a no-op if the integer type is pointer sized. 262536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.getValueType(AddrInst->getOperand(0)->getType()) == 262636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI.getPointerTy(AddrInst->getType()->getPointerAddressSpace())) 262736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(AddrInst->getOperand(0), Depth); 262836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 262936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::BitCast: 2630dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case Instruction::AddrSpaceCast: 263136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // BitCast is always a noop, and we can handle it as long as it is 263236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // int->int or pointer->pointer (we don't want int<->fp or something). 263336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if ((AddrInst->getOperand(0)->getType()->isPointerTy() || 263436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrInst->getOperand(0)->getType()->isIntegerTy()) && 263536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Don't touch identity bitcasts. These were probably put here by LSR, 263636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // and we don't want to mess around with them. Assume it knows what it 263736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // is doing. 263836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrInst->getOperand(0)->getType() != AddrInst->getType()) 263936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchAddr(AddrInst->getOperand(0), Depth); 264036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 264136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Add: { 264236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if we can merge in the RHS then the LHS. If so, we win. 264336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 264436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 264536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Start a transaction at this point. 264636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The LHS may match but not the RHS. 264736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Therefore, we need a higher level restoration point to undo partially 264836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // matched operation. 264936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 265036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 265136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 265236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchAddr(AddrInst->getOperand(1), Depth+1) && 265336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MatchAddr(AddrInst->getOperand(0), Depth+1)) 265436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 265536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 265636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Restore the old addr mode info. 265736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 265836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 265936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 266036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 266136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise this was over-aggressive. Try merging in the LHS then the RHS. 266236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchAddr(AddrInst->getOperand(0), Depth+1) && 266336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MatchAddr(AddrInst->getOperand(1), Depth+1)) 266436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 266536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 266636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Otherwise we definitely can't merge the ADD in. 266736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 266836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 266936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 267036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 267136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 267236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines //case Instruction::Or: 267336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD. 267436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines //break; 267536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Mul: 267636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::Shl: { 267736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Can only handle X*C and X << C. 267836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1)); 267937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!RHS) 268037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 268136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t Scale = RHS->getSExtValue(); 268236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Opcode == Instruction::Shl) 268336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Scale = 1LL << Scale; 268436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 268536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MatchScaledValue(AddrInst->getOperand(0), Scale, Depth); 268636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 268736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines case Instruction::GetElementPtr: { 268836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Scan the GEP. We check it if it contains constant offsets and at most 268936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // one variable offset. 269036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int VariableOperand = -1; 269136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned VariableScale = 0; 269236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 269336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int64_t ConstantOffset = 0; 269436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const DataLayout *TD = TLI.getDataLayout(); 269536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines gep_type_iterator GTI = gep_type_begin(AddrInst); 269636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) { 269736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StructType *STy = dyn_cast<StructType>(*GTI)) { 269836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const StructLayout *SL = TD->getStructLayout(STy); 269936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned Idx = 270036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue(); 270136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantOffset += SL->getElementOffset(Idx); 270236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 270336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines uint64_t TypeSize = TD->getTypeAllocSize(GTI.getIndexedType()); 270436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) { 270536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantOffset += CI->getSExtValue()*TypeSize; 270636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (TypeSize) { // Scales of zero don't do anything. 270736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We only allow one variable index at the moment. 270836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VariableOperand != -1) 270936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 271036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 271136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Remember the variable index. 271236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VariableOperand = i; 271336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VariableScale = TypeSize; 271436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 271536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 271636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 271736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 271836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // A common case is for the GEP to only do a constant offset. In this case, 271936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // just add it to the disp field and check validity. 272036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VariableOperand == -1) { 272136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += ConstantOffset; 272236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstantOffset == 0 || TLI.isLegalAddressingMode(AddrMode, AccessTy)){ 272336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if we can fold the base pointer in too. 272436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchAddr(AddrInst->getOperand(0), Depth+1)) 272536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 272636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 272736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs -= ConstantOffset; 272836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 272936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 273036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 273136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Save the valid addressing mode in case we can't match. 273236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 273336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 273436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 273536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // See if the scale and offset amount is valid for this target. 273636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += ConstantOffset; 273736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 273836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Match the base operand of the GEP. 273936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchAddr(AddrInst->getOperand(0), Depth+1)) { 274036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If it couldn't be matched, just stuff the value in a register. 274136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.HasBaseReg) { 274236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 274336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 274436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 274536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 274636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = true; 274736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseReg = AddrInst->getOperand(0); 274836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 274936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 275036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Match the remaining variable portion of the GEP. 275136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale, 275236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Depth)) { 275336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If it couldn't be matched, try stuffing the base into a register 275436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // instead of matching it, and retrying the match of the scale. 275536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 275636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 275736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.HasBaseReg) 275836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 275936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = true; 276036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseReg = AddrInst->getOperand(0); 276136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += ConstantOffset; 276236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchScaledValue(AddrInst->getOperand(VariableOperand), 276336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines VariableScale, Depth)) { 276436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If even that didn't work, bail. 276536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 276636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 276736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 276836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 276936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 277036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 277136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 277236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 277337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::SExt: 277437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::ZExt: { 277537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *Ext = dyn_cast<Instruction>(AddrInst); 277637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!Ext) 277737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 277837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 277937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Try to move this ext out of the way of the addressing mode. 278036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Ask for a method for doing so. 278137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TypePromotionHelper::Action TPH = 278237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TypePromotionHelper::getAction(Ext, InsertedTruncs, TLI, PromotedInsts); 278336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TPH) 278436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 278536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 278636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 278736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 27884c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned CreatedInstsCost = 0; 27894c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned ExtCost = !TLI.isExtFree(Ext); 2790ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *PromotedOperand = 27914c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI); 279236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // SExt has been moved away. 279336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Thus either it will be rematched later in the recursive calls or it is 279436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // gone. Anyway, we must not fold it into the addressing mode at this point. 279536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // E.g., 279636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // op = add opnd, 1 279737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // idx = ext op 279836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // addr = gep base, idx 279936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // is now: 280037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // promotedOpnd = ext opnd <- no match here 280136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls) 280236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // addr = gep base, op <- match 280336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MovedAway) 280436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines *MovedAway = true; 280536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 280636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(PromotedOperand && 280736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "TypePromotionHelper should have filtered out those cases"); 280836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 280936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 281036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 281136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 281236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MatchAddr(PromotedOperand, Depth) || 28134c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // The total of the new cost is equals to the cost of the created 28144c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // instructions. 28154c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // The total of the old cost is equals to the cost of the extension plus 28164c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // what we have saved in the addressing mode. 28174c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar !IsPromotionProfitable(CreatedInstsCost, 28184c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar ExtCost + (AddrModeInsts.size() - OldSize), 281936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedOperand)) { 282036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 282136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 282236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Sign extension does not pay off: rollback\n"); 282336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 282436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 282536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 282636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 282736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 282836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 282936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 283036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 283136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 283236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MatchAddr - If we can, try to add the value of 'Addr' into the current 283336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// addressing mode. If Addr can't be added to AddrMode this returns false and 283436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// leaves AddrMode unmodified. This assumes that Addr is either a pointer type 283536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// or intptr_t for the target. 283636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 283736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::MatchAddr(Value *Addr, unsigned Depth) { 283836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Start a transaction at this point that we will rollback if the matching 283936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // fails. 284036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 284136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 284236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) { 284336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Fold in immediates if legal for the target. 284436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs += CI->getSExtValue(); 284536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 284636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 284736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseOffs -= CI->getSExtValue(); 284836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) { 284936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is a global variable, try to fold it into the addressing mode. 2850dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!AddrMode.BaseGV) { 285136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseGV = GV; 285236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 285336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 2854dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseGV = nullptr; 285536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 285636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (Instruction *I = dyn_cast<Instruction>(Addr)) { 285736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode BackupAddrMode = AddrMode; 285836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OldSize = AddrModeInsts.size(); 285936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 286036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if it is possible to fold this operation. 286136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MovedAway = false; 286236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) { 286336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This instruction may have been move away. If so, there is nothing 286436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to check here. 286536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MovedAway) 286636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 286736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Okay, it's possible to fold this. Check to see if it is actually 286836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // *profitable* to do so. We use a simple cost model to avoid increasing 286936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // register pressure too much. 287036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (I->hasOneUse() || 287136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IsProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) { 287236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.push_back(I); 287336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 287436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 287536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 287636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It isn't profitable to do this, roll back. 287736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines //cerr << "NOT FOLDING: " << *I; 287836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = BackupAddrMode; 287936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts.resize(OldSize); 288036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 288136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 288236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) { 288336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (MatchOperationAddr(CE, CE->getOpcode(), Depth)) 288436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 288536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 288636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (isa<ConstantPointerNull>(Addr)) { 288736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Null pointer gets folded without affecting the addressing mode. 288836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 288936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 289036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 289136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Worse case, the target should support [reg] addressing modes. :) 289236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!AddrMode.HasBaseReg) { 289336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = true; 289436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.BaseReg = Addr; 289536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Still check for legality in case the target supports [imm] but not [i+r]. 289636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 289736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 289836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.HasBaseReg = false; 2899dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseReg = nullptr; 290036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 290136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 290236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the base register is already taken, see if we can do [r+r]. 290336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale == 0) { 290436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.Scale = 1; 290536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.ScaledReg = Addr; 290636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI.isLegalAddressingMode(AddrMode, AccessTy)) 290736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 290836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode.Scale = 0; 2909dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.ScaledReg = nullptr; 291036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 291136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Couldn't match. 291236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 291336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 291436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 291536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 291636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsOperandAMemoryOperand - Check to see if all uses of OpVal by the specified 291736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// inline asm call are due to memory operands. If so, return true, otherwise 291836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// return false. 291936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal, 2920ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetMachine &TM) { 2921ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const Function *F = CI->getParent()->getParent(); 2922ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering(); 2923ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetRegisterInfo *TRI = TM.getSubtargetImpl(*F)->getRegisterInfo(); 2924ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TargetLowering::AsmOperandInfoVector TargetConstraints = 2925ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TLI->ParseConstraints(TRI, ImmutableCallSite(CI)); 292636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 292736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 292836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 292936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Compute the constraint code and ConstraintType to use. 2930ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TLI->ComputeConstraintToUse(OpInfo, SDValue()); 293136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 293236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this asm operand is our Value*, and if it isn't an indirect memory 293336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // operand, we can't fold it! 293436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OpInfo.CallOperandVal == OpVal && 293536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (OpInfo.ConstraintType != TargetLowering::C_Memory || 293636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !OpInfo.isIndirect)) 293736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 293836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 293936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 294036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 294136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 294236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 294336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// FindAllMemoryUses - Recursively walk all the uses of I until we find a 294436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// memory use. If we find an obviously non-foldable instruction, return true. 294536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Add the ultimately found memory instructions to MemoryUses. 2946ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic bool FindAllMemoryUses( 2947ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *I, 2948ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVectorImpl<std::pair<Instruction *, unsigned>> &MemoryUses, 2949ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallPtrSetImpl<Instruction *> &ConsideredInsts, const TargetMachine &TM) { 295036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we already considered this instruction, we're done. 295137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ConsideredInsts.insert(I).second) 295236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 295336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 295436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is an obviously unfoldable instruction, bail out. 295536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!MightBeFoldableInst(I)) 295636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 295736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 295836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Loop over all the uses, recursively processing them. 295936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Use &U : I->uses()) { 296036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UserI = cast<Instruction>(U.getUser()); 296136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 296236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) { 296336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryUses.push_back(std::make_pair(LI, U.getOperandNo())); 296436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 296536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 296636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 296736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) { 296836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned opNo = U.getOperandNo(); 296936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (opNo == 0) return true; // Storing addr, not into addr. 297036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryUses.push_back(std::make_pair(SI, opNo)); 297136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 297236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 297336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 297436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CallInst *CI = dyn_cast<CallInst>(UserI)) { 297536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue()); 297636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!IA) return true; 297736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 297836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If this is a memory operand, we're cool, otherwise bail out. 2979ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!IsOperandAMemoryOperand(CI, IA, I, TM)) 298036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 298136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 298236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 298336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 2984ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TM)) 298536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 298636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 298736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 298836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 298936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 299036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 299136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// ValueAlreadyLiveAtInst - Retrn true if Val is already known to be live at 299236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// the use site that we're folding it into. If so, there is no cost to 299336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// include it in the addressing mode. KnownLive1 and KnownLive2 are two values 299436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// that we know are live at the instruction already. 299536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher::ValueAlreadyLiveAtInst(Value *Val,Value *KnownLive1, 299636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *KnownLive2) { 299736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If Val is either of the known-live values, we know it is live! 2998dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2) 299936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 300036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 300136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // All values other than instructions and arguments (e.g. constants) are live. 300236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<Instruction>(Val) && !isa<Argument>(Val)) return true; 300336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 300436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If Val is a constant sized alloca in the entry block, it is live, this is 300536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // true because it is just a reference to the stack/frame pointer, which is 300636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // live for the whole function. 300736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AllocaInst *AI = dyn_cast<AllocaInst>(Val)) 300836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AI->isStaticAlloca()) 300936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 301036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 301136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if this value is already used in the memory instruction's 301236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // block. If so, it's already live into the block at the very least, so we 301336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // can reasonably fold it. 301436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Val->isUsedInBasicBlock(MemoryInst->getParent()); 301536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 301636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 301736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsProfitableToFoldIntoAddressingMode - It is possible for the addressing 301836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// mode of the machine to fold the specified instruction into a load or store 301936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// that ultimately uses it. However, the specified instruction has multiple 302036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// uses. Given this, it may actually increase register pressure to fold it 302136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// into the load. For example, consider this code: 302236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 302336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// X = ... 302436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Y = X+1 302536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// use(Y) -> nonload/store 302636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Z = Y+1 302736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// load Z 302836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 302936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// In this case, Y has multiple uses, and can be folded into the load of Z 303036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (yielding load [X+2]). However, doing this will cause both "X" and "X+1" to 303136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// be live at the use(Y) line. If we don't fold Y into load Z, we use one 303236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// fewer register. Since Y can't be folded into "use(Y)" we don't increase the 303336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// number of computations either. 303436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 303536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Note that this (like most of CodeGenPrepare) is just a rough heuristic. If 303636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// X was live across 'load Z' for other reasons, we actually *would* want to 303736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// fold the addressing mode in the Z case. This would make Y die earlier. 303836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool AddressingModeMatcher:: 303936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesIsProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore, 304036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode &AMAfter) { 304136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IgnoreProfitability) return true; 304236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 304336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // AMBefore is the addressing mode before this instruction was folded into it, 304436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // and AMAfter is the addressing mode after the instruction was folded. Get 304536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the set of registers referenced by AMAfter and subtract out those 304636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // referenced by AMBefore: this is the set of values which folding in this 304736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // address extends the lifetime of. 304836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // 304936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Note that there are only two potential values being referenced here, 305036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // BaseReg and ScaleReg (global addresses are always available, as are any 305136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // folded immediates). 305236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg; 305336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 305436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the BaseReg or ScaledReg was referenced by the previous addrmode, their 305536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // lifetime wasn't extended by adding this instruction. 305636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ValueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg)) 3057dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BaseReg = nullptr; 305836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ValueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg)) 3059dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ScaledReg = nullptr; 306036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 306136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If folding this instruction (and it's subexprs) didn't extend any live 306236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // ranges, we're ok with it. 3063dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!BaseReg && !ScaledReg) 306436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 306536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 306636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If all uses of this instruction are ultimately load/store/inlineasm's, 306736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // check to see if their addressing modes will include this instruction. If 306836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // so, we can fold it into all uses, so it doesn't matter if it has multiple 306936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // uses. 307036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<std::pair<Instruction*,unsigned>, 16> MemoryUses; 307136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<Instruction*, 16> ConsideredInsts; 3072ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TM)) 307336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; // Has a non-memory, non-foldable use! 307436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 307536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Now that we know that all uses of this instruction are part of a chain of 307636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // computation involving only operations that could theoretically be folded 307736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // into a memory use, loop over each of these uses and see if they could 307836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // *actually* fold the instruction. 307936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Instruction*, 32> MatchedAddrModeInsts; 308036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = MemoryUses.size(); i != e; ++i) { 308136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = MemoryUses[i].first; 308236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned OpNo = MemoryUses[i].second; 308336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 308436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Get the access type of this use. If the use isn't a pointer, we don't 308536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // know what it accesses. 308636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Address = User->getOperand(OpNo); 308736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Address->getType()->isPointerTy()) 308836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 308936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *AddressAccessTy = Address->getType()->getPointerElementType(); 309036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 309136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do a match against the root of this address, ignoring profitability. This 309236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // will tell us if the addressing mode for the memory operation will 309336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // *actually* cover the shared instruction. 309436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode Result; 309536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 309636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 3097ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines AddressingModeMatcher Matcher(MatchedAddrModeInsts, TM, AddressAccessTy, 309836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryInst, Result, InsertedTruncs, 309936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts, TPT); 310036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Matcher.IgnoreProfitability = true; 310136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool Success = Matcher.MatchAddr(Address, 0); 310236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (void)Success; assert(Success && "Couldn't select *anything*?"); 310336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 310436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The match was to check the profitability, the changes made are not 310536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // part of the original matcher. Therefore, they should be dropped 310636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // otherwise the original matcher will not present the right state. 310736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 310836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 310936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the match didn't cover I, then it won't be shared by it. 311036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (std::find(MatchedAddrModeInsts.begin(), MatchedAddrModeInsts.end(), 311136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I) == MatchedAddrModeInsts.end()) 311236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 311336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 311436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MatchedAddrModeInsts.clear(); 311536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 311636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 311736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 311836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 311936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 312036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} // end anonymous namespace 312136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 312236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// IsNonLocalValue - Return true if the specified values are defined in a 312336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// different basic block than BB. 312436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool IsNonLocalValue(Value *V, BasicBlock *BB) { 312536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Instruction *I = dyn_cast<Instruction>(V)) 312636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return I->getParent() != BB; 312736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 312836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 312936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 313036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeMemoryInst - Load and Store Instructions often have 313136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// addressing modes that can do significant amounts of computation. As such, 313236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// instruction selection will try to get the load or store to do as much 313336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// computation as possible for the program. The problem is that isel can only 313436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// see within a single block. As such, we sink as much legal addressing mode 313536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// stuff into the block as possible. 313636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 313736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// This method is used to optimize both load/store and inline asms with memory 313836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// operands. 313936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr, 314036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *AccessTy) { 314136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Repl = Addr; 314236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 314336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Try to collapse single-value PHI nodes. This is necessary to undo 314436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // unprofitable PRE transformations. 314536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Value*, 8> worklist; 314636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallPtrSet<Value*, 16> Visited; 314736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines worklist.push_back(Addr); 314836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 314936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use a worklist to iteratively look through PHI nodes, and ensure that 315036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the addressing mode obtained from the non-PHI roots of the graph 315136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // are equivalent. 3152dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Consensus = nullptr; 315336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumUsesConsensus = 0; 315436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool IsNumUsesConsensusValid = false; 315536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Instruction*, 16> AddrModeInsts; 315636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode AddrMode; 315736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction TPT; 315836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 315936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.getRestorationPoint(); 316036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines while (!worklist.empty()) { 316136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = worklist.back(); 316236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines worklist.pop_back(); 316336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 316436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Break use-def graph loops. 316537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!Visited.insert(V).second) { 3166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Consensus = nullptr; 316736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 316836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 316936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 317036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // For a PHI node, push all of its incoming values. 317136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *P = dyn_cast<PHINode>(V)) { 317236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = P->getNumIncomingValues(); i != e; ++i) 317336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines worklist.push_back(P->getIncomingValue(i)); 317436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 317536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 317636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 317736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // For non-PHIs, determine the addressing mode being computed. 317836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<Instruction*, 16> NewAddrModeInsts; 317936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ExtAddrMode NewAddrMode = AddressingModeMatcher::Match( 3180ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines V, AccessTy, MemoryInst, NewAddrModeInsts, *TM, InsertedTruncsSet, 318136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PromotedInsts, TPT); 318236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 318336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This check is broken into two cases with very similar code to avoid using 318436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // getNumUses() as much as possible. Some values have a lot of uses, so 318536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // calling getNumUses() unconditionally caused a significant compile-time 318636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // regression. 318736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Consensus) { 318836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Consensus = V; 318936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrMode = NewAddrMode; 319036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts = NewAddrModeInsts; 319136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 319236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (NewAddrMode == AddrMode) { 319336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!IsNumUsesConsensusValid) { 319436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NumUsesConsensus = Consensus->getNumUses(); 319536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IsNumUsesConsensusValid = true; 319636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 319736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 319836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Ensure that the obtained addressing mode is equivalent to that obtained 319936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // for all other roots of the PHI traversal. Also, when choosing one 320036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // such root as representative, select the one with the most uses in order 320136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to keep the cost modeling heuristics in AddressingModeMatcher 320236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // applicable. 320336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned NumUses = V->getNumUses(); 320436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (NumUses > NumUsesConsensus) { 320536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Consensus = V; 320636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NumUsesConsensus = NumUses; 320736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AddrModeInsts = NewAddrModeInsts; 320836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 320936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 321036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 321136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3212dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Consensus = nullptr; 321336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 321436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 321536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 321636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the addressing mode couldn't be determined, or if multiple different 321736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // ones were determined, bail out now. 321836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Consensus) { 321936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.rollback(LastKnownGood); 322036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 322136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 322236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TPT.commit(); 322336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 322436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check to see if any of the instructions supersumed by this addr mode are 322536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // non-local to I's BB. 322636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool AnyNonLocal = false; 322736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = AddrModeInsts.size(); i != e; ++i) { 322836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IsNonLocalValue(AddrModeInsts[i], MemoryInst->getParent())) { 322936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines AnyNonLocal = true; 323036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 323136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 323236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 323336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 323436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If all the instructions matched are already in this BB, don't do anything. 323536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!AnyNonLocal) { 323636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode << "\n"); 323736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 323836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 323936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 324036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Insert this computation right after this user. Since our caller is 324136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // scanning from the top of the BB to the bottom, reuse of the expr are 324236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // guaranteed to happen later. 324336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IRBuilder<> Builder(MemoryInst); 324436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 324536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Now that we determined the addressing expression we want to use and know 324636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // that we have to sink it into this block. Check to see if we have already 324736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // done this for some other load/store instr in this block. If so, reuse the 324836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // computation. 324936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *&SunkAddr = SunkAddrs[Addr]; 325036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SunkAddr) { 325136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for " 3252dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines << *MemoryInst << "\n"); 325336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SunkAddr->getType() != Addr->getType()) 325436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType()); 3255ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } else if (AddrSinkUsingGEPs || 3256ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines (!AddrSinkUsingGEPs.getNumOccurrences() && TM && 3257ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TM->getSubtargetImpl(*MemoryInst->getParent()->getParent()) 3258ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ->useAA())) { 3259dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // By default, we use the GEP-based method when AA is used later. This 3260dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities. 3261dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for " 3262dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines << *MemoryInst << "\n"); 3263dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(Addr->getType()); 3264dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *ResultPtr = nullptr, *ResultIndex = nullptr; 3265dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3266dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // First, find the pointer. 3267dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) { 3268dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = AddrMode.BaseReg; 3269dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseReg = nullptr; 3270dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3271dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3272dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) { 3273dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // We can't add more than one pointer together, nor can we scale a 3274dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // pointer (both of which seem meaningless). 3275dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr || AddrMode.Scale != 1) 3276dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 3277dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3278dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = AddrMode.ScaledReg; 3279dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.Scale = 0; 3280dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3281dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3282dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseGV) { 3283dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr) 3284dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 3285dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3286dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = AddrMode.BaseGV; 3287dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3288dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3289dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // If the real base value actually came from an inttoptr, then the matcher 3290dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // will look through it and provide only the integer value. In that case, 3291dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // use it here. 3292dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ResultPtr && AddrMode.BaseReg) { 3293dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = 3294dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(), "sunkaddr"); 3295dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.BaseReg = nullptr; 3296dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (!ResultPtr && AddrMode.Scale == 1) { 3297dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = 3298dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(), "sunkaddr"); 3299dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines AddrMode.Scale = 0; 3300dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3301dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3302dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ResultPtr && 3303dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) { 3304dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SunkAddr = Constant::getNullValue(Addr->getType()); 3305dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (!ResultPtr) { 3306dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 3307dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3308dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Type *I8PtrTy = 33092c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace()); 33102c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar Type *I8Ty = Builder.getInt8Ty(); 3311dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3312dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Start with the base register. Do this first so that subsequent address 3313dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // matching finds it last, which will prevent it from trying to match it 3314dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // as the scaled value in case it happens to be a mul. That would be 3315dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // problematic if we've sunk a different mul for the scale, because then 3316dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // we'd end up sinking both muls. 3317dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseReg) { 3318dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *V = AddrMode.BaseReg; 3319dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (V->getType() != IntPtrTy) 3320dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr"); 3321dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3322dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = V; 3323dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3324dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3325dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add the scale value. 3326dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.Scale) { 3327dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *V = AddrMode.ScaledReg; 3328dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (V->getType() == IntPtrTy) { 3329dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // done. 3330dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < 3331dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines cast<IntegerType>(V->getType())->getBitWidth()) { 3332dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); 3333dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3334dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // It is only safe to sign extend the BaseReg if we know that the math 3335dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // required to create it did not overflow before we extend it. Since 3336dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the original IR value was tossed in favor of a constant back when 3337dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // the AddrMode was created we need to bail out gracefully if widths 3338dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // do not match instead of extending it. 3339dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *I = dyn_cast_or_null<Instruction>(ResultIndex); 3340dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (I && (ResultIndex != AddrMode.BaseReg)) 3341dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines I->eraseFromParent(); 3342dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 3343dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3344dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3345dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.Scale != 1) 3346dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale), 3347dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines "sunkaddr"); 3348dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultIndex) 3349dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr"); 3350dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines else 3351dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = V; 3352dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3353dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3354dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Add in the Base Offset if present. 3355dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (AddrMode.BaseOffs) { 3356dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); 3357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultIndex) { 335837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We need to add this separately from the scale above to help with 335937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // SDAG consecutive load/store merging. 3360dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr->getType() != I8PtrTy) 3361dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy); 33622c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); 3363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultIndex = V; 3366dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3368dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!ResultIndex) { 3369dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SunkAddr = ResultPtr; 3370dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } else { 3371dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (ResultPtr->getType() != I8PtrTy) 3372dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy); 33732c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); 3374dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 3375dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 3376dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (SunkAddr->getType() != Addr->getType()) 3377dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType()); 3378dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 337936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 338036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for " 3381dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines << *MemoryInst << "\n"); 338236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(Addr->getType()); 3383dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Value *Result = nullptr; 338436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 338536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Start with the base register. Do this first so that subsequent address 338636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // matching finds it last, which will prevent it from trying to match it 338736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // as the scaled value in case it happens to be a mul. That would be 338836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // problematic if we've sunk a different mul for the scale, because then 338936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // we'd end up sinking both muls. 339036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.BaseReg) { 339136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = AddrMode.BaseReg; 339236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V->getType()->isPointerTy()) 339336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr"); 339436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V->getType() != IntPtrTy) 339536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr"); 339636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 339736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 339836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 339936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add the scale value. 340036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale) { 340136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = AddrMode.ScaledReg; 340236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (V->getType() == IntPtrTy) { 340336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // done. 340436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (V->getType()->isPointerTy()) { 340536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr"); 340636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < 340736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines cast<IntegerType>(V->getType())->getBitWidth()) { 340836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); 340936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 341036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It is only safe to sign extend the BaseReg if we know that the math 341136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // required to create it did not overflow before we extend it. Since 341236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the original IR value was tossed in favor of a constant back when 341336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the AddrMode was created we need to bail out gracefully if widths 341436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // do not match instead of extending it. 3415dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *I = dyn_cast_or_null<Instruction>(Result); 3416dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (I && (Result != AddrMode.BaseReg)) 3417dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines I->eraseFromParent(); 341836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 341936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 342036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.Scale != 1) 342136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale), 342236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "sunkaddr"); 342336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Result) 342436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = Builder.CreateAdd(Result, V, "sunkaddr"); 342536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 342636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 342736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 342836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 342936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add in the BaseGV if present. 343036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.BaseGV) { 343136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr"); 343236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Result) 343336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = Builder.CreateAdd(Result, V, "sunkaddr"); 343436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 343536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 343636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 343736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 343836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Add in the Base Offset if present. 343936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (AddrMode.BaseOffs) { 344036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); 344136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Result) 344236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = Builder.CreateAdd(Result, V, "sunkaddr"); 344336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 344436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Result = V; 344536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 344636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3447dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!Result) 344836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddr = Constant::getNullValue(Addr->getType()); 344936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 345036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr"); 345136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 345236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 345336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MemoryInst->replaceUsesOfWith(Repl, SunkAddr); 345436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 345536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we have no uses, recursively delete the value and all dead instructions 345636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // using it. 345736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Repl->use_empty()) { 345836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // This can cause recursive deletion, which can invalidate our iterator. 345936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Use a WeakVH to hold onto it in case this happens. 346036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines WeakVH IterHandle(CurInstIterator); 346136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = CurInstIterator->getParent(); 346236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 346336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines RecursivelyDeleteTriviallyDeadInstructions(Repl, TLInfo); 346436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 346536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (IterHandle != CurInstIterator) { 346636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the iterator instruction was recursively deleted, start over at the 346736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // start of the block. 346836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = BB->begin(); 346936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 347036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 347136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 347236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumMemoryInsts; 347336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 347436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 347536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 347636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeInlineAsmInst - If there are any memory operands, use 347736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// OptimizeMemoryInst to sink their address computing into the block when 347836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// possible / profitable. 347936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeInlineAsmInst(CallInst *CS) { 348036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 348136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3482ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const TargetRegisterInfo *TRI = 3483ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TM->getSubtargetImpl(*CS->getParent()->getParent())->getRegisterInfo(); 348436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::AsmOperandInfoVector 3485ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TargetConstraints = TLI->ParseConstraints(TRI, CS); 348636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned ArgNo = 0; 348736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 348836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 348936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 349036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Compute the constraint code and ConstraintType to use. 349136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->ComputeConstraintToUse(OpInfo, SDValue()); 349236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 349336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (OpInfo.ConstraintType == TargetLowering::C_Memory && 349436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines OpInfo.isIndirect) { 349536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *OpVal = CS->getArgOperand(ArgNo++); 349636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= OptimizeMemoryInst(CS, OpVal, OpVal->getType()); 349736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else if (OpInfo.Type == InlineAsm::isInput) 349836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ArgNo++; 349936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 350036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 350136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 350236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 350336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3504ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Check if all the uses of \p Inst are equivalent (or free) zero or 3505ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// sign extensions. 3506ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic bool hasSameExtUse(Instruction *Inst, const TargetLowering &TLI) { 3507ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(!Inst->use_empty() && "Input must have at least one use"); 3508ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const Instruction *FirstUser = cast<Instruction>(*Inst->user_begin()); 3509ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool IsSExt = isa<SExtInst>(FirstUser); 3510ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *ExtTy = FirstUser->getType(); 3511ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (const User *U : Inst->users()) { 3512ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const Instruction *UI = cast<Instruction>(U); 3513ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI))) 3514ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 3515ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *CurTy = UI->getType(); 3516ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Same input and output types: Same instruction after CSE. 3517ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (CurTy == ExtTy) 3518ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 3519ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 3520ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If IsSExt is true, we are in this situation: 3521ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // a = Inst 3522ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // b = sext ty1 a to ty2 3523ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // c = sext ty1 a to ty3 3524ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Assuming ty2 is shorter than ty3, this could be turned into: 3525ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // a = Inst 3526ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // b = sext ty1 a to ty2 3527ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // c = sext ty2 b to ty3 3528ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // However, the last sext is not free. 3529ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (IsSExt) 3530ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 3531ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 3532ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // This is a ZExt, maybe this is free to extend from one type to another. 3533ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // In that case, we would not account for a different use. 3534ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *NarrowTy; 3535ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *LargeTy; 3536ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (ExtTy->getScalarType()->getIntegerBitWidth() > 3537ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines CurTy->getScalarType()->getIntegerBitWidth()) { 3538ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NarrowTy = CurTy; 3539ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LargeTy = ExtTy; 3540ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } else { 3541ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NarrowTy = ExtTy; 3542ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LargeTy = CurTy; 3543ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 3544ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 3545ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!TLI.isZExtFree(NarrowTy, LargeTy)) 3546ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 3547ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 3548ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // All uses are the same or can be derived from one another for free. 3549ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 3550ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 3551ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 3552ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Try to form ExtLd by promoting \p Exts until they reach a 3553ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// load instruction. 3554ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// If an ext(load) can be formed, it is returned via \p LI for the load 3555ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// and \p Inst for the extension. 3556ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Otherwise LI == nullptr and Inst == nullptr. 3557ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// When some promotion happened, \p TPT contains the proper state to 3558ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// revert them. 3559ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// 3560ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \return true when promoting was necessary to expose the ext(load) 3561ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// opportunity, false otherwise. 3562ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// 3563ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Example: 3564ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \code 3565ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %ld = load i32* %addr 3566ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %add = add nuw i32 %ld, 4 3567ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %zext = zext i32 %add to i64 3568ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \endcode 3569ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// => 3570ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \code 3571ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %ld = load i32* %addr 3572ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %zext = zext i32 %ld to i64 3573ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %add = add nuw i64 %zext, 4 3574ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \encode 3575ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// Thanks to the promotion, we can match zext(load i32*) to i64. 3576ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool CodeGenPrepare::ExtLdPromotion(TypePromotionTransaction &TPT, 3577ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LoadInst *&LI, Instruction *&Inst, 3578ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const SmallVectorImpl<Instruction *> &Exts, 35794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned CreatedInstsCost = 0) { 3580ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Iterate over all the extensions to see if one form an ext(load). 3581ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto I : Exts) { 3582ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Check if we directly have ext(load). 3583ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if ((LI = dyn_cast<LoadInst>(I->getOperand(0)))) { 3584ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Inst = I; 3585ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // No promotion happened here. 3586ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 3587ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 3588ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Check whether or not we want to do any promotion. 3589ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!TLI || !TLI->enableExtLdPromotion() || DisableExtLdPromotion) 3590ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 3591ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Get the action to perform the promotion. 3592ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TypePromotionHelper::Action TPH = TypePromotionHelper::getAction( 3593ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines I, InsertedTruncsSet, *TLI, PromotedInsts); 3594ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Check if we can promote. 3595ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!TPH) 3596ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 3597ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Save the current state. 3598ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 3599ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.getRestorationPoint(); 3600ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<Instruction *, 4> NewExts; 36014c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned NewCreatedInstsCost = 0; 36024c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar unsigned ExtCost = !TLI->isExtFree(I); 3603ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Promote. 36044c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost, 36054c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar &NewExts, nullptr, *TLI); 3606ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(PromotedVal && 3607ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "TypePromotionHelper should have filtered out those cases"); 3608ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 3609ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // We would be able to merge only one extension in a load. 3610ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Therefore, if we have more than 1 new extension we heuristically 3611ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // cut this search path, because it means we degrade the code quality. 3612ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // With exactly 2, the transformation is neutral, because we will merge 3613ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // one extension but leave one. However, we optimistically keep going, 3614ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // because the new extension may be removed too. 36154c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost; 36164c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar TotalCreatedInstsCost -= ExtCost; 3617ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!StressExtLdPromotion && 36184c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar (TotalCreatedInstsCost > 1 || 3619ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !isPromotedInstructionLegal(*TLI, PromotedVal))) { 3620ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The promotion is not profitable, rollback to the previous state. 3621ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.rollback(LastKnownGood); 3622ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 3623ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 3624ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The promotion is profitable. 3625ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Check if it exposes an ext(load). 36264c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar (void)ExtLdPromotion(TPT, LI, Inst, NewExts, TotalCreatedInstsCost); 36274c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (LI && (StressExtLdPromotion || NewCreatedInstsCost <= ExtCost || 3628ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If we have created a new extension, i.e., now we have two 3629ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // extensions. We must make sure one of them is merged with 3630ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // the load, otherwise we may degrade the code quality. 3631ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines (LI->hasOneUse() || hasSameExtUse(LI, *TLI)))) 3632ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Promotion happened. 3633ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 3634ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If this does not help to expose an ext(load) then, rollback. 3635ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.rollback(LastKnownGood); 3636ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 3637ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // None of the extension can form an ext(load). 3638ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LI = nullptr; 3639ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Inst = nullptr; 3640ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 3641ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 3642ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 364336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// MoveExtToFormExtLoad - Move a zext or sext fed by a load into the same 364436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// basic block as the load, unless conditions are unfavorable. This allows 364536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// SelectionDAG to fold the extend into the load. 3646ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \p I[in/out] the extension may be modified during the process if some 3647ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// promotions apply. 364836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// 3649ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool CodeGenPrepare::MoveExtToFormExtLoad(Instruction *&I) { 3650ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Try to promote a chain of computation if it allows to form 3651ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // an extended load. 3652ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TypePromotionTransaction TPT; 3653ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TypePromotionTransaction::ConstRestorationPt LastKnownGood = 3654ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.getRestorationPoint(); 3655ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallVector<Instruction *, 1> Exts; 3656ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Exts.push_back(I); 365736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Look for a load being extended. 3658ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LoadInst *LI = nullptr; 3659ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *OldExt = I; 3660ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool HasPromoted = ExtLdPromotion(TPT, LI, I, Exts); 3661ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!LI || !I) { 3662ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(!HasPromoted && !LI && "If we did not match any load instruction " 3663ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "the code must remain the same"); 3664ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines I = OldExt; 3665ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 3666ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 366736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 366836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If they're already in the same block, there's nothing to do. 3669ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Make the cheap checks first if we did not promote. 3670ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // If we promoted, we need to check if it is indeed profitable. 3671ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!HasPromoted && LI->getParent() == I->getParent()) 367236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 367336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3674ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EVT VT = TLI->getValueType(I->getType()); 3675ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines EVT LoadVT = TLI->getValueType(LI->getType()); 3676ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 367736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the load has other users and the truncate is not free, this probably 367836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // isn't worthwhile. 3679ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!LI->hasOneUse() && TLI && 3680ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines (TLI->isTypeLegal(LoadVT) || !TLI->isTypeLegal(VT)) && 3681ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !TLI->isTruncateFree(I->getType(), LI->getType())) { 3682ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines I = OldExt; 3683ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.rollback(LastKnownGood); 368436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 3685ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 368636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 368736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check whether the target supports casts folded into loads. 368836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines unsigned LType; 368936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<ZExtInst>(I)) 369036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LType = ISD::ZEXTLOAD; 369136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else { 369236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines assert(isa<SExtInst>(I) && "Unexpected ext type!"); 369336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines LType = ISD::SEXTLOAD; 369436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 3695ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (TLI && !TLI->isLoadExtLegal(LType, VT, LoadVT)) { 3696ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines I = OldExt; 3697ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.rollback(LastKnownGood); 369836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 3699ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 370036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 370136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Move the extend into the same block as the load, so that SelectionDAG 370236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // can fold it. 3703ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TPT.commit(); 370436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I->removeFromParent(); 370536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines I->insertAfter(LI); 370636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumExtsMoved; 370736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 370836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 370936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 371036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeExtUses(Instruction *I) { 371136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = I->getParent(); 371236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 371336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the result of a {s|z}ext and its source are both live out, rewrite all 371436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // other uses of the source with result of extension. 371536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *Src = I->getOperand(0); 371636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (Src->hasOneUse()) 371736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 371836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 371936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Only do this xform if truncating is free. 372036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && !TLI->isTruncateFree(I->getType(), Src->getType())) 372136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 372236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 372336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Only safe to perform the optimization if the source is also defined in 372436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // this block. 372536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent()) 372636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 372736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 372836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool DefIsLiveOut = false; 372936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (User *U : I->users()) { 373036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UI = cast<Instruction>(U); 373136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 373236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this ext is used in. 373336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = UI->getParent(); 373436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 373536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DefIsLiveOut = true; 373636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines break; 373736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 373836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!DefIsLiveOut) 373936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 374036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 374136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Make sure none of the uses are PHI nodes. 374236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (User *U : Src->users()) { 374336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UI = cast<Instruction>(U); 374436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = UI->getParent(); 374536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 374636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Be conservative. We don't want this xform to end up introducing 374736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // reloads just before load / store instructions. 374836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI)) 374936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 375036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 375136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 375236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // InsertedTruncs - Only insert one trunc in each block once. 375336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, Instruction*> InsertedTruncs; 375436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 375536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 375636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Use &U : Src->uses()) { 375736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *User = cast<Instruction>(U.getUser()); 375836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 375936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this ext is used in. 376036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = User->getParent(); 376136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 376236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 376336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Both src and def are live in this block. Rewrite the use. 376436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *&InsertedTrunc = InsertedTruncs[UserBB]; 376536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 376636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedTrunc) { 376736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 376836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTrunc = new TruncInst(I, Src->getType(), "", InsertPt); 376936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedTruncsSet.insert(InsertedTrunc); 377036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 377136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 377236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Replace a use of the {s|z}ext source with a use of the result. 377336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines U = InsertedTrunc; 377436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumExtUses; 377536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 377636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 377736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 377836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 377936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 378036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 378136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// isFormingBranchFromSelectProfitable - Returns true if a SelectInst should be 378236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// turned into an explicit branch. 378336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool isFormingBranchFromSelectProfitable(SelectInst *SI) { 378436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // FIXME: This should use the same heuristics as IfConversion to determine 378536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // whether a select is better represented as a branch. This requires that 378636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // branch probability metadata is preserved for the select, which is not the 378736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // case currently. 378836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 378936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition()); 379036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 379136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the branch is predicted right, an out of order CPU can avoid blocking on 379236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the compare. Emit cmovs on compares with a memory operand as branches to 379336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // avoid stalls on the load from memory. If the compare has more than one use 379436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // there's probably another cmov or setcc around so it's not worth emitting a 379536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // branch. 379636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Cmp) 379736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 379836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 379936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *CmpOp0 = Cmp->getOperand(0); 380036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Value *CmpOp1 = Cmp->getOperand(1); 380136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 380236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We check that the memory operand has one use to avoid uses of the loaded 380336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // value directly after the compare, making branches unprofitable. 380436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return Cmp->hasOneUse() && 380536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ((isa<LoadInst>(CmpOp0) && CmpOp0->hasOneUse()) || 380636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines (isa<LoadInst>(CmpOp1) && CmpOp1->hasOneUse())); 380736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 380836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 380936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 381036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// If we have a SelectInst that will likely profit from branch prediction, 381136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// turn it into a branch. 381236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeSelectInst(SelectInst *SI) { 381336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1); 381436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 381536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Can we convert the 'select' to CF ? 381636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (DisableSelectToBranch || OptSize || !TLI || VectorCond) 381736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 381836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 381936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::SelectSupportKind SelectKind; 382036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VectorCond) 382136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectKind = TargetLowering::VectorMaskSelect; 382236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else if (SI->getType()->isVectorTy()) 382336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectKind = TargetLowering::ScalarCondVectorVal; 382436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 382536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SelectKind = TargetLowering::ScalarValSelect; 382636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 382736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Do we have efficient codegen support for this kind of 'selects' ? 382836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI->isSelectSupported(SelectKind)) { 382936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We have efficient codegen support for the select instruction. 383036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Check if it is profitable to keep this 'select'. 383136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI->isPredictableSelectExpensive() || 383236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines !isFormingBranchFromSelectProfitable(SI)) 383336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 383436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 383536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 383636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ModifiedDT = true; 383736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 383836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // First, we split the block containing the select into 2 blocks. 383936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *StartBlock = SI->getParent(); 384036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(SI)); 384136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *NextBlock = StartBlock->splitBasicBlock(SplitPt, "select.end"); 384236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 384336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Create a new block serving as the landing pad for the branch. 384436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *SmallBlock = BasicBlock::Create(SI->getContext(), "select.mid", 384536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines NextBlock->getParent(), NextBlock); 384636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 384736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Move the unconditional branch from the block with the select in it into our 384836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // landing pad block. 384936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines StartBlock->getTerminator()->eraseFromParent(); 385036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst::Create(NextBlock, SmallBlock); 385136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 385236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Insert the real conditional branch based on the original condition. 385336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst::Create(NextBlock, SmallBlock, SI->getCondition(), SI); 385436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 385536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // The select itself is replaced with a PHI Node. 385636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PHINode *PN = PHINode::Create(SI->getType(), 2, "", NextBlock->begin()); 385736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->takeName(SI); 385836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(SI->getTrueValue(), StartBlock); 385936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PN->addIncoming(SI->getFalseValue(), SmallBlock); 386036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI->replaceAllUsesWith(PN); 386136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI->eraseFromParent(); 386236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 386336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Instruct OptimizeBlock to skip to the next block. 386436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = StartBlock->end(); 386536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumSelectsExpanded; 386636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 386736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 386836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 386936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesstatic bool isBroadcastShuffle(ShuffleVectorInst *SVI) { 387036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SmallVector<int, 16> Mask(SVI->getShuffleMask()); 387136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines int SplatElem = -1; 387236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (unsigned i = 0; i < Mask.size(); ++i) { 387336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem) 387436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 387536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SplatElem = Mask[i]; 387636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 387736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 387836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 387936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 388036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 388136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// Some targets have expensive vector shifts if the lanes aren't all the same 388236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// (e.g. x86 only introduced "vpsllvd" and friends with AVX2). In these cases 388336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// it's often worth sinking a shufflevector splat down to its use so that 388436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines/// codegen can spot all lanes are identical. 388536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::OptimizeShuffleVectorInst(ShuffleVectorInst *SVI) { 388636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *DefBB = SVI->getParent(); 388736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 388836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Only do this xform if variable vector shifts are particularly expensive. 388936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI || !TLI->isVectorShiftByScalarCheap(SVI->getType())) 389036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 389136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 389236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // We only expect better codegen by sinking a shuffle if we can recognise a 389336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // constant splat. 389436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!isBroadcastShuffle(SVI)) 389536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 389636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 389736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // InsertedShuffles - Only insert a shuffle in each block once. 389836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DenseMap<BasicBlock*, Instruction*> InsertedShuffles; 389936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 390036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 390136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (User *U : SVI->users()) { 390236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *UI = cast<Instruction>(U); 390336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 390436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Figure out which BB this ext is used in. 390536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = UI->getParent(); 390636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == DefBB) continue; 390736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 390836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // For now only apply this when the splat is used by a shift instruction. 390936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!UI->isShift()) continue; 391036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 391136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Everything checks out, sink the shuffle if the user's block doesn't 391236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // already have a copy. 391336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *&InsertedShuffle = InsertedShuffles[UserBB]; 391436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 391536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!InsertedShuffle) { 391636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 391736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines InsertedShuffle = new ShuffleVectorInst(SVI->getOperand(0), 391836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SVI->getOperand(1), 391936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SVI->getOperand(2), "", InsertPt); 392036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 392136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 392236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI->replaceUsesOfWith(SVI, InsertedShuffle); 392336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 392436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 392536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 392636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If we removed all uses, nuke the shuffle. 392736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SVI->use_empty()) { 392836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SVI->eraseFromParent(); 392936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 393036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 393136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 393236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 393336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 393436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 393537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesnamespace { 393637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// \brief Helper class to promote a scalar operation to a vector one. 393737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// This class is used to move downward extractelement transition. 393837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// E.g., 393937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// a = vector_op <2 x i32> 394037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// b = extractelement <2 x i32> a, i32 0 394137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// c = scalar_op b 394237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// store c 394337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// 394437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// => 394537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// a = vector_op <2 x i32> 394637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// c = vector_op a (equivalent to scalar_op on the related lane) 394737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// * d = extractelement <2 x i32> c, i32 0 394837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// * store d 394937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// Assuming both extractelement and store can be combine, we get rid of the 395037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// transition. 395137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesclass VectorPromoteHelper { 395237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Used to perform some checks on the legality of vector operations. 395337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const TargetLowering &TLI; 395437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 395537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Used to estimated the cost of the promoted chain. 395637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const TargetTransformInfo &TTI; 395737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 395837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// The transition being moved downwards. 395937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *Transition; 396037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// The sequence of instructions to be promoted. 396137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<Instruction *, 4> InstsToBePromoted; 396237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Cost of combining a store and an extract. 396337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned StoreExtractCombineCost; 396437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Instruction that will be combined with the transition. 396537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *CombineInst; 396637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 396737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief The instruction that represents the current end of the transition. 396837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Since we are faking the promotion until we reach the end of the chain 396937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// of computation, we need a way to get the current end of the transition. 397037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *getEndOfTransition() const { 397137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (InstsToBePromoted.empty()) 397237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Transition; 397337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return InstsToBePromoted.back(); 397437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 397537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 397637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Return the index of the original value in the transition. 397737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// E.g., for "extractelement <2 x i32> c, i32 1" the original value, 397837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// c, is at index 0. 397937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned getTransitionOriginalValueIdx() const { 398037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(isa<ExtractElementInst>(Transition) && 398137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "Other kind of transitions are not supported yet"); 398237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return 0; 398337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 398437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 398537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Return the index of the index in the transition. 398637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// E.g., for "extractelement <2 x i32> c, i32 0" the index 398737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// is at index 1. 398837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned getTransitionIdx() const { 398937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(isa<ExtractElementInst>(Transition) && 399037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "Other kind of transitions are not supported yet"); 399137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return 1; 399237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 399337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 399437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Get the type of the transition. 399537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// This is the type of the original value. 399637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the 399737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// transition is <2 x i32>. 399837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *getTransitionType() const { 399937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Transition->getOperand(getTransitionOriginalValueIdx())->getType(); 400037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 400137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 400237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Promote \p ToBePromoted by moving \p Def downward through. 400337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// I.e., we have the following sequence: 400437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Def = Transition <ty1> a to <ty2> 400537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// b = ToBePromoted <ty2> Def, ... 400637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// => 400737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// b = ToBePromoted <ty1> a, ... 400837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// Def = Transition <ty1> ToBePromoted to <ty2> 400937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines void promoteImpl(Instruction *ToBePromoted); 401037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 401137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Check whether or not it is profitable to promote all the 401237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// instructions enqueued to be promoted. 401337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool isProfitableToPromote() { 401437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx()); 401537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Index = isa<ConstantInt>(ValIdx) 401637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ? cast<ConstantInt>(ValIdx)->getZExtValue() 401737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : -1; 401837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *PromotedType = getTransitionType(); 401937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 402037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreInst *ST = cast<StoreInst>(CombineInst); 402137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned AS = ST->getPointerAddressSpace(); 402237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Align = ST->getAlignment(); 402337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check if this store is supported. 402437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!TLI.allowsMisalignedMemoryAccesses( 402537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI.getValueType(ST->getValueOperand()->getType()), AS, Align)) { 402637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If this is not supported, there is no way we can combine 402737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the extract with the store. 402837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 402937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 403037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 403137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The scalar chain of computation has to pay for the transition 403237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // scalar to vector. 403337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // The vector chain has to account for the combining cost. 403437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines uint64_t ScalarCost = 403537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index); 403637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines uint64_t VectorCost = StoreExtractCombineCost; 403737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (const auto &Inst : InstsToBePromoted) { 403837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Compute the cost. 403937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // By construction, all instructions being promoted are arithmetic ones. 404037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Moreover, one argument is a constant that can be viewed as a splat 404137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // constant. 404237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Arg0 = Inst->getOperand(0); 404337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) || 404437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isa<ConstantFP>(Arg0); 404537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TargetTransformInfo::OperandValueKind Arg0OVK = 404637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue 404737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : TargetTransformInfo::OK_AnyValue; 404837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TargetTransformInfo::OperandValueKind Arg1OVK = 404937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue 405037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : TargetTransformInfo::OK_AnyValue; 405137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ScalarCost += TTI.getArithmeticInstrCost( 405237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Inst->getOpcode(), Inst->getType(), Arg0OVK, Arg1OVK); 405337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType, 405437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Arg0OVK, Arg1OVK); 405537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 405637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Estimated cost of computation to be promoted:\nScalar: " 405737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines << ScalarCost << "\nVector: " << VectorCost << '\n'); 405837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return ScalarCost > VectorCost; 405937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 406037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 406137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Generate a constant vector with \p Val with the same 406237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// number of elements as the transition. 406337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \p UseSplat defines whether or not \p Val should be replicated 406437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// accross the whole vector. 406537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>, 406637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// otherwise we generate a vector with as many undef as possible: 406737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// <undef, ..., undef, Val, undef, ..., undef> where \p Val is only 406837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// used at the index of the extract. 406937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *getConstantVector(Constant *Val, bool UseSplat) const { 407037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned ExtractIdx = UINT_MAX; 407137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!UseSplat) { 407237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If we cannot determine where the constant must be, we have to 407337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // use a splat constant. 407437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *ValExtractIdx = Transition->getOperand(getTransitionIdx()); 407537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx)) 407637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ExtractIdx = CstVal->getSExtValue(); 407737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else 407837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines UseSplat = true; 407937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 408037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 408137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned End = getTransitionType()->getVectorNumElements(); 408237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (UseSplat) 408337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return ConstantVector::getSplat(End, Val); 408437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 408537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines SmallVector<Constant *, 4> ConstVec; 408637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines UndefValue *UndefVal = UndefValue::get(Val->getType()); 408737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (unsigned Idx = 0; Idx != End; ++Idx) { 408837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Idx == ExtractIdx) 408937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ConstVec.push_back(Val); 409037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else 409137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ConstVec.push_back(UndefVal); 409237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 409337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return ConstantVector::get(ConstVec); 409437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 409537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 409637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Check if promoting to a vector type an operand at \p OperandIdx 409737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// in \p Use can trigger undefined behavior. 409837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines static bool canCauseUndefinedBehavior(const Instruction *Use, 409937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned OperandIdx) { 410037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // This is not safe to introduce undef when the operand is on 410137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // the right hand side of a division-like instruction. 410237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (OperandIdx != 1) 410337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 410437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines switch (Use->getOpcode()) { 410537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines default: 410637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 410737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::SDiv: 410837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::UDiv: 410937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::SRem: 411037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::URem: 411137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 411237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::FDiv: 411337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case Instruction::FRem: 411437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return !Use->hasNoNaNs(); 411537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 411637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines llvm_unreachable(nullptr); 411737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 411837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 411937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinespublic: 412037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VectorPromoteHelper(const TargetLowering &TLI, const TargetTransformInfo &TTI, 412137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *Transition, unsigned CombineCost) 412237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines : TLI(TLI), TTI(TTI), Transition(Transition), 412337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines StoreExtractCombineCost(CombineCost), CombineInst(nullptr) { 412437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(Transition && "Do not know how to promote null"); 412537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 412637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 412737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Check if we can promote \p ToBePromoted to \p Type. 412837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool canPromote(const Instruction *ToBePromoted) const { 412937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // We could support CastInst too. 413037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return isa<BinaryOperator>(ToBePromoted); 413137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 413237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 413337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Check if it is profitable to promote \p ToBePromoted 413437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// by moving downward the transition through. 413537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool shouldPromote(const Instruction *ToBePromoted) const { 413637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Promote only if all the operands can be statically expanded. 413737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Indeed, we do not want to introduce any new kind of transitions. 413837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (const Use &U : ToBePromoted->operands()) { 413937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const Value *Val = U.get(); 414037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Val == getEndOfTransition()) { 414137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If the use is a division and the transition is on the rhs, 414237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // we cannot promote the operation, otherwise we may create a 414337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // division by zero. 414437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo())) 414537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 414637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines continue; 414737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 414837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) && 414937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !isa<ConstantFP>(Val)) 415037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 415137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 415237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check that the resulting operation is legal. 415337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode()); 415437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!ISDOpcode) 415537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 415637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return StressStoreExtract || 415737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines TLI.isOperationLegalOrCustom( 415837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ISDOpcode, TLI.getValueType(getTransitionType(), true)); 415937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 416037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 416137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Check whether or not \p Use can be combined 416237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// with the transition. 416337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// I.e., is it possible to do Use(Transition) => AnotherUse? 416437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool canCombine(const Instruction *Use) { return isa<StoreInst>(Use); } 416537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 416637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Record \p ToBePromoted as part of the chain to be promoted. 416737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines void enqueueForPromotion(Instruction *ToBePromoted) { 416837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines InstsToBePromoted.push_back(ToBePromoted); 416937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 417037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 417137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Set the instruction that will be combined with the transition. 417237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines void recordCombineInstruction(Instruction *ToBeCombined) { 417337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(canCombine(ToBeCombined) && "Unsupported instruction to combine"); 417437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CombineInst = ToBeCombined; 417537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 417637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 417737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \brief Promote all the instructions enqueued for promotion if it is 417837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// is profitable. 417937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines /// \return True if the promotion happened, false otherwise. 418037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool promote() { 418137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check if there is something to promote. 418237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Right now, if we do not have anything to combine with, 418337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // we assume the promotion is not profitable. 418437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (InstsToBePromoted.empty() || !CombineInst) 418537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 418637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 418737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Check cost. 418837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!StressStoreExtract && !isProfitableToPromote()) 418937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 419037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 419137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Promote. 419237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (auto &ToBePromoted : InstsToBePromoted) 419337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines promoteImpl(ToBePromoted); 419437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines InstsToBePromoted.clear(); 419537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 419637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 419737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines}; 419837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} // End of anonymous namespace. 419937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 420037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesvoid VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) { 420137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // At this point, we know that all the operands of ToBePromoted but Def 420237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // can be statically promoted. 420337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // For Def, we need to use its parameter in ToBePromoted: 420437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // b = ToBePromoted ty1 a 420537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Def = Transition ty1 b to ty2 420637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Move the transition down. 420737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 1. Replace all uses of the promoted operation by the transition. 420837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // = ... b => = ... Def. 420937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(ToBePromoted->getType() == Transition->getType() && 421037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "The type of the result of the transition does not match " 421137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "the final type"); 421237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ToBePromoted->replaceAllUsesWith(Transition); 421337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 2. Update the type of the uses. 421437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def. 421537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Type *TransitionTy = getTransitionType(); 421637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ToBePromoted->mutateType(TransitionTy); 421737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // 3. Update all the operands of the promoted operation with promoted 421837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // operands. 421937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a. 422037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines for (Use &U : ToBePromoted->operands()) { 422137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *Val = U.get(); 422237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Value *NewVal = nullptr; 422337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Val == Transition) 422437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = Transition->getOperand(getTransitionOriginalValueIdx()); 422537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) || 422637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isa<ConstantFP>(Val)) { 422737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Use a splat constant if it is not safe to use undef. 422837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NewVal = getConstantVector( 422937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines cast<Constant>(Val), 423037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines isa<UndefValue>(Val) || 423137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo())); 423237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else 4233ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines llvm_unreachable("Did you modified shouldPromote and forgot to update " 4234ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "this?"); 423537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ToBePromoted->setOperand(U.getOperandNo(), NewVal); 423637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 423737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Transition->removeFromParent(); 423837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Transition->insertAfter(ToBePromoted); 423937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted); 424037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 424137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 424237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// Some targets can do store(extractelement) with one instruction. 424337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// Try to push the extractelement towards the stores when the target 424437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines/// has this feature and this is profitable. 424537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool CodeGenPrepare::OptimizeExtractElementInst(Instruction *Inst) { 424637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned CombineCost = UINT_MAX; 424737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (DisableStoreExtract || !TLI || 424837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines (!StressStoreExtract && 424937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(), 425037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Inst->getOperand(1), CombineCost))) 425137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 425237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 425337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // At this point we know that Inst is a vector to scalar transition. 425437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // Try to move it down the def-use chain, until: 425537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // - We can combine the transition with its single use 425637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // => we got rid of the transition. 425737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // - We escape the current basic block 425837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // => we would need to check that we are moving it at a cheaper place and 425937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // we do not do that for now. 426037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines BasicBlock *Parent = Inst->getParent(); 426137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n'); 426237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VectorPromoteHelper VPH(*TLI, *TTI, Inst, CombineCost); 426337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // If the transition has more than one use, assume this is not going to be 426437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // beneficial. 426537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines while (Inst->hasOneUse()) { 426637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin()); 426737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Use: " << *ToBePromoted << '\n'); 426837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 426937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (ToBePromoted->getParent() != Parent) { 427037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Instruction to promote is in a different block (" 427137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines << ToBePromoted->getParent()->getName() 427237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines << ") than the transition (" << Parent->getName() << ").\n"); 427337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 427437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 427537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 427637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (VPH.canCombine(ToBePromoted)) { 427737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Assume " << *Inst << '\n' 427837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines << "will be combined with: " << *ToBePromoted << '\n'); 427937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VPH.recordCombineInstruction(ToBePromoted); 428037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool Changed = VPH.promote(); 428137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines NumStoreExtractExposed += Changed; 428237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Changed; 428337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 428437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 428537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Try promoting.\n"); 428637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted)) 428737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 428837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 428937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n"); 429037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 429137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines VPH.enqueueForPromotion(ToBePromoted); 429237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Inst = ToBePromoted; 429337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 429437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 429537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 429637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 4297ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool CodeGenPrepare::OptimizeInst(Instruction *I, bool& ModifiedDT) { 429836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (PHINode *P = dyn_cast<PHINode>(I)) { 429936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // It is possible for very late stage optimizations (such as SimplifyCFG) 430036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // to introduce PHI nodes too late to be cleaned up. If we detect such a 430136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // trivial PHI, go ahead and zap it here. 43024c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL = I->getModule()->getDataLayout(); 43034c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (Value *V = SimplifyInstruction(P, DL, TLInfo, nullptr)) { 430436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines P->replaceAllUsesWith(V); 430536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines P->eraseFromParent(); 430636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumPHIsElim; 430736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 430836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 430936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 431036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 431136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 431236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CastInst *CI = dyn_cast<CastInst>(I)) { 431336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // If the source of the cast is a constant, then this should have 431436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // already been constant folded. The only reason NOT to constant fold 431536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // it is if something (e.g. LSR) was careful to place the constant 431636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // evaluation in a block other than then one that uses it (e.g. to hoist 431736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // the address of globals out of a loop). If this is the case, we don't 431836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // want to forward-subst the cast. 431936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<Constant>(CI->getOperand(0))) 432036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 432136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 432236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && OptimizeNoopCopyExpression(CI, *TLI)) 432336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 432436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 432536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<ZExtInst>(I) || isa<SExtInst>(I)) { 432636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// Sink a zext or sext into its user blocks if the target type doesn't 432736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// fit in one register 432836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI && TLI->getTypeAction(CI->getContext(), 432936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TLI->getValueType(CI->getType())) == 433036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TargetLowering::TypeExpandInteger) { 433136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return SinkCast(CI); 433236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } else { 433336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = MoveExtToFormExtLoad(I); 433436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange | OptimizeExtUses(I); 433536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 433636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 433736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 433836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 433936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 434036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CmpInst *CI = dyn_cast<CmpInst>(I)) 434136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI || !TLI->hasMultipleConditionRegisters()) 434236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeCmpExpression(CI); 434336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 434436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 434536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI) 434636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeMemoryInst(I, I->getOperand(0), LI->getType()); 434736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 434836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 434936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 435036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 435136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (TLI) 435236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeMemoryInst(I, SI->getOperand(1), 435336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SI->getOperand(0)->getType()); 435436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 435536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 435636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4357dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I); 4358dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4359dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (BinOp && (BinOp->getOpcode() == Instruction::AShr || 4360dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines BinOp->getOpcode() == Instruction::LShr)) { 4361dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)); 4362dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (TLI && CI && TLI->hasExtractBitsInsn()) 4363dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return OptimizeExtractBits(BinOp, CI, *TLI); 4364dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 4365dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return false; 4366dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 4367dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 436836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) { 436936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (GEPI->hasAllZeroIndices()) { 437036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines /// The GEP operand must be a pointer, so must its result -> BitCast 437136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(), 437236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GEPI->getName(), GEPI); 437336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GEPI->replaceAllUsesWith(NC); 437436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines GEPI->eraseFromParent(); 437536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumGEPsElim; 4376ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OptimizeInst(NC, ModifiedDT); 437736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return true; 437836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 437936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 438036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 438136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 438236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (CallInst *CI = dyn_cast<CallInst>(I)) 4383ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return OptimizeCallInst(CI, ModifiedDT); 438436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 438536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (SelectInst *SI = dyn_cast<SelectInst>(I)) 438636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeSelectInst(SI); 438736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 438836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I)) 438936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return OptimizeShuffleVectorInst(SVI); 439036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 439137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (isa<ExtractElementInst>(I)) 439237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return OptimizeExtractElementInst(I); 439337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 439436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 439536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 439636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 439736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// In this pass we look for GEP and cast instructions that are used 439836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// across basic blocks and rewrite them to improve basic-block-at-a-time 439936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// selection. 4400ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool CodeGenPrepare::OptimizeBlock(BasicBlock &BB, bool& ModifiedDT) { 440136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines SunkAddrs.clear(); 440236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 440336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 440436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CurInstIterator = BB.begin(); 4405ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines while (CurInstIterator != BB.end()) { 4406ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MadeChange |= OptimizeInst(CurInstIterator++, ModifiedDT); 4407ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (ModifiedDT) 4408ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 4409ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 441036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange |= DupRetToEnableTailCallOpts(&BB); 441136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 441236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 441336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 441436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 441536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// llvm.dbg.value is far away from the value then iSel may not be able 441636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// handle it properly. iSel will drop llvm.dbg.value if it can not 441736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// find a node corresponding to the value. 441836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::PlaceDbgValues(Function &F) { 441936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 4420ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (BasicBlock &BB : F) { 4421dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Instruction *PrevNonDbgInst = nullptr; 4422ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) { 4423ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Instruction *Insn = BI++; 442436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DbgValueInst *DVI = dyn_cast<DbgValueInst>(Insn); 4425dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Leave dbg.values that refer to an alloca alone. These 4426dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // instrinsics describe the address of a variable (= the alloca) 4427dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // being taken. They should not be moved next to the alloca 4428dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // (and to the beginning of the scope), but rather stay close to 4429dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // where said address is used. 4430dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!DVI || (DVI->getValue() && isa<AllocaInst>(DVI->getValue()))) { 443136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines PrevNonDbgInst = Insn; 443236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 443336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 443436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 443536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue()); 443636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) { 443736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "Moving Debug Value before :\n" << *DVI << ' ' << *VI); 443836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DVI->removeFromParent(); 443936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<PHINode>(VI)) 444036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DVI->insertBefore(VI->getParent()->getFirstInsertionPt()); 444136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines else 444236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DVI->insertAfter(VI); 444336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 444436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumDbgValueMoved; 444536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 444636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 444736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 444836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 444936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 445036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 445136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// If there is a sequence that branches based on comparing a single bit 445236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// against zero that can be combined into a single instruction, and the 445336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// target supports folding these into a single instruction, sink the 445436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// mask and compare into the branch uses. Do this before OptimizeBlock -> 445536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// OptimizeInst -> OptimizeCmpExpression, which perturbs the pattern being 445636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// searched for. 445736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesbool CodeGenPrepare::sinkAndCmp(Function &F) { 445836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!EnableAndCmpSinking) 445936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 446036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!TLI || !TLI->isMaskAndBranchFoldingLegal()) 446136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return false; 446236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool MadeChange = false; 446336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Function::iterator I = F.begin(), E = F.end(); I != E; ) { 446436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *BB = I++; 446536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 446636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Does this BB end with the following? 446736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // %andVal = and %val, #single-bit-set 446836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // %icmpVal = icmp %andResult, 0 446936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // br i1 %cmpVal label %dest1, label %dest2" 447036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *Brcc = dyn_cast<BranchInst>(BB->getTerminator()); 447136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Brcc || !Brcc->isConditional()) 447236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 447336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ICmpInst *Cmp = dyn_cast<ICmpInst>(Brcc->getOperand(0)); 447436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Cmp || Cmp->getParent() != BB) 447536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 447636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt *Zero = dyn_cast<ConstantInt>(Cmp->getOperand(1)); 447736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Zero || !Zero->isZero()) 447836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 447936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Instruction *And = dyn_cast<Instruction>(Cmp->getOperand(0)); 448036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!And || And->getOpcode() != Instruction::And || And->getParent() != BB) 448136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 448236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ConstantInt* Mask = dyn_cast<ConstantInt>(And->getOperand(1)); 448336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!Mask || !Mask->getUniqueInteger().isPowerOf2()) 448436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 448536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "found and; icmp ?,0; brcc\n"); DEBUG(BB->dump()); 448636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 448736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Push the "and; icmp" for any users that are conditional branches. 448836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Since there can only be one branch use per BB, we don't need to keep 448936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // track of which BBs we insert into. 449036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for (Value::use_iterator UI = Cmp->use_begin(), E = Cmp->use_end(); 449136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines UI != E; ) { 449236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines Use &TheUse = *UI; 449336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Find brcc use. 449436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BranchInst *BrccUser = dyn_cast<BranchInst>(*UI); 449536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++UI; 449636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (!BrccUser || !BrccUser->isConditional()) 449736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue; 449836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BasicBlock *UserBB = BrccUser->getParent(); 449936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (UserBB == BB) continue; 450036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(dbgs() << "found Brcc use\n"); 450136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 450236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // Sink the "and; icmp" to use. 450336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines MadeChange = true; 450436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BinaryOperator *NewAnd = 450536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BinaryOperator::CreateAnd(And->getOperand(0), And->getOperand(1), "", 450636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines BrccUser); 450736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst *NewCmp = 450836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(), NewAnd, Zero, 450936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "", BrccUser); 451036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines TheUse = NewCmp; 451136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines ++NumAndCmpsMoved; 451236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines DEBUG(BrccUser->getParent()->dump()); 451336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 451436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 451536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return MadeChange; 451636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines} 4517ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4518ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Retrieve the probabilities of a conditional branch. Returns true on 4519ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// success, or returns false if no or invalid metadata was found. 4520ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic bool extractBranchMetadata(BranchInst *BI, 4521ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t &ProbTrue, uint64_t &ProbFalse) { 4522ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines assert(BI->isConditional() && 4523ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines "Looking for probabilities on unconditional branch?"); 4524ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto *ProfileData = BI->getMetadata(LLVMContext::MD_prof); 4525ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!ProfileData || ProfileData->getNumOperands() != 3) 4526ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 4527ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4528ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const auto *CITrue = 4529ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1)); 4530ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const auto *CIFalse = 4531ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)); 4532ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!CITrue || !CIFalse) 4533ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 4534ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4535ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ProbTrue = CITrue->getValue().getZExtValue(); 4536ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ProbFalse = CIFalse->getValue().getZExtValue(); 4537ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4538ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return true; 4539ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 4540ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4541ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Scale down both weights to fit into uint32_t. 4542ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesstatic void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) { 4543ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse; 4544ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint32_t Scale = (NewMax / UINT32_MAX) + 1; 4545ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NewTrue = NewTrue / Scale; 4546ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NewFalse = NewFalse / Scale; 4547ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 4548ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4549ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \brief Some targets prefer to split a conditional branch like: 4550ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \code 4551ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %0 = icmp ne i32 %a, 0 4552ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %1 = icmp ne i32 %b, 0 4553ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %or.cond = or i1 %0, %1 4554ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// br i1 %or.cond, label %TrueBB, label %FalseBB 4555ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \endcode 4556ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// into multiple branch instructions like: 4557ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \code 4558ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// bb1: 4559ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %0 = icmp ne i32 %a, 0 4560ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// br i1 %0, label %TrueBB, label %bb2 4561ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// bb2: 4562ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// %1 = icmp ne i32 %b, 0 4563ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// br i1 %1, label %TrueBB, label %FalseBB 4564ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// \endcode 4565ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// This usually allows instruction selection to do even further optimizations 4566ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// and combine the compare with the branch instruction. Currently this is 4567ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// applied for targets which have "cheap" jump instructions. 4568ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// 4569ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// FIXME: Remove the (equivalent?) implementation in SelectionDAG. 4570ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines/// 4571ebe69fe11e48d322045d5949c83283927a0d790bStephen Hinesbool CodeGenPrepare::splitBranchCondition(Function &F) { 45724c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive()) 4573ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return false; 4574ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4575ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines bool MadeChange = false; 4576ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &BB : F) { 4577ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Does this BB end with the following? 4578ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %cond1 = icmp|fcmp|binary instruction ... 4579ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %cond2 = icmp|fcmp|binary instruction ... 4580ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // %cond.or = or|and i1 %cond1, cond2 4581ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // br i1 %cond.or label %dest1, label %dest2" 4582ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BinaryOperator *LogicOp; 4583ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BasicBlock *TBB, *FBB; 4584ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!match(BB.getTerminator(), m_Br(m_OneUse(m_BinOp(LogicOp)), TBB, FBB))) 4585ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 4586ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4587ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines unsigned Opc; 4588ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Cond1, *Cond2; 4589ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (match(LogicOp, m_And(m_OneUse(m_Value(Cond1)), 4590ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines m_OneUse(m_Value(Cond2))))) 4591ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Opc = Instruction::And; 4592ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines else if (match(LogicOp, m_Or(m_OneUse(m_Value(Cond1)), 4593ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines m_OneUse(m_Value(Cond2))))) 4594ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Opc = Instruction::Or; 4595ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines else 4596ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 4597ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4598ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!match(Cond1, m_CombineOr(m_Cmp(), m_BinOp())) || 4599ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !match(Cond2, m_CombineOr(m_Cmp(), m_BinOp())) ) 4600ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 4601ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4602ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << "Before branch condition splitting\n"; BB.dump()); 4603ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4604ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Create a new BB. 4605ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto *InsertBefore = std::next(Function::iterator(BB)) 4606ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines .getNodePtrUnchecked(); 4607ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto TmpBB = BasicBlock::Create(BB.getContext(), 4608ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BB.getName() + ".cond.split", 4609ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines BB.getParent(), InsertBefore); 4610ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4611ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Update original basic block by using the first condition directly by the 4612ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // branch instruction and removing the no longer needed and/or instruction. 4613ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto *Br1 = cast<BranchInst>(BB.getTerminator()); 4614ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Br1->setCondition(Cond1); 4615ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines LogicOp->eraseFromParent(); 4616ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4617ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Depending on the conditon we have to either replace the true or the false 4618ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // successor of the original branch instruction. 4619ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Opc == Instruction::And) 4620ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Br1->setSuccessor(0, TmpBB); 4621ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines else 4622ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Br1->setSuccessor(1, TmpBB); 4623ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4624ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Fill in the new basic block. 4625ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto *Br2 = IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB); 4626ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (auto *I = dyn_cast<Instruction>(Cond2)) { 4627ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines I->removeFromParent(); 4628ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines I->insertBefore(Br2); 4629ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 4630ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4631ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Update PHI nodes in both successors. The original BB needs to be 4632ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // replaced in one succesor's PHI nodes, because the branch comes now from 4633ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // the newly generated BB (NewBB). In the other successor we need to add one 4634ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // incoming edge to the PHI nodes, because both branch instructions target 4635ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // now the same successor. Depending on the original branch condition 4636ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // (and/or) we have to swap the successors (TrueDest, FalseDest), so that 4637ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // we perfrom the correct update for the PHI nodes. 4638ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // This doesn't change the successor order of the just created branch 4639ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // instruction (or any other instruction). 4640ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Opc == Instruction::Or) 4641ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines std::swap(TBB, FBB); 4642ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4643ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Replace the old BB with the new BB. 4644ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &I : *TBB) { 4645ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines PHINode *PN = dyn_cast<PHINode>(&I); 4646ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!PN) 4647ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines break; 4648ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines int i; 4649ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines while ((i = PN->getBasicBlockIndex(&BB)) >= 0) 4650ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines PN->setIncomingBlock(i, TmpBB); 4651ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 4652ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4653ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Add another incoming edge form the new BB. 4654ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines for (auto &I : *FBB) { 4655ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines PHINode *PN = dyn_cast<PHINode>(&I); 4656ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (!PN) 4657ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines break; 4658ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines auto *Val = PN->getIncomingValueForBlock(&BB); 4659ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines PN->addIncoming(Val, TmpBB); 4660ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 4661ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4662ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Update the branch weights (from SelectionDAGBuilder:: 4663ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // FindMergedConditions). 4664ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Opc == Instruction::Or) { 4665ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Codegen X | Y as: 4666ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // BB1: 4667ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp_if_X TBB 4668ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp TmpBB 4669ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // TmpBB: 4670ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp_if_Y TBB 4671ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp FBB 4672ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 4673ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4674ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // We have flexibility in setting Prob for BB1 and Prob for NewBB. 4675ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The requirement is that 4676ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) 4677ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // = TrueProb for orignal BB. 4678ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Assuming the orignal weights are A and B, one choice is to set BB1's 4679ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice 4680ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // assumes that 4681ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. 4682ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Another choice is to assume TrueProb for BB1 equals to TrueProb for 4683ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // TmpBB, but the math is more complicated. 4684ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t TrueWeight, FalseWeight; 4685ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (extractBranchMetadata(Br1, TrueWeight, FalseWeight)) { 4686ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t NewTrueWeight = TrueWeight; 4687ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight; 4688ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines scaleWeights(NewTrueWeight, NewFalseWeight); 4689ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) 4690ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines .createBranchWeights(TrueWeight, FalseWeight)); 4691ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4692ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NewTrueWeight = TrueWeight; 4693ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NewFalseWeight = 2 * FalseWeight; 4694ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines scaleWeights(NewTrueWeight, NewFalseWeight); 4695ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) 4696ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines .createBranchWeights(TrueWeight, FalseWeight)); 4697ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 4698ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } else { 4699ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Codegen X & Y as: 4700ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // BB1: 4701ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp_if_X TmpBB 4702ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp FBB 4703ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // TmpBB: 4704ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp_if_Y TBB 4705ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // jmp FBB 4706ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // 4707ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // This requires creation of TmpBB after CurBB. 4708ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4709ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // We have flexibility in setting Prob for BB1 and Prob for TmpBB. 4710ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The requirement is that 4711ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) 4712ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // = FalseProb for orignal BB. 4713ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Assuming the orignal weights are A and B, one choice is to set BB1's 4714ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice 4715ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // assumes that 4716ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB. 4717ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t TrueWeight, FalseWeight; 4718ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (extractBranchMetadata(Br1, TrueWeight, FalseWeight)) { 4719ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight; 4720ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines uint64_t NewFalseWeight = FalseWeight; 4721ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines scaleWeights(NewTrueWeight, NewFalseWeight); 4722ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) 4723ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines .createBranchWeights(TrueWeight, FalseWeight)); 4724ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4725ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NewTrueWeight = 2 * TrueWeight; 4726ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NewFalseWeight = FalseWeight; 4727ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines scaleWeights(NewTrueWeight, NewFalseWeight); 4728ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) 4729ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines .createBranchWeights(TrueWeight, FalseWeight)); 4730ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 4731ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 4732ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4733ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // Note: No point in getting fancy here, since the DT info is never 47344c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar // available to CodeGenPrepare. 4735ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ModifiedDT = true; 4736ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4737ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines MadeChange = true; 4738ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 4739ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump(); 4740ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines TmpBB->dump()); 4741ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 4742ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines return MadeChange; 4743ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines} 4744