1713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===// 2713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth// 3713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth// The LLVM Compiler Infrastructure 4713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth// 5713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth// This file is distributed under the University of Illinois Open Source 6713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth// License. See LICENSE.TXT for details. 7713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth// 8713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth//===----------------------------------------------------------------------===// 9713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \file 10713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// This transformation implements the well known scalar replacement of 11713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// aggregates transformation. It tries to identify promotable elements of an 12713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// aggregate alloca, and promote them to registers. It will also try to 13713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// convert uses of an element (or set of elements) of an alloca into a vector 14713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// or bitfield-style integer scalar if appropriate. 15713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 16713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// It works to do this with minimal slicing of the alloca so that regions 17713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// which are merely transferred in and out of external memory remain unchanged 18713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// and are not decomposed to scalar code. 19713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 20713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// Because this also performs alloca promotion, it can be thought of as also 21713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// serving the purpose of SSA formation. The algorithm iterates on the 22713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// function until all opportunities for promotion have been realized. 23713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 24713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth//===----------------------------------------------------------------------===// 25713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 26713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#define DEBUG_TYPE "sroa" 27713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Transforms/Scalar.h" 28d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/STLExtras.h" 29d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SetVector.h" 30d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallVector.h" 31d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/Statistic.h" 32d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Analysis/Dominators.h" 33d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Analysis/Loads.h" 34ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth#include "llvm/Analysis/PtrUseVisitor.h" 35d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Analysis/ValueTracking.h" 36713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/DIBuilder.h" 37713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/DebugInfo.h" 380b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Constants.h" 390b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DataLayout.h" 400b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DerivedTypes.h" 410b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Function.h" 420b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/IRBuilder.h" 430b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Instructions.h" 440b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/IntrinsicInst.h" 450b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/LLVMContext.h" 460b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Operator.h" 4784bcf93e0fd225de2217d1b712c01586a633a6d8Chandler Carruth#include "llvm/InstVisitor.h" 48713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Pass.h" 491c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth#include "llvm/Support/CommandLine.h" 50713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Support/Debug.h" 51713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Support/ErrorHandling.h" 52713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Support/MathExtras.h" 53713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Support/raw_ostream.h" 54713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Transforms/Utils/Local.h" 55713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Transforms/Utils/PromoteMemToReg.h" 56713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth#include "llvm/Transforms/Utils/SSAUpdater.h" 57713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthusing namespace llvm; 58713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 59713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthSTATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement"); 60713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthSTATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced"); 61713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthSTATISTIC(NumPromoted, "Number of allocas promoted to SSA values"); 62713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthSTATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion"); 63713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthSTATISTIC(NumDeleted, "Number of instructions deleted"); 64713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthSTATISTIC(NumVectorized, "Number of vectorized aggregates"); 65713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 661c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// Hidden option to force the pass to not use DomTree and mem2reg, instead 671c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// forming SSA values through the SSAUpdater infrastructure. 681c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruthstatic cl::opt<bool> 691c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler CarruthForceSSAUpdater("force-ssa-updater", cl::init(false), cl::Hidden); 701c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 71713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthnamespace { 725e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// \brief A common base class for representing a half-open byte range. 735e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthstruct ByteRange { 745e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief The beginning offset of the range. 755e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth uint64_t BeginOffset; 76713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 775e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief The ending offset, not included in the range. 785e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth uint64_t EndOffset; 792d1c2a2fb8aadc180ce486670e6600afb14f7096Benjamin Kramer 805e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth ByteRange() : BeginOffset(), EndOffset() {} 815e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth ByteRange(uint64_t BeginOffset, uint64_t EndOffset) 825e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth : BeginOffset(BeginOffset), EndOffset(EndOffset) {} 83713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 845e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Support for ordering ranges. 85713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 865e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// This provides an ordering over ranges such that start offsets are 875e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// always increasing, and within equal start offsets, the end offsets are 885e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// decreasing. Thus the spanning range comes first in a cluster with the 895e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// same start position. 905e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth bool operator<(const ByteRange &RHS) const { 915e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth if (BeginOffset < RHS.BeginOffset) return true; 925e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth if (BeginOffset > RHS.BeginOffset) return false; 935e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth if (EndOffset > RHS.EndOffset) return true; 945e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth return false; 955e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth } 96713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 975e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Support comparison with a single offset to allow binary searches. 985e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth friend bool operator<(const ByteRange &LHS, uint64_t RHSOffset) { 995e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth return LHS.BeginOffset < RHSOffset; 1005e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth } 1015e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1025e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth friend LLVM_ATTRIBUTE_UNUSED bool operator<(uint64_t LHSOffset, 1035e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth const ByteRange &RHS) { 1045e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth return LHSOffset < RHS.BeginOffset; 1055e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth } 106fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 1075e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth bool operator==(const ByteRange &RHS) const { 1085e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth return BeginOffset == RHS.BeginOffset && EndOffset == RHS.EndOffset; 1095e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth } 1105e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth bool operator!=(const ByteRange &RHS) const { return !operator==(RHS); } 1115e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth}; 1125e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1135e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// \brief A partition of an alloca. 1145e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// 1155e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// This structure represents a contiguous partition of the alloca. These are 1165e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// formed by examining the uses of the alloca. During formation, they may 1175e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// overlap but once an AllocaPartitioning is built, the Partitions within it 1185e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// are all disjoint. 1195e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthstruct Partition : public ByteRange { 1205e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Whether this partition is splittable into smaller partitions. 1215e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// 1225e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// We flag partitions as splittable when they are formed entirely due to 1235e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// accesses by trivially splittable operations such as memset and memcpy. 1245e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth bool IsSplittable; 1255e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1265e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Test whether a partition has been marked as dead. 1275e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth bool isDead() const { 1285e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth if (BeginOffset == UINT64_MAX) { 1295e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth assert(EndOffset == UINT64_MAX); 1305e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth return true; 131fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth } 1325e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth return false; 1335e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth } 134fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 1355e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Kill a partition. 1365e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// This is accomplished by setting both its beginning and end offset to 1375e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// the maximum possible value. 1385e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth void kill() { 1395e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth assert(!isDead() && "He's Dead, Jim!"); 1405e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth BeginOffset = EndOffset = UINT64_MAX; 1415e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth } 1425e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1435e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth Partition() : ByteRange(), IsSplittable() {} 1445e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth Partition(uint64_t BeginOffset, uint64_t EndOffset, bool IsSplittable) 1455e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth : ByteRange(BeginOffset, EndOffset), IsSplittable(IsSplittable) {} 1465e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth}; 1475e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1485e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// \brief A particular use of a partition of the alloca. 1495e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// 1505e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// This structure is used to associate uses of a partition with it. They 1515e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// mark the range of bytes which are referenced by a particular instruction, 1525e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// and includes a handle to the user itself and the pointer value in use. 1535e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// The bounds of these uses are determined by intersecting the bounds of the 1545e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// memory use itself with a particular partition. As a consequence there is 1555e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// intentionally overlap between various uses of the same partition. 1565e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthclass PartitionUse : public ByteRange { 1575e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Combined storage for both the Use* and split state. 1585e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth PointerIntPair<Use*, 1, bool> UsePtrAndIsSplit; 1595e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1605e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthpublic: 1615e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth PartitionUse() : ByteRange(), UsePtrAndIsSplit() {} 1625e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth PartitionUse(uint64_t BeginOffset, uint64_t EndOffset, Use *U, 1635e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth bool IsSplit) 1645e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth : ByteRange(BeginOffset, EndOffset), UsePtrAndIsSplit(U, IsSplit) {} 165713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1665e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief The use in question. Provides access to both user and used value. 167713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 1685e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// Note that this may be null if the partition use is *dead*, that is, it 1695e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// should be ignored. 1705e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth Use *getUse() const { return UsePtrAndIsSplit.getPointer(); } 1715e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1725e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Set the use for this partition use range. 1735e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth void setUse(Use *U) { UsePtrAndIsSplit.setPointer(U); } 1745e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1755e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth /// \brief Whether this use is split across multiple partitions. 1765e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth bool isSplit() const { return UsePtrAndIsSplit.getInt(); } 1775e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth}; 1785e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth} 1795e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth 1805e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthnamespace llvm { 1815e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthtemplate <> struct isPodLike<Partition> : llvm::true_type {}; 1825e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthtemplate <> struct isPodLike<PartitionUse> : llvm::true_type {}; 1835e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth} 184713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1855e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthnamespace { 1865e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// \brief Alloca partitioning representation. 1875e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// 1885e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// This class represents a partitioning of an alloca into slices, and 1895e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// information about the nature of uses of each slice of the alloca. The goal 1905e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// is that this information is sufficient to decide if and how to split the 1915e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// alloca apart and replace slices with scalars. It is also intended that this 1925e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// structure can capture the relevant information needed both to decide about 1935e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth/// and to enact these transformations. 1945e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthclass AllocaPartitioning { 1955e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruthpublic: 196713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Construct a partitioning of a particular alloca. 197713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 198713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// Construction does most of the work for partitioning the alloca. This 199713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// performs the necessary walks of users and builds a partitioning from it. 2003574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow AllocaPartitioning(const DataLayout &TD, AllocaInst &AI); 201713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 202713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Test whether a pointer to the allocation escapes our analysis. 203713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 204713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// If this is true, the partitioning is never fully built and should be 205713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// ignored. 206713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool isEscaped() const { return PointerEscapingInstr; } 207713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 208713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Support for iterating over the partitions. 209713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @{ 210713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth typedef SmallVectorImpl<Partition>::iterator iterator; 211713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth iterator begin() { return Partitions.begin(); } 212713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth iterator end() { return Partitions.end(); } 213713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 214713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth typedef SmallVectorImpl<Partition>::const_iterator const_iterator; 215713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const_iterator begin() const { return Partitions.begin(); } 216713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const_iterator end() const { return Partitions.end(); } 217713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @} 218713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 219713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Support for iterating over and manipulating a particular 220713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// partition's uses. 221713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 222713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// The iteration support provided for uses is more limited, but also 223713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// includes some manipulation routines to support rewriting the uses of 224713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// partitions during SROA. 225713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @{ 226713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth typedef SmallVectorImpl<PartitionUse>::iterator use_iterator; 227713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth use_iterator use_begin(unsigned Idx) { return Uses[Idx].begin(); } 228713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth use_iterator use_begin(const_iterator I) { return Uses[I - begin()].begin(); } 229713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth use_iterator use_end(unsigned Idx) { return Uses[Idx].end(); } 230713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth use_iterator use_end(const_iterator I) { return Uses[I - begin()].end(); } 231713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 232713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth typedef SmallVectorImpl<PartitionUse>::const_iterator const_use_iterator; 233713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const_use_iterator use_begin(unsigned Idx) const { return Uses[Idx].begin(); } 234713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const_use_iterator use_begin(const_iterator I) const { 235713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return Uses[I - begin()].begin(); 236713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 237713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const_use_iterator use_end(unsigned Idx) const { return Uses[Idx].end(); } 238713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const_use_iterator use_end(const_iterator I) const { 239713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return Uses[I - begin()].end(); 240713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 241a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth 242a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth unsigned use_size(unsigned Idx) const { return Uses[Idx].size(); } 243a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth unsigned use_size(const_iterator I) const { return Uses[I - begin()].size(); } 244a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth const PartitionUse &getUse(unsigned PIdx, unsigned UIdx) const { 245a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth return Uses[PIdx][UIdx]; 246a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth } 247a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth const PartitionUse &getUse(const_iterator I, unsigned UIdx) const { 248a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth return Uses[I - begin()][UIdx]; 249a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth } 250a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth 251a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth void use_push_back(unsigned Idx, const PartitionUse &PU) { 252a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth Uses[Idx].push_back(PU); 253a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth } 254a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth void use_push_back(const_iterator I, const PartitionUse &PU) { 255a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth Uses[I - begin()].push_back(PU); 256a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth } 257713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @} 258713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 259713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Allow iterating the dead users for this alloca. 260713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 261713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// These are instructions which will never actually use the alloca as they 262713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// are outside the allocated range. They are safe to replace with undef and 263713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// delete. 264713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @{ 265713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth typedef SmallVectorImpl<Instruction *>::const_iterator dead_user_iterator; 266713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth dead_user_iterator dead_user_begin() const { return DeadUsers.begin(); } 267713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth dead_user_iterator dead_user_end() const { return DeadUsers.end(); } 268713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @} 269713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2707f5bedec7406f76383a2367c4d051d7aaf0d6308Chandler Carruth /// \brief Allow iterating the dead expressions referring to this alloca. 271713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 272713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// These are operands which have cannot actually be used to refer to the 273713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// alloca as they are outside its range and the user doesn't correct for 274713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// that. These mostly consist of PHI node inputs and the like which we just 275713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// need to replace with undef. 276713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @{ 277713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth typedef SmallVectorImpl<Use *>::const_iterator dead_op_iterator; 278713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth dead_op_iterator dead_op_begin() const { return DeadOperands.begin(); } 279713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth dead_op_iterator dead_op_end() const { return DeadOperands.end(); } 280713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// @} 281713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 282713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief MemTransferInst auxiliary data. 283713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// This struct provides some auxiliary data about memory transfer 284713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// intrinsics such as memcpy and memmove. These intrinsics can use two 285713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// different ranges within the same alloca, and provide other challenges to 286713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// correctly represent. We stash extra data to help us untangle this 287713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// after the partitioning is complete. 288713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth struct MemTransferOffsets { 289fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth /// The destination begin and end offsets when the destination is within 290fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth /// this alloca. If the end offset is zero the destination is not within 291fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth /// this alloca. 292713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t DestBegin, DestEnd; 293fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 294fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth /// The source begin and end offsets when the source is within this alloca. 295fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth /// If the end offset is zero, the source is not within this alloca. 296713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t SourceBegin, SourceEnd; 297fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 298fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth /// Flag for whether an alloca is splittable. 299713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool IsSplittable; 300713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth }; 301713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth MemTransferOffsets getMemTransferOffsets(MemTransferInst &II) const { 302713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return MemTransferInstData.lookup(&II); 303713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 304713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 305713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Map from a PHI or select operand back to a partition. 306713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 307713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// When manipulating PHI nodes or selects, they can use more than one 308713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// partition of an alloca. We store a special mapping to allow finding the 309713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// partition referenced by each of these operands, if any. 31077c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth iterator findPartitionForPHIOrSelectOperand(Use *U) { 31177c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth SmallDenseMap<Use *, std::pair<unsigned, unsigned> >::const_iterator MapIt 31277c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth = PHIOrSelectOpMap.find(U); 313713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (MapIt == PHIOrSelectOpMap.end()) 314713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return end(); 315713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 316713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return begin() + MapIt->second.first; 317713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 318713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 319713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Map from a PHI or select operand back to the specific use of 320713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// a partition. 321713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 322713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// Similar to mapping these operands back to the partitions, this maps 323713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// directly to the use structure of that partition. 32477c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth use_iterator findPartitionUseForPHIOrSelectOperand(Use *U) { 32577c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth SmallDenseMap<Use *, std::pair<unsigned, unsigned> >::const_iterator MapIt 32677c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth = PHIOrSelectOpMap.find(U); 327713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(MapIt != PHIOrSelectOpMap.end()); 328713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return Uses[MapIt->second.first].begin() + MapIt->second.second; 329713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 330713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 331713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Compute a common type among the uses of a particular partition. 332713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 333713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// This routines walks all of the uses of a particular partition and tries 334713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// to find a common type between them. Untyped operations such as memset and 335713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// memcpy are ignored. 336713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *getCommonType(iterator I) const; 337713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 338ba13d2efbac767ff1b19217ee26cd2e548562cfdChandler Carruth#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 339713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const; 340713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void printUsers(raw_ostream &OS, const_iterator I, 341713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth StringRef Indent = " ") const; 342713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void print(raw_ostream &OS) const; 343ad9f5b89677583ab6ac036dfb4227b7a59f394eeNAKAMURA Takumi void LLVM_ATTRIBUTE_NOINLINE LLVM_ATTRIBUTE_USED dump(const_iterator I) const; 344ad9f5b89677583ab6ac036dfb4227b7a59f394eeNAKAMURA Takumi void LLVM_ATTRIBUTE_NOINLINE LLVM_ATTRIBUTE_USED dump() const; 345ba13d2efbac767ff1b19217ee26cd2e548562cfdChandler Carruth#endif 346713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 347713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthprivate: 348713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth template <typename DerivedT, typename RetT = void> class BuilderBase; 349713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth class PartitionBuilder; 350713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth friend class AllocaPartitioning::PartitionBuilder; 351713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth class UseBuilder; 352713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth friend class AllocaPartitioning::UseBuilder; 353713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3543a902d0ae7763e952eb2fa70e2b07ccb038ae235Chandler Carruth#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 355713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Handle to alloca instruction to simplify method interfaces. 356713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaInst &AI; 357d0807690de6c735af84e35dd76a30420eaded16bBenjamin Kramer#endif 358713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 359713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief The instruction responsible for this alloca having no partitioning. 360713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 361713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// When an instruction (potentially) escapes the pointer to the alloca, we 362713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// store a pointer to that here and abort trying to partition the alloca. 363713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// This will be null if the alloca is partitioned successfully. 364713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Instruction *PointerEscapingInstr; 365713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 366713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief The partitions of the alloca. 367713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 368713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// We store a vector of the partitions over the alloca here. This vector is 369713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// sorted by increasing begin offset, and then by decreasing end offset. See 3707f5bedec7406f76383a2367c4d051d7aaf0d6308Chandler Carruth /// the Partition inner class for more details. Initially (during 3717f5bedec7406f76383a2367c4d051d7aaf0d6308Chandler Carruth /// construction) there are overlaps, but we form a disjoint sequence of 3727f5bedec7406f76383a2367c4d051d7aaf0d6308Chandler Carruth /// partitions while finishing construction and a fully constructed object is 3737f5bedec7406f76383a2367c4d051d7aaf0d6308Chandler Carruth /// expected to always have this as a disjoint space. 374713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallVector<Partition, 8> Partitions; 375713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 376713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief The uses of the partitions. 377713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 378713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// This is essentially a mapping from each partition to a list of uses of 379713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// that partition. The mapping is done with a Uses vector that has the exact 380713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// same number of entries as the partition vector. Each entry is itself 381713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// a vector of the uses. 382713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallVector<SmallVector<PartitionUse, 2>, 8> Uses; 383713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 384713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Instructions which will become dead if we rewrite the alloca. 385713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 386713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// Note that these are not separated by partition. This is because we expect 387713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// a partitioned alloca to be completely rewritten or not rewritten at all. 388713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// If rewritten, all these instructions can simply be removed and replaced 389713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// with undef as they come from outside of the allocated space. 390713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallVector<Instruction *, 8> DeadUsers; 391713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 392713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Operands which will become dead if we rewrite the alloca. 393713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 394713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// These are operands that in their particular use can be replaced with 395713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// undef when we rewrite the alloca. These show up in out-of-bounds inputs 396713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// to PHI nodes and the like. They aren't entirely dead (there might be 397713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we 398713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// want to swap this particular input for undef to simplify the use lists of 399713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// the alloca. 400713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallVector<Use *, 8> DeadOperands; 401713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 402713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief The underlying storage for auxiliary memcpy and memset info. 403713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallDenseMap<MemTransferInst *, MemTransferOffsets, 4> MemTransferInstData; 404713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 405713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief A side datastructure used when building up the partitions and uses. 406713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 407713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// This mapping is only really used during the initial building of the 408713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// partitioning so that we can retain information about PHI and select nodes 409713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// processed. 410713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallDenseMap<Instruction *, std::pair<uint64_t, bool> > PHIOrSelectSizes; 411713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 412713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Auxiliary information for particular PHI or select operands. 41377c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth SmallDenseMap<Use *, std::pair<unsigned, unsigned>, 4> PHIOrSelectOpMap; 414713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 415713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief A utility routine called from the constructor. 416713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 417713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// This does what it says on the tin. It is the key of the alloca partition 418713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// splitting and merging. After it is called we have the desired disjoint 419713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// collection of partitions. 420713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void splitAndMergePartitions(); 421713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth}; 422713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 423713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 424ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruthstatic Value *foldSelectInst(SelectInst &SI) { 425ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth // If the condition being selected on is a constant or the same value is 426ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth // being selected between, fold the select. Yes this does (rarely) happen 427ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth // early on. 428ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition())) 429ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return SI.getOperand(1+CI->isZero()); 43050573b1c274d290a81ab76331104206ea0b09bdeJakub Staszak if (SI.getOperand(1) == SI.getOperand(2)) 431ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return SI.getOperand(1); 43250573b1c274d290a81ab76331104206ea0b09bdeJakub Staszak 433ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return 0; 434ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth} 435713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 436713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief Builder for the alloca partitioning. 437713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 438713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// This class builds an alloca partitioning by recursively visiting the uses 439713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// of an alloca and splitting the partitions for each load and store at each 440713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// offset. 441713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthclass AllocaPartitioning::PartitionBuilder 442ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : public PtrUseVisitor<PartitionBuilder> { 443ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth friend class PtrUseVisitor<PartitionBuilder>; 444ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth friend class InstVisitor<PartitionBuilder>; 445ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth typedef PtrUseVisitor<PartitionBuilder> Base; 446ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 447ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth const uint64_t AllocSize; 448ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth AllocaPartitioning &P; 449713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 450713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallDenseMap<Instruction *, unsigned> MemTransferPartitionMap; 451713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 452713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthpublic: 453ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth PartitionBuilder(const DataLayout &DL, AllocaInst &AI, AllocaPartitioning &P) 454ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : PtrUseVisitor<PartitionBuilder>(DL), 455ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth AllocSize(DL.getTypeAllocSize(AI.getAllocatedType())), 456ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth P(P) {} 457713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 458713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthprivate: 459ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void insertUse(Instruction &I, const APInt &Offset, uint64_t Size, 46063392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth bool IsSplittable = false) { 461e74a4a791535b9756fa7312e5cd6f746ecc1c989Chandler Carruth // Completely skip uses which have a zero size or start either before or 462e74a4a791535b9756fa7312e5cd6f746ecc1c989Chandler Carruth // past the end of the allocation. 463ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (Size == 0 || Offset.isNegative() || Offset.uge(AllocSize)) { 464713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset 465e74a4a791535b9756fa7312e5cd6f746ecc1c989Chandler Carruth << " which has zero size or starts outside of the " 466e74a4a791535b9756fa7312e5cd6f746ecc1c989Chandler Carruth << AllocSize << " byte alloca:\n" 467713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " alloca: " << P.AI << "\n" 468713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " use: " << I << "\n"); 469713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return; 470713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 471713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 472ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t BeginOffset = Offset.getZExtValue(); 473ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t EndOffset = BeginOffset + Size; 47402e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth 47502e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth // Clamp the end offset to the end of the allocation. Note that this is 47602e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth // formulated to handle even the case where "BeginOffset + Size" overflows. 47741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // This may appear superficially to be something we could ignore entirely, 47841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // but that is not so! There may be widened loads or PHI-node uses where 47941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // some instructions are dead but not others. We can't completely ignore 48041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // them, and so have to record at least the information here. 48102e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth assert(AllocSize >= BeginOffset); // Established above. 48202e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth if (Size > AllocSize - BeginOffset) { 483713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset 484713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " to remain within the " << AllocSize << " byte alloca:\n" 485713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " alloca: " << P.AI << "\n" 486713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " use: " << I << "\n"); 487713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth EndOffset = AllocSize; 488713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 489713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 490713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Partition New(BeginOffset, EndOffset, IsSplittable); 491713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth P.Partitions.push_back(New); 492713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 493713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 494ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset, 49541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth uint64_t Size, bool IsVolatile) { 496a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // We allow splitting of loads and stores where the type is an integer type 49741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // and cover the entire alloca. This prevents us from splitting over 49841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // eagerly. 49941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // FIXME: In the great blue eventually, we should eagerly split all integer 50041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // loads and stores, and then have a separate step that merges adjacent 50141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // alloca partitions into a single partition suitable for integer widening. 50241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // Or we should skip the merge step and rely on GVN and other passes to 50341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // merge adjacent loads and stores that survive mem2reg. 50441b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth bool IsSplittable = 50541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth Ty->isIntegerTy() && !IsVolatile && Offset == 0 && Size >= AllocSize; 506a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth 507a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth insertUse(I, Offset, Size, IsSplittable); 508713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 509713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 510ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitLoadInst(LoadInst &LI) { 511c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth assert((!LI.isSimple() || LI.getType()->isSingleValueType()) && 512c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth "All simple FCA loads should have been pre-split"); 513ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 514ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (!IsOffsetKnown) 515ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(&LI); 516ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 51741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth uint64_t Size = DL.getTypeStoreSize(LI.getType()); 51841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth return handleLoadOrStore(LI.getType(), LI, Offset, Size, LI.isVolatile()); 519713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 520713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 521ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitStoreInst(StoreInst &SI) { 522c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth Value *ValOp = SI.getValueOperand(); 523c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth if (ValOp == *U) 524ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setEscapedAndAborted(&SI); 525ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (!IsOffsetKnown) 526ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(&SI); 527713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 52841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth uint64_t Size = DL.getTypeStoreSize(ValOp->getType()); 52941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth 53041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // If this memory access can be shown to *statically* extend outside the 53141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // bounds of of the allocation, it's behavior is undefined, so simply 53241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // ignore it. Note that this is more strict than the generic clamping 53341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // behavior of insertUse. We also try to handle cases which might run the 53441b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // risk of overflow. 53541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // FIXME: We should instead consider the pointer to have escaped if this 53641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // function is being instrumented for addressing bugs or race conditions. 53741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (Offset.isNegative() || Size > AllocSize || 53841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth Offset.ugt(AllocSize - Size)) { 53941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset 54041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth << " which extends past the end of the " << AllocSize 54141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth << " byte alloca:\n" 54241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth << " alloca: " << P.AI << "\n" 54341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth << " use: " << SI << "\n"); 54441b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth return; 54541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth } 54641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth 547c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) && 548c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth "All simple FCA stores should have been pre-split"); 54941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); 550713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 551713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 552713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 553ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitMemSetInst(MemSetInst &II) { 554b3dd9a13e8454fe069fafc1b259b64851f70e6e7Chandler Carruth assert(II.getRawDest() == *U && "Pointer use is not the destination?"); 555713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); 556ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if ((Length && Length->getValue() == 0) || 557ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth (IsOffsetKnown && !Offset.isNegative() && Offset.uge(AllocSize))) 558ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth // Zero-length mem transfer intrinsics can be ignored entirely. 559ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 560ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 561ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (!IsOffsetKnown) 562ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(&II); 563ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 564ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth insertUse(II, Offset, 565ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Length ? Length->getLimitedValue() 566ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : AllocSize - Offset.getLimitedValue(), 567ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth (bool)Length); 568713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 569713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 570ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitMemTransferInst(MemTransferInst &II) { 571713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); 572ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if ((Length && Length->getValue() == 0) || 573ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth (IsOffsetKnown && !Offset.isNegative() && Offset.uge(AllocSize))) 574713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Zero-length mem transfer intrinsics can be ignored entirely. 575ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 576ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 577ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (!IsOffsetKnown) 578ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(&II); 579ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 580ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t RawOffset = Offset.getLimitedValue(); 581ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t Size = Length ? Length->getLimitedValue() 582ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : AllocSize - RawOffset; 583713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 584713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth MemTransferOffsets &Offsets = P.MemTransferInstData[&II]; 585713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 586713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Only intrinsics with a constant length can be split. 587713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Offsets.IsSplittable = Length; 588713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 589fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (*U == II.getRawDest()) { 590ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Offsets.DestBegin = RawOffset; 591ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Offsets.DestEnd = RawOffset + Size; 592713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 593fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (*U == II.getRawSource()) { 594ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Offsets.SourceBegin = RawOffset; 595ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Offsets.SourceEnd = RawOffset + Size; 596fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth } 597713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 598fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // If we have set up end offsets for both the source and the destination, 599fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // we have found both sides of this transfer pointing at the same alloca. 600fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth bool SeenBothEnds = Offsets.SourceEnd && Offsets.DestEnd; 601fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (SeenBothEnds && II.getRawDest() != II.getRawSource()) { 602fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth unsigned PrevIdx = MemTransferPartitionMap[&II]; 603fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 604fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Check if the begin offsets match and this is a non-volatile transfer. 605fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // In that case, we can completely elide the transfer. 606fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (!II.isVolatile() && Offsets.SourceBegin == Offsets.DestBegin) { 607fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth P.Partitions[PrevIdx].kill(); 608ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 609fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth } 610fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 611fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Otherwise we have an offset transfer within the same alloca. We can't 612fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // split those. 613fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth P.Partitions[PrevIdx].IsSplittable = Offsets.IsSplittable = false; 614fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth } else if (SeenBothEnds) { 615fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Handle the case where this exact use provides both ends of the 616fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // operation. 617fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth assert(II.getRawDest() == II.getRawSource()); 618fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 619fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // For non-volatile transfers this is a no-op. 620fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (!II.isVolatile()) 621ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 622fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 623fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Otherwise just suppress splitting. 624713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Offsets.IsSplittable = false; 625fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth } 626fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 627fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 628fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Insert the use now that we've fixed up the splittable nature. 629fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth insertUse(II, Offset, Size, Offsets.IsSplittable); 630fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 631fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Setup the mapping from intrinsic to partition of we've not seen both 632fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // ends of this transfer. 633fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (!SeenBothEnds) { 634fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth unsigned NewIdx = P.Partitions.size() - 1; 635fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth bool Inserted 636fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth = MemTransferPartitionMap.insert(std::make_pair(&II, NewIdx)).second; 637fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth assert(Inserted && 638fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth "Already have intrinsic in map but haven't seen both ends"); 6390559d31c374050096e1071208661f97f24dbe299NAKAMURA Takumi (void)Inserted; 640713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 641713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 642713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 643713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Disable SRoA for any intrinsics except for lifetime invariants. 644ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak // FIXME: What about debug intrinsics? This matches old behavior, but 64550754f089265bd6a4208d8b89c653e7f7854796aChandler Carruth // doesn't make sense. 646ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitIntrinsicInst(IntrinsicInst &II) { 647ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (!IsOffsetKnown) 648ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(&II); 649ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 650713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (II.getIntrinsicID() == Intrinsic::lifetime_start || 651713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.getIntrinsicID() == Intrinsic::lifetime_end) { 652713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0)); 653ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(), 654ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Length->getLimitedValue()); 65563392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertUse(II, Offset, Size, true); 656ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 657713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 658713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 659ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Base::visitIntrinsicInst(II); 660713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 661713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 662713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) { 663713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // We consider any PHI or select that results in a direct load or store of 664713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // the same offset to be a viable use for partitioning purposes. These uses 665713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // are considered unsplittable and the size is the maximum loaded or stored 666713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // size. 667713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallPtrSet<Instruction *, 4> Visited; 668713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses; 669713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Visited.insert(Root); 670713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Uses.push_back(std::make_pair(cast<Instruction>(*U), Root)); 671c303463dcceb6a3b15e6f4d11d7c31c1ac2ce276Chandler Carruth // If there are no loads or stores, the access is dead. We mark that as 672c303463dcceb6a3b15e6f4d11d7c31c1ac2ce276Chandler Carruth // a size zero access. 673c303463dcceb6a3b15e6f4d11d7c31c1ac2ce276Chandler Carruth Size = 0; 674713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth do { 675713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Instruction *I, *UsedI; 676713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth llvm::tie(UsedI, I) = Uses.pop_back_val(); 677713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 678713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 679ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Size = std::max(Size, DL.getTypeStoreSize(LI->getType())); 680713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth continue; 681713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 682713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 683713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *Op = SI->getOperand(0); 684713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Op == UsedI) 685713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return SI; 686ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth Size = std::max(Size, DL.getTypeStoreSize(Op->getType())); 687713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth continue; 688713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 689713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 690713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) { 691713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!GEP->hasAllZeroIndices()) 692713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return GEP; 693713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) && 694713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth !isa<SelectInst>(I)) { 695713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return I; 696713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 697713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 698713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (Value::use_iterator UI = I->use_begin(), UE = I->use_end(); UI != UE; 699713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++UI) 700713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Visited.insert(cast<Instruction>(*UI))) 701713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Uses.push_back(std::make_pair(I, cast<Instruction>(*UI))); 702713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } while (!Uses.empty()); 703713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 704713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 705713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 706713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 707ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitPHINode(PHINode &PN) { 708ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (PN.use_empty()) 709ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 710ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (!IsOffsetKnown) 711ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(&PN); 712ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 713713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // See if we already have computed info on this node. 714713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth std::pair<uint64_t, bool> &PHIInfo = P.PHIOrSelectSizes[&PN]; 715713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (PHIInfo.first) { 716713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth PHIInfo.second = true; 71763392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertUse(PN, Offset, PHIInfo.first); 718ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 719713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 720713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 721713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Check for an unsafe use of the PHI node. 722ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&PN, PHIInfo.first)) 723ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(UnsafeI); 724713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 72563392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertUse(PN, Offset, PHIInfo.first); 726713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 727713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 728ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitSelectInst(SelectInst &SI) { 729ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (SI.use_empty()) 730ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 731713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Value *Result = foldSelectInst(SI)) { 732713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Result == *U) 733713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If the result of the constant fold will be the pointer, recurse 734713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // through the select as if we had RAUW'ed it. 735ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth enqueueUsers(SI); 736713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 737ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 738713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 739ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (!IsOffsetKnown) 740ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(&SI); 741713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 742713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // See if we already have computed info on this node. 743713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth std::pair<uint64_t, bool> &SelectInfo = P.PHIOrSelectSizes[&SI]; 744713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (SelectInfo.first) { 745713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SelectInfo.second = true; 74663392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertUse(SI, Offset, SelectInfo.first); 747ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return; 748713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 749713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 750713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Check for an unsafe use of the PHI node. 751ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&SI, SelectInfo.first)) 752ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return PI.setAborted(UnsafeI); 753713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 75463392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertUse(SI, Offset, SelectInfo.first); 755713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 756713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 757713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Disable SROA entirely if there are unhandled users of the alloca. 758ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void visitInstruction(Instruction &I) { 759ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth PI.setAborted(&I); 760ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth } 761713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth}; 762713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 763713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief Use adder for the alloca partitioning. 764713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 7657f5bedec7406f76383a2367c4d051d7aaf0d6308Chandler Carruth/// This class adds the uses of an alloca to all of the partitions which they 7667f5bedec7406f76383a2367c4d051d7aaf0d6308Chandler Carruth/// use. For splittable partitions, this can end up doing essentially a linear 767713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// walk of the partitions, but the number of steps remains bounded by the 768713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// total result instruction size: 769713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// - The number of partitions is a result of the number unsplittable 770713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// instructions using the alloca. 771713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// - The number of users of each partition is at worst the total number of 772713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// splittable instructions using the alloca. 773713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// Thus we will produce N * M instructions in the end, where N are the number 774713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// of unsplittable uses and M are the number of splittable. This visitor does 775713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// the exact same number of updates to the partitioning. 776713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 777713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// In the more common case, this visitor will leverage the fact that the 778713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// partition space is pre-sorted, and do a logarithmic search for the 779713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// partition needed, making the total visit a classical ((N + M) * log(N)) 780713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// complexity operation. 781ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruthclass AllocaPartitioning::UseBuilder : public PtrUseVisitor<UseBuilder> { 782ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth friend class PtrUseVisitor<UseBuilder>; 783713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth friend class InstVisitor<UseBuilder>; 784ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth typedef PtrUseVisitor<UseBuilder> Base; 785ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 786ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth const uint64_t AllocSize; 787ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth AllocaPartitioning &P; 788713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 789713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Set to de-duplicate dead instructions found in the use walk. 790713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SmallPtrSet<Instruction *, 4> VisitedDeadInsts; 791713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 792713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthpublic: 7933574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow UseBuilder(const DataLayout &TD, AllocaInst &AI, AllocaPartitioning &P) 794ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : PtrUseVisitor<UseBuilder>(TD), 795ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth AllocSize(TD.getTypeAllocSize(AI.getAllocatedType())), 796ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth P(P) {} 797713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 798713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthprivate: 799713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void markAsDead(Instruction &I) { 800713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (VisitedDeadInsts.insert(&I)) 801713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth P.DeadUsers.push_back(&I); 802713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 803713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 804ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void insertUse(Instruction &User, const APInt &Offset, uint64_t Size) { 805c303463dcceb6a3b15e6f4d11d7c31c1ac2ce276Chandler Carruth // If the use has a zero size or extends outside of the allocation, record 806c303463dcceb6a3b15e6f4d11d7c31c1ac2ce276Chandler Carruth // it as a dead use for elimination later. 807ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (Size == 0 || Offset.isNegative() || Offset.uge(AllocSize)) 808713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return markAsDead(User); 809713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 810ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t BeginOffset = Offset.getZExtValue(); 811ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t EndOffset = BeginOffset + Size; 81202e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth 81302e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth // Clamp the end offset to the end of the allocation. Note that this is 81402e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth // formulated to handle even the case where "BeginOffset + Size" overflows. 81502e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth assert(AllocSize >= BeginOffset); // Established above. 81602e92a0b5dd75514c8489d3f87fe7c5262ca6290Chandler Carruth if (Size > AllocSize - BeginOffset) 817713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth EndOffset = AllocSize; 818713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 819713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // NB: This only works if we have zero overlapping partitions. 82041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth iterator I = std::lower_bound(P.begin(), P.end(), BeginOffset); 82141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (I != P.begin() && llvm::prior(I)->EndOffset > BeginOffset) 82241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth I = llvm::prior(I); 82341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth iterator E = P.end(); 82441b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth bool IsSplit = llvm::next(I) != E && llvm::next(I)->BeginOffset < EndOffset; 82541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth for (; I != E && I->BeginOffset < EndOffset; ++I) { 82677c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth PartitionUse NewPU(std::max(I->BeginOffset, BeginOffset), 82741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth std::min(I->EndOffset, EndOffset), U, IsSplit); 82877c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth P.use_push_back(I, NewPU); 829713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (isa<PHINode>(U->getUser()) || isa<SelectInst>(U->getUser())) 83077c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth P.PHIOrSelectOpMap[U] 831713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth = std::make_pair(I - P.begin(), P.Uses[I - P.begin()].size() - 1); 832713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 833713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 834713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 835713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitBitCastInst(BitCastInst &BC) { 836713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (BC.use_empty()) 837713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return markAsDead(BC); 838713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 839ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return Base::visitBitCastInst(BC); 840713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 841713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 842713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitGetElementPtrInst(GetElementPtrInst &GEPI) { 843713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (GEPI.use_empty()) 844713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return markAsDead(GEPI); 845713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 846ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return Base::visitGetElementPtrInst(GEPI); 847713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 848713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 849713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitLoadInst(LoadInst &LI) { 850ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(IsOffsetKnown); 85141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth uint64_t Size = DL.getTypeStoreSize(LI.getType()); 85241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth insertUse(LI, Offset, Size); 853713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 854713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 855713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitStoreInst(StoreInst &SI) { 856ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(IsOffsetKnown); 85741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth uint64_t Size = DL.getTypeStoreSize(SI.getOperand(0)->getType()); 85841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth 85941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // If this memory access can be shown to *statically* extend outside the 86041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // bounds of of the allocation, it's behavior is undefined, so simply 86141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // ignore it. Note that this is more strict than the generic clamping 86241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth // behavior of insertUse. 86341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (Offset.isNegative() || Size > AllocSize || 86441b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth Offset.ugt(AllocSize - Size)) 86541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth return markAsDead(SI); 86641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth 86741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth insertUse(SI, Offset, Size); 868713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 869713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 870713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitMemSetInst(MemSetInst &II) { 871713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); 872ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if ((Length && Length->getValue() == 0) || 873ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth (IsOffsetKnown && !Offset.isNegative() && Offset.uge(AllocSize))) 874ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth return markAsDead(II); 875ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 876ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(IsOffsetKnown); 877ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth insertUse(II, Offset, Length ? Length->getLimitedValue() 878ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : AllocSize - Offset.getLimitedValue()); 879713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 880713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 881713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitMemTransferInst(MemTransferInst &II) { 882713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); 883ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if ((Length && Length->getValue() == 0) || 884ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth (IsOffsetKnown && !Offset.isNegative() && Offset.uge(AllocSize))) 885fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth return markAsDead(II); 886fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 887ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(IsOffsetKnown); 888ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth uint64_t Size = Length ? Length->getLimitedValue() 889ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : AllocSize - Offset.getLimitedValue(); 890ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 891fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth MemTransferOffsets &Offsets = P.MemTransferInstData[&II]; 892fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (!II.isVolatile() && Offsets.DestEnd && Offsets.SourceEnd && 893fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth Offsets.DestBegin == Offsets.SourceBegin) 894fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth return markAsDead(II); // Skip identity transfers without side-effects. 895fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 89663392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertUse(II, Offset, Size); 897713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 898713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 899713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitIntrinsicInst(IntrinsicInst &II) { 900ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(IsOffsetKnown); 901713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(II.getIntrinsicID() == Intrinsic::lifetime_start || 902713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.getIntrinsicID() == Intrinsic::lifetime_end); 903713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 904713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0)); 905ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth insertUse(II, Offset, std::min(Length->getLimitedValue(), 906ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth AllocSize - Offset.getLimitedValue())); 907713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 908713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 909ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth void insertPHIOrSelect(Instruction &User, const APInt &Offset) { 910713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t Size = P.PHIOrSelectSizes.lookup(&User).first; 911713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 912713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // For PHI and select operands outside the alloca, we can't nuke the entire 913713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // phi or select -- the other side might still be relevant, so we special 914713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // case them here and use a separate structure to track the operands 915713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // themselves which should be replaced with undef. 916ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if ((Offset.isNegative() && Offset.uge(Size)) || 917ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth (!Offset.isNegative() && Offset.uge(AllocSize))) { 918713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth P.DeadOperands.push_back(U); 919713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return; 920713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 921713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 92263392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertUse(User, Offset, Size); 923713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 924ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 925713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitPHINode(PHINode &PN) { 926713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (PN.use_empty()) 927713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return markAsDead(PN); 928713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 929ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(IsOffsetKnown); 93063392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertPHIOrSelect(PN, Offset); 931713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 932ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth 933713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitSelectInst(SelectInst &SI) { 934713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (SI.use_empty()) 935713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return markAsDead(SI); 936713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 937713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Value *Result = foldSelectInst(SI)) { 938713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Result == *U) 939713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If the result of the constant fold will be the pointer, recurse 940713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // through the select as if we had RAUW'ed it. 941ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth enqueueUsers(SI); 942d54a6b56a91c72cc33ce3dafcf1653123e6fdd87Chandler Carruth else 943d54a6b56a91c72cc33ce3dafcf1653123e6fdd87Chandler Carruth // Otherwise the operand to the select is dead, and we can replace it 944d54a6b56a91c72cc33ce3dafcf1653123e6fdd87Chandler Carruth // with undef. 945d54a6b56a91c72cc33ce3dafcf1653123e6fdd87Chandler Carruth P.DeadOperands.push_back(U); 946713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 947713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return; 948713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 949713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 950ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(IsOffsetKnown); 95163392ea3ba295d59260553245b14a435b5f71a3eChandler Carruth insertPHIOrSelect(SI, Offset); 952713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 953713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 954713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Unreachable, we've already visited the alloca once. 955713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void visitInstruction(Instruction &I) { 956713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth llvm_unreachable("Unhandled instruction in use builder."); 957713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 958713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth}; 959713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 960713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthvoid AllocaPartitioning::splitAndMergePartitions() { 961713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth size_t NumDeadPartitions = 0; 962713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 963713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Track the range of splittable partitions that we pass when accumulating 964713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // overlapping unsplittable partitions. 965713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t SplitEndOffset = 0ull; 966713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 967713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Partition New(0ull, 0ull, false); 968713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 969713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (unsigned i = 0, j = i, e = Partitions.size(); i != e; i = j) { 970713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++j; 971713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 972713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!Partitions[i].IsSplittable || New.BeginOffset == New.EndOffset) { 973713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(New.BeginOffset == New.EndOffset); 974713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New = Partitions[i]; 975713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } else { 976713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(New.IsSplittable); 977713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.EndOffset = std::max(New.EndOffset, Partitions[i].EndOffset); 978713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 979713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(New.BeginOffset != New.EndOffset); 980713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 981713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Scan the overlapping partitions. 982713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth while (j != e && New.EndOffset > Partitions[j].BeginOffset) { 983713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If the new partition we are forming is splittable, stop at the first 984713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // unsplittable partition. 985713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (New.IsSplittable && !Partitions[j].IsSplittable) 986713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth break; 987713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 988713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Grow the new partition to include any equally splittable range. 'j' is 989713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // always equally splittable when New is splittable, but when New is not 990713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // splittable, we may subsume some (or part of some) splitable partition 991713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // without growing the new one. 992713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (New.IsSplittable == Partitions[j].IsSplittable) { 993713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.EndOffset = std::max(New.EndOffset, Partitions[j].EndOffset); 994713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } else { 995713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(!New.IsSplittable); 996713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(Partitions[j].IsSplittable); 997713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SplitEndOffset = std::max(SplitEndOffset, Partitions[j].EndOffset); 998713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 999713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1000fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth Partitions[j].kill(); 1001713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++NumDeadPartitions; 1002713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++j; 1003713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1004713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1005713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If the new partition is splittable, chop off the end as soon as the 1006713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // unsplittable subsequent partition starts and ensure we eventually cover 1007713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // the splittable area. 1008713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (j != e && New.IsSplittable) { 1009713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SplitEndOffset = std::max(SplitEndOffset, New.EndOffset); 1010713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.EndOffset = std::min(New.EndOffset, Partitions[j].BeginOffset); 1011713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1012713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1013713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Add the new partition if it differs from the original one and is 1014713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // non-empty. We can end up with an empty partition here if it was 1015713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // splittable but there is an unsplittable one that starts at the same 1016713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // offset. 1017713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (New != Partitions[i]) { 1018713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (New.BeginOffset != New.EndOffset) 1019713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Partitions.push_back(New); 1020713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Mark the old one for removal. 1021fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth Partitions[i].kill(); 1022713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++NumDeadPartitions; 1023713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1024713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1025713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.BeginOffset = New.EndOffset; 1026713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!New.IsSplittable) { 1027713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.EndOffset = std::max(New.EndOffset, SplitEndOffset); 1028713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (j != e && !Partitions[j].IsSplittable) 1029713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.EndOffset = std::min(New.EndOffset, Partitions[j].BeginOffset); 1030713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.IsSplittable = true; 1031713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If there is a trailing splittable partition which won't be fused into 1032713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // the next splittable partition go ahead and add it onto the partitions 1033713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // list. 1034713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (New.BeginOffset < New.EndOffset && 1035713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth (j == e || !Partitions[j].IsSplittable || 1036713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.EndOffset < Partitions[j].BeginOffset)) { 1037713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Partitions.push_back(New); 1038713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New.BeginOffset = New.EndOffset = 0ull; 1039713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1040713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1041713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1042713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1043713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Re-sort the partitions now that they have been split and merged into 1044713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // disjoint set of partitions. Also remove any of the dead partitions we've 1045713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // replaced in the process. 1046713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth std::sort(Partitions.begin(), Partitions.end()); 1047713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (NumDeadPartitions) { 1048fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth assert(Partitions.back().isDead()); 1049713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert((ptrdiff_t)NumDeadPartitions == 1050713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth std::count(Partitions.begin(), Partitions.end(), Partitions.back())); 1051713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1052713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Partitions.erase(Partitions.end() - NumDeadPartitions, Partitions.end()); 1053713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1054713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 10553574eca1b02600bac4e625297f4ecf745f4c4f32Micah VillmowAllocaPartitioning::AllocaPartitioning(const DataLayout &TD, AllocaInst &AI) 1056d0807690de6c735af84e35dd76a30420eaded16bBenjamin Kramer : 10573a902d0ae7763e952eb2fa70e2b07ccb038ae235Chandler Carruth#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1058d0807690de6c735af84e35dd76a30420eaded16bBenjamin Kramer AI(AI), 1059d0807690de6c735af84e35dd76a30420eaded16bBenjamin Kramer#endif 1060d0807690de6c735af84e35dd76a30420eaded16bBenjamin Kramer PointerEscapingInstr(0) { 1061713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth PartitionBuilder PB(TD, AI, *this); 1062ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth PartitionBuilder::PtrInfo PtrI = PB.visitPtr(AI); 1063ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth if (PtrI.isEscaped() || PtrI.isAborted()) { 1064ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth // FIXME: We should sink the escape vs. abort info into the caller nicely, 1065ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth // possibly by just storing the PtrInfo in the AllocaPartitioning. 1066ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst() 1067ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth : PtrI.getAbortingInst(); 1068ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(PointerEscapingInstr && "Did not track a bad instruction"); 1069713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return; 1070ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth } 1071713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1072fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Sort the uses. This arranges for the offsets to be in ascending order, 1073fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // and the sizes to be in descending order. 1074fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth std::sort(Partitions.begin(), Partitions.end()); 1075fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 1076fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // Remove any partitions from the back which are marked as dead. 1077fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth while (!Partitions.empty() && Partitions.back().isDead()) 1078fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth Partitions.pop_back(); 1079713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1080fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (Partitions.size() > 1) { 1081713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Intersect splittability for all partitions with equal offsets and sizes. 1082713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Then remove all but the first so that we have a sequence of non-equal but 1083713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // potentially overlapping partitions. 1084713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (iterator I = Partitions.begin(), J = I, E = Partitions.end(); I != E; 1085713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth I = J) { 1086713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++J; 1087713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth while (J != E && *I == *J) { 1088713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth I->IsSplittable &= J->IsSplittable; 1089713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++J; 1090713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1091713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1092713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Partitions.erase(std::unique(Partitions.begin(), Partitions.end()), 1093713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Partitions.end()); 1094713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1095713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Split splittable and merge unsplittable partitions into a disjoint set 1096713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // of partitions over the used space of the allocation. 1097713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth splitAndMergePartitions(); 1098713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1099713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1100713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Now build up the user lists for each of these disjoint partitions by 1101713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // re-walking the recursive users of the alloca. 1102713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Uses.resize(Partitions.size()); 1103713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth UseBuilder UB(TD, AI, *this); 1104ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth PtrI = UB.visitPtr(AI); 1105ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(!PtrI.isEscaped() && "Previously analyzed pointer now escapes!"); 1106ed90ed077a58d6eff6aede206273ae0aeefa3a94Chandler Carruth assert(!PtrI.isAborted() && "Early aborted the visit of the pointer."); 1107713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1108713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1109713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthType *AllocaPartitioning::getCommonType(iterator I) const { 1110713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *Ty = 0; 1111713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (const_use_iterator UI = use_begin(I), UE = use_end(I); UI != UE; ++UI) { 111241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth Use *U = UI->getUse(); 111341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (!U) 1114fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth continue; // Skip dead uses. 111541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (isa<IntrinsicInst>(*U->getUser())) 1116713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth continue; 1117713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (UI->BeginOffset != I->BeginOffset || UI->EndOffset != I->EndOffset) 11187c8df7aa0c02908a29f0b5e6cfccb531dccdb96bChandler Carruth continue; 1119713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1120713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *UserTy = 0; 112141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) 1122713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth UserTy = LI->getType(); 112341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) 1124713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth UserTy = SI->getValueOperand()->getType(); 112511687d4982200fe99a820ea513100f237dfc1609Jakub Staszak else 1126a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth return 0; // Bail if we have weird uses. 1127a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth 1128a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth if (IntegerType *ITy = dyn_cast<IntegerType>(UserTy)) { 1129a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // If the type is larger than the partition, skip it. We only encounter 1130a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // this for split integer operations where we want to use the type of the 1131a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // entity causing the split. 1132a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth if (ITy->getBitWidth() > (I->EndOffset - I->BeginOffset)*8) 1133a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth continue; 1134a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth 1135a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // If we have found an integer type use covering the alloca, use that 1136a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // regardless of the other types, as integers are often used for a "bucket 1137a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // of bits" type. 1138a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth return ITy; 1139713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1140713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1141713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Ty && Ty != UserTy) 1142713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 1143713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1144713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Ty = UserTy; 1145713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1146713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return Ty; 1147713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1148713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1149ba13d2efbac767ff1b19217ee26cd2e548562cfdChandler Carruth#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1150ba13d2efbac767ff1b19217ee26cd2e548562cfdChandler Carruth 1151713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthvoid AllocaPartitioning::print(raw_ostream &OS, const_iterator I, 1152713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth StringRef Indent) const { 1153713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth OS << Indent << "partition #" << (I - begin()) 1154713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " [" << I->BeginOffset << "," << I->EndOffset << ")" 1155713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << (I->IsSplittable ? " (splittable)" : "") 1156713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << (Uses[I - begin()].empty() ? " (zero uses)" : "") 1157713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << "\n"; 1158713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1159713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1160713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthvoid AllocaPartitioning::printUsers(raw_ostream &OS, const_iterator I, 1161713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth StringRef Indent) const { 116250573b1c274d290a81ab76331104206ea0b09bdeJakub Staszak for (const_use_iterator UI = use_begin(I), UE = use_end(I); UI != UE; ++UI) { 116341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (!UI->getUse()) 1164fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth continue; // Skip dead uses. 1165713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth OS << Indent << " [" << UI->BeginOffset << "," << UI->EndOffset << ") " 116641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth << "used by: " << *UI->getUse()->getUser() << "\n"; 116741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (MemTransferInst *II = 116841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth dyn_cast<MemTransferInst>(UI->getUse()->getUser())) { 1169713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const MemTransferOffsets &MTO = MemTransferInstData.lookup(II); 1170713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool IsDest; 1171713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!MTO.IsSplittable) 1172713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IsDest = UI->BeginOffset == MTO.DestBegin; 1173713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth else 1174713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IsDest = MTO.DestBegin != 0u; 1175713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth OS << Indent << " (original " << (IsDest ? "dest" : "source") << ": " 1176713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << "[" << (IsDest ? MTO.DestBegin : MTO.SourceBegin) 1177713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << "," << (IsDest ? MTO.DestEnd : MTO.SourceEnd) << ")\n"; 1178713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1179713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1180713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1181713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1182713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthvoid AllocaPartitioning::print(raw_ostream &OS) const { 1183713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (PointerEscapingInstr) { 1184713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth OS << "No partitioning for alloca: " << AI << "\n" 1185713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " A pointer to this alloca escaped by:\n" 1186713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << " " << *PointerEscapingInstr << "\n"; 1187713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return; 1188713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1189713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1190713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth OS << "Partitioning of alloca: " << AI << "\n"; 1191bcff7b77348e299d25e7ada9ea5bffb80aab55b2Jakub Staszak for (const_iterator I = begin(), E = end(); I != E; ++I) { 1192713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth print(OS, I); 1193713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth printUsers(OS, I); 1194713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1195713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1196713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1197713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthvoid AllocaPartitioning::dump(const_iterator I) const { print(dbgs(), I); } 1198713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthvoid AllocaPartitioning::dump() const { print(dbgs()); } 1199713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1200ba13d2efbac767ff1b19217ee26cd2e548562cfdChandler Carruth#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1201ba13d2efbac767ff1b19217ee26cd2e548562cfdChandler Carruth 1202713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1203713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthnamespace { 12041c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// \brief Implementation of LoadAndStorePromoter for promoting allocas. 12051c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// 12061c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// This subclass of LoadAndStorePromoter adds overrides to handle promoting 12071c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// the loads and stores of an alloca instruction, as well as updating its 12081c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// debug information. This is used when a domtree is unavailable and thus 12091c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// mem2reg in its full form can't be used to handle promotion of allocas to 12101c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// scalar values. 12111c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruthclass AllocaPromoter : public LoadAndStorePromoter { 12121c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth AllocaInst &AI; 12131c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DIBuilder &DIB; 12141c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12151c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth SmallVector<DbgDeclareInst *, 4> DDIs; 12161c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth SmallVector<DbgValueInst *, 4> DVIs; 12171c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12181c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruthpublic: 12191c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth AllocaPromoter(const SmallVectorImpl<Instruction*> &Insts, SSAUpdater &S, 12201c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth AllocaInst &AI, DIBuilder &DIB) 12211c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth : LoadAndStorePromoter(Insts, S), AI(AI), DIB(DIB) {} 12221c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12231c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth void run(const SmallVectorImpl<Instruction*> &Insts) { 12241c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth // Remember which alloca we're promoting (for isInstInList). 12251c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (MDNode *DebugNode = MDNode::getIfExists(AI.getContext(), &AI)) { 12261c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth for (Value::use_iterator UI = DebugNode->use_begin(), 12271c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth UE = DebugNode->use_end(); 12281c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth UI != UE; ++UI) 12291c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(*UI)) 12301c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DDIs.push_back(DDI); 12311c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth else if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(*UI)) 12321c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DVIs.push_back(DVI); 12331c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 12341c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12351c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth LoadAndStorePromoter::run(Insts); 12361c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth AI.eraseFromParent(); 12371c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth while (!DDIs.empty()) 12381c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DDIs.pop_back_val()->eraseFromParent(); 12391c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth while (!DVIs.empty()) 12401c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DVIs.pop_back_val()->eraseFromParent(); 12411c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 12421c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12431c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth virtual bool isInstInList(Instruction *I, 12441c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth const SmallVectorImpl<Instruction*> &Insts) const { 12451c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (LoadInst *LI = dyn_cast<LoadInst>(I)) 12461c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth return LI->getOperand(0) == &AI; 12471c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth return cast<StoreInst>(I)->getPointerOperand() == &AI; 12481c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 12491c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12501c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth virtual void updateDebugInfo(Instruction *Inst) const { 12511c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth for (SmallVector<DbgDeclareInst *, 4>::const_iterator I = DDIs.begin(), 12521c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth E = DDIs.end(); I != E; ++I) { 12531c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DbgDeclareInst *DDI = *I; 12541c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) 12551c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth ConvertDebugDeclareToDebugValue(DDI, SI, DIB); 12561c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) 12571c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth ConvertDebugDeclareToDebugValue(DDI, LI, DIB); 12581c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 12591c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth for (SmallVector<DbgValueInst *, 4>::const_iterator I = DVIs.begin(), 12601c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth E = DVIs.end(); I != E; ++I) { 12611c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DbgValueInst *DVI = *I; 126250573b1c274d290a81ab76331104206ea0b09bdeJakub Staszak Value *Arg = 0; 12631c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { 12641c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth // If an argument is zero extended then use argument directly. The ZExt 12651c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth // may be zapped by an optimization pass in future. 12661c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (ZExtInst *ZExt = dyn_cast<ZExtInst>(SI->getOperand(0))) 12671c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Arg = dyn_cast<Argument>(ZExt->getOperand(0)); 12681c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (SExtInst *SExt = dyn_cast<SExtInst>(SI->getOperand(0))) 12691c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Arg = dyn_cast<Argument>(SExt->getOperand(0)); 12701c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (!Arg) 12711c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Arg = SI->getOperand(0); 12721c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) { 12731c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Arg = LI->getOperand(0); 12741c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } else { 12751c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth continue; 12761c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 12771c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Instruction *DbgVal = 12781c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DIB.insertDbgValueIntrinsic(Arg, 0, DIVariable(DVI->getVariable()), 12791c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Inst); 12801c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DbgVal->setDebugLoc(DVI->getDebugLoc()); 12811c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 12821c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 12831c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth}; 12841c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth} // end anon namespace 12851c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12861c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 12871c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruthnamespace { 1288713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief An optimization pass providing Scalar Replacement of Aggregates. 1289713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 1290713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// This pass takes allocations which can be completely analyzed (that is, they 1291713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// don't escape) and tries to turn them into scalar SSA values. There are 1292713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// a few steps to this process. 1293713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 1294713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 1) It takes allocations of aggregates and analyzes the ways in which they 1295713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// are used to try to split them into smaller allocations, ideally of 1296713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// a single scalar data type. It will split up memcpy and memset accesses 1297ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak/// as necessary and try to isolate individual scalar accesses. 1298713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 2) It will transform accesses into forms which are suitable for SSA value 1299713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// promotion. This can be replacing a memset with a scalar store of an 1300713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// integer value, or it can involve speculating operations on a PHI or 1301713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// select to be a PHI or select of the results. 1302713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 3) Finally, this will try to detect a pattern of accesses which map cleanly 1303713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// onto insert and extract operations on a vector value, and convert them to 1304713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// this form. By doing so, it will enable promotion of vector aggregates to 1305713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// SSA vector values. 1306713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthclass SROA : public FunctionPass { 13071c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth const bool RequiresDomTree; 13081c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 1309713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth LLVMContext *C; 13103574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow const DataLayout *TD; 1311713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DominatorTree *DT; 1312713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1313713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Worklist of alloca instructions to simplify. 1314713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// 1315713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// Each alloca in the function is added to this. Each new alloca formed gets 1316713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// added to it as well to recursively simplify unless that alloca can be 1317713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// directly promoted. Finally, each time we rewrite a use of an alloca other 1318713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// the one being actively rewritten, we add it back onto the list if not 1319713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// already present to ensure it is re-visited. 1320713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SetVector<AllocaInst *, SmallVector<AllocaInst *, 16> > Worklist; 1321713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1322713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief A collection of instructions to delete. 1323713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// We try to batch deletions to simplify code and make things a bit more 1324713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// efficient. 1325f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth SetVector<Instruction *, SmallVector<Instruction *, 8> > DeadInsts; 1326713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1327b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// \brief Post-promotion worklist. 1328b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// 1329b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// Sometimes we discover an alloca which has a high probability of becoming 1330b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// viable for SROA after a round of promotion takes place. In those cases, 1331b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// the alloca is enqueued here for re-processing. 1332b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// 1333b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// Note that we have to be very careful to clear allocas out of this list in 1334b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth /// the event they are deleted. 1335b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth SetVector<AllocaInst *, SmallVector<AllocaInst *, 16> > PostPromotionWorklist; 1336b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth 1337713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief A collection of alloca instructions we can directly promote. 1338713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth std::vector<AllocaInst *> PromotableAllocas; 1339713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1340713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthpublic: 13411c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth SROA(bool RequiresDomTree = true) 13421c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth : FunctionPass(ID), RequiresDomTree(RequiresDomTree), 13431c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth C(0), TD(0), DT(0) { 1344713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth initializeSROAPass(*PassRegistry::getPassRegistry()); 1345713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1346713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool runOnFunction(Function &F); 1347713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void getAnalysisUsage(AnalysisUsage &AU) const; 1348713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1349713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const char *getPassName() const { return "SROA"; } 1350713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth static char ID; 1351713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1352713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthprivate: 13531e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth friend class PHIOrSelectSpeculator; 1354713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth friend class AllocaPartitionRewriter; 1355713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth friend class AllocaPartitionVectorRewriter; 1356713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1357713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool rewriteAllocaPartition(AllocaInst &AI, 1358713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning &P, 1359713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning::iterator PI); 1360713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool splitAlloca(AllocaInst &AI, AllocaPartitioning &P); 1361713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool runOnAlloca(AllocaInst &AI); 13628615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth void deleteDeadInstructions(SmallPtrSet<AllocaInst *, 4> &DeletedAllocas); 13631c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth bool promoteAllocas(Function &F); 1364713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth}; 1365713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1366713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1367713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthchar SROA::ID = 0; 1368713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 13691c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler CarruthFunctionPass *llvm::createSROAPass(bool RequiresDomTree) { 13701c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth return new SROA(RequiresDomTree); 1371713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1372713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 1373713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthINITIALIZE_PASS_BEGIN(SROA, "sroa", "Scalar Replacement Of Aggregates", 1374713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth false, false) 1375713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthINITIALIZE_PASS_DEPENDENCY(DominatorTree) 1376713aa9431d84805c5b7ddce57159d14ee94cd548Chandler CarruthINITIALIZE_PASS_END(SROA, "sroa", "Scalar Replacement Of Aggregates", 1377713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth false, false) 1378713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 13790e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruthnamespace { 13800e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// \brief Visitor to speculate PHIs and Selects where possible. 13810e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruthclass PHIOrSelectSpeculator : public InstVisitor<PHIOrSelectSpeculator> { 13820e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Befriend the base class so it can delegate to private visit methods. 13830e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth friend class llvm::InstVisitor<PHIOrSelectSpeculator>; 1384713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 13853574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow const DataLayout &TD; 13860e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning &P; 13870e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SROA &Pass; 1388713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 13890e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruthpublic: 13903574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow PHIOrSelectSpeculator(const DataLayout &TD, AllocaPartitioning &P, SROA &Pass) 13910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth : TD(TD), P(P), Pass(Pass) {} 1392713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 13930e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// \brief Visit the users of an alloca partition and rewrite them. 13940e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth void visitUsers(AllocaPartitioning::const_iterator PI) { 13950e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Note that we need to use an index here as the underlying vector of uses 13960e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // may be grown during speculation. However, we never need to re-visit the 13970e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // new uses, and so we can use the initial size bound. 13980e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (unsigned Idx = 0, Size = P.use_size(PI); Idx != Size; ++Idx) { 13995e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth const PartitionUse &PU = P.getUse(PI, Idx); 140041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (!PU.getUse()) 14010e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth continue; // Skip dead use. 14020e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth 140341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth visit(cast<Instruction>(PU.getUse()->getUser())); 14040e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 1405713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1406713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14070e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruthprivate: 14080e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // By default, skip this instruction. 14090e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth void visitInstruction(Instruction &I) {} 1410713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14110e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// PHI instructions that use an alloca and are subsequently loaded can be 14120e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// rewritten to load both input pointers in the pred blocks and then PHI the 14130e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// results, allowing the load of the alloca to be promoted. 14140e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// From this: 14150e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %P2 = phi [i32* %Alloca, i32* %Other] 14160e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V = load i32* %P2 14170e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// to: 14180e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V1 = load i32* %Alloca -> will be mem2reg'd 14190e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// ... 14200e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V2 = load i32* %Other 14210e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// ... 14220e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V = phi [i32 %V1, i32 %V2] 14230e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// 14240e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// We can do this to a select if its only uses are loads and if the operands 14250e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// to the select can be loaded unconditionally. 14260e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// 14270e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// FIXME: This should be hoisted into a generic utility, likely in 14280e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// Transforms/Util/Local.h 14290e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth bool isSafePHIToSpeculate(PHINode &PN, SmallVectorImpl<LoadInst *> &Loads) { 14300e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // For now, we can only do this promotion if the load is in the same block 14310e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // as the PHI, and if there are no stores between the phi and load. 14320e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // TODO: Allow recursive phi users. 14330e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // TODO: Allow stores. 14340e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth BasicBlock *BB = PN.getParent(); 14350e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth unsigned MaxAlign = 0; 14360e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (Value::use_iterator UI = PN.use_begin(), UE = PN.use_end(); 14370e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth UI != UE; ++UI) { 14380e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *LI = dyn_cast<LoadInst>(*UI); 14390e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (LI == 0 || !LI->isSimple()) return false; 1440713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14410e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // For now we only allow loads in the same block as the PHI. This is 14420e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // a common case that happens when instcombine merges two loads through 14430e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // a PHI. 14440e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (LI->getParent() != BB) return false; 1445713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14460e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Ensure that there are no instructions between the PHI and the load that 14470e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // could store. 14480e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (BasicBlock::iterator BBI = &PN; &*BBI != LI; ++BBI) 14490e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (BBI->mayWriteToMemory()) 14500e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 1451713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14520e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth MaxAlign = std::max(MaxAlign, LI->getAlignment()); 14530e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Loads.push_back(LI); 1454713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1455713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14560e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // We can only transform this if it is safe to push the loads into the 14570e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // predecessor blocks. The only thing to watch out for is that we can't put 14580e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // a possibly trapping load in the predecessor if it is a critical edge. 145950573b1c274d290a81ab76331104206ea0b09bdeJakub Staszak for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { 14600e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth TerminatorInst *TI = PN.getIncomingBlock(Idx)->getTerminator(); 14610e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *InVal = PN.getIncomingValue(Idx); 1462713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14630e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // If the value is produced by the terminator of the predecessor (an 14640e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // invoke) or it has side-effects, there is no valid place to put a load 14650e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // in the predecessor. 14660e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (TI == InVal || TI->mayHaveSideEffects()) 14670e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 1468713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14690e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // If the predecessor has a single successor, then the edge isn't 14700e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // critical. 14710e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (TI->getNumSuccessors() == 1) 14720e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth continue; 1473713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14740e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // If this pointer is always safe to load, or if we can prove that there 14750e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // is already a load in the block, then we can move the load to the pred 14760e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // block. 14770e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (InVal->isDereferenceablePointer() || 14780e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth isSafeToLoadUnconditionally(InVal, TI, MaxAlign, &TD)) 14790e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth continue; 1480713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14810e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 14820e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 1483713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14840e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return true; 1485713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1486713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14870e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth void visitPHINode(PHINode &PN) { 14880e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth DEBUG(dbgs() << " original: " << PN << "\n"); 1489713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14900e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVector<LoadInst *, 4> Loads; 14910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!isSafePHIToSpeculate(PN, Loads)) 14920e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return; 1493713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14940e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth assert(!Loads.empty()); 1495713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 14960e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *LoadTy = cast<PointerType>(PN.getType())->getElementType(); 14970e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth IRBuilder<> PHIBuilder(&PN); 14980e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(), 14990e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth PN.getName() + ".sroa.speculated"); 1500713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15010e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Get the TBAA tag and alignment to use from one of the loads. It doesn't 1502ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak // matter which one we get and if any differ. 15030e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *SomeLoad = cast<LoadInst>(Loads.back()); 15040e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth MDNode *TBAATag = SomeLoad->getMetadata(LLVMContext::MD_tbaa); 15050e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth unsigned Align = SomeLoad->getAlignment(); 1506713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15070e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Rewrite all loads of the PN to use the new PHI. 15080e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth do { 15090e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *LI = Loads.pop_back_val(); 15100e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LI->replaceAllUsesWith(NewPN); 1511f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(LI); 15120e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } while (!Loads.empty()); 1513713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15140e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Inject loads into all of the pred blocks. 15150e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { 15160e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth BasicBlock *Pred = PN.getIncomingBlock(Idx); 15170e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth TerminatorInst *TI = Pred->getTerminator(); 15180e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Use *InUse = &PN.getOperandUse(PN.getOperandNumForIncomingValue(Idx)); 15190e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *InVal = PN.getIncomingValue(Idx); 15200e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth IRBuilder<> PredBuilder(TI); 1521713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15220e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *Load 15230e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth = PredBuilder.CreateLoad(InVal, (PN.getName() + ".sroa.speculate.load." + 15240e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Pred->getName())); 15250e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth ++NumLoadsSpeculated; 15260e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Load->setAlignment(Align); 15270e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (TBAATag) 15280e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Load->setMetadata(LLVMContext::MD_tbaa, TBAATag); 15290e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth NewPN->addIncoming(Load, Pred); 1530713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15310e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Instruction *Ptr = dyn_cast<Instruction>(InVal); 15320e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!Ptr) 15330e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // No uses to rewrite. 15340e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth continue; 1535713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15360e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Try to lookup and rewrite any partition uses corresponding to this phi 15370e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // input. 15380e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning::iterator PI 15390e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth = P.findPartitionForPHIOrSelectOperand(InUse); 15400e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (PI == P.end()) 15410e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth continue; 1542713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15430e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Replace the Use in the PartitionUse for this operand with the Use 15440e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // inside the load. 15450e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning::use_iterator UI 15460e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth = P.findPartitionUseForPHIOrSelectOperand(InUse); 154741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth assert(isa<PHINode>(*UI->getUse()->getUser())); 154841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth UI->setUse(&Load->getOperandUse(Load->getPointerOperandIndex())); 1549713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 15500e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth DEBUG(dbgs() << " speculated to: " << *NewPN << "\n"); 15510e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 1552713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15530e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// Select instructions that use an alloca and are subsequently loaded can be 15540e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// rewritten to load both input pointers and then select between the result, 15550e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// allowing the load of the alloca to be promoted. 15560e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// From this: 15570e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %P2 = select i1 %cond, i32* %Alloca, i32* %Other 15580e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V = load i32* %P2 15590e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// to: 15600e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V1 = load i32* %Alloca -> will be mem2reg'd 15610e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V2 = load i32* %Other 15620e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// %V = select i1 %cond, i32 %V1, i32 %V2 15630e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// 15640e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// We can do this to a select if its only uses are loads and if the operand 15650e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth /// to the select can be loaded unconditionally. 15660e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth bool isSafeSelectToSpeculate(SelectInst &SI, 15670e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVectorImpl<LoadInst *> &Loads) { 15680e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *TValue = SI.getTrueValue(); 15690e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *FValue = SI.getFalseValue(); 15700e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth bool TDerefable = TValue->isDereferenceablePointer(); 15710e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth bool FDerefable = FValue->isDereferenceablePointer(); 1572713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15730e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (Value::use_iterator UI = SI.use_begin(), UE = SI.use_end(); 15740e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth UI != UE; ++UI) { 15750e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *LI = dyn_cast<LoadInst>(*UI); 15760e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (LI == 0 || !LI->isSimple()) return false; 1577713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15780e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Both operands to the select need to be dereferencable, either 15790e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // absolutely (e.g. allocas) or at this point because we can see other 15800e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // accesses to it. 15810e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!TDerefable && !isSafeToLoadUnconditionally(TValue, LI, 15820e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LI->getAlignment(), &TD)) 15830e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 15840e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!FDerefable && !isSafeToLoadUnconditionally(FValue, LI, 15850e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LI->getAlignment(), &TD)) 15860e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 15870e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Loads.push_back(LI); 1588713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1589713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15900e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return true; 1591713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1592713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15930e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth void visitSelectInst(SelectInst &SI) { 15940e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth DEBUG(dbgs() << " original: " << SI << "\n"); 1595713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 15960e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // If the select isn't safe to speculate, just use simple logic to emit it. 15970e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVector<LoadInst *, 4> Loads; 15980e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!isSafeSelectToSpeculate(SI, Loads)) 15990e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return; 1600713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 16019497005d38b1c0d5183374493c08982eb8586eadJakub Staszak IRBuilder<> IRB(&SI); 16020e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Use *Ops[2] = { &SI.getOperandUse(1), &SI.getOperandUse(2) }; 16030e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning::iterator PIs[2]; 16045e8da1773c8a16a9e3f08df444420f04d71ba673Chandler Carruth PartitionUse PUs[2]; 16050e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (unsigned i = 0, e = 2; i != e; ++i) { 16060e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth PIs[i] = P.findPartitionForPHIOrSelectOperand(Ops[i]); 16070e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (PIs[i] != P.end()) { 16080e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // If the pointer is within the partitioning, remove the select from 16090e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // its uses. We'll add in the new loads below. 16100e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning::use_iterator UI 16110e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth = P.findPartitionUseForPHIOrSelectOperand(Ops[i]); 16120e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth PUs[i] = *UI; 16130e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Clear out the use here so that the offsets into the use list remain 16140e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // stable but this use is ignored when rewriting. 161541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth UI->setUse(0); 16160e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 16170e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 1618713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 16190e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *TV = SI.getTrueValue(); 16200e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *FV = SI.getFalseValue(); 16210e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Replace the loads of the select with a select of two loads. 16220e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth while (!Loads.empty()) { 16230e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *LI = Loads.pop_back_val(); 1624713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 16250e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth IRB.SetInsertPoint(LI); 16260e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *TL = 16270e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth IRB.CreateLoad(TV, LI->getName() + ".sroa.speculate.load.true"); 16280e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *FL = 16290e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth IRB.CreateLoad(FV, LI->getName() + ".sroa.speculate.load.false"); 16300e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth NumLoadsSpeculated += 2; 1631713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 16320e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Transfer alignment and TBAA info if present. 16330e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth TL->setAlignment(LI->getAlignment()); 16340e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth FL->setAlignment(LI->getAlignment()); 16350e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (MDNode *Tag = LI->getMetadata(LLVMContext::MD_tbaa)) { 16360e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth TL->setMetadata(LLVMContext::MD_tbaa, Tag); 16370e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth FL->setMetadata(LLVMContext::MD_tbaa, Tag); 16380e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 1639fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth 16400e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, 16410e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LI->getName() + ".sroa.speculated"); 1642713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 16430e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LoadInst *Loads[2] = { TL, FL }; 16440e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (unsigned i = 0, e = 2; i != e; ++i) { 16450e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (PIs[i] != P.end()) { 16460e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Use *LoadUse = &Loads[i]->getOperandUse(0); 164741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth assert(PUs[i].getUse()->get() == LoadUse->get()); 164841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth PUs[i].setUse(LoadUse); 16490e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth P.use_push_back(PIs[i], PUs[i]); 16500e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 1651713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 16520e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth 16530e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth DEBUG(dbgs() << " speculated to: " << *V << "\n"); 16540e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth LI->replaceAllUsesWith(V); 1655f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(LI); 1656713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 1657713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 16580e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth}; 1659713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 1660713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 16610e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// \brief Build a GEP out of a base pointer and indices. 16620e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 16630e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// This will return the BasePtr if that is valid, or build a new GEP 16640e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// instruction using the IRBuilder if GEP-ing is needed. 16650e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruthstatic Value *buildGEP(IRBuilder<> &IRB, Value *BasePtr, 16660e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVectorImpl<Value *> &Indices, 16670e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const Twine &Prefix) { 16680e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Indices.empty()) 16690e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return BasePtr; 16701e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 16710e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // A single zero index is a no-op, so check for this and avoid building a GEP 16720e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // in that case. 16730e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Indices.size() == 1 && cast<ConstantInt>(Indices.back())->isZero()) 16740e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return BasePtr; 16751e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 16760e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return IRB.CreateInBoundsGEP(BasePtr, Indices, Prefix + ".idx"); 16770e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth} 16781e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 16790e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// \brief Get a natural GEP off of the BasePtr walking through Ty toward 16800e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// TargetTy without changing the offset of the pointer. 16810e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 16820e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// This routine assumes we've already established a properly offset GEP with 16830e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// Indices, and arrived at the Ty type. The goal is to continue to GEP with 16840e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// zero-indices down through type layers until we find one the same as 16850e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// TargetTy. If we can't find one with the same type, we at least try to use 16860e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// one with the same size. If none of that works, we just produce the GEP as 16870e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// indicated by Indices to have the correct offset. 16883574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmowstatic Value *getNaturalGEPWithType(IRBuilder<> &IRB, const DataLayout &TD, 16890e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *BasePtr, Type *Ty, Type *TargetTy, 16900e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVectorImpl<Value *> &Indices, 16910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const Twine &Prefix) { 16920e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Ty == TargetTy) 16930e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return buildGEP(IRB, BasePtr, Indices, Prefix); 1694fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth 16950e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // See if we can descend into a struct and locate a field with the correct 16960e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // type. 16970e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth unsigned NumLayers = 0; 16980e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *ElementTy = Ty; 16990e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth do { 17000e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (ElementTy->isPointerTy()) 17010e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth break; 17020e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (SequentialType *SeqTy = dyn_cast<SequentialType>(ElementTy)) { 17030e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth ElementTy = SeqTy->getElementType(); 1704020d9d5feb59b4e92b4b55c5850fe3a5f77671b0Chandler Carruth // Note that we use the default address space as this index is over an 1705020d9d5feb59b4e92b4b55c5850fe3a5f77671b0Chandler Carruth // array or a vector, not a pointer. 1706020d9d5feb59b4e92b4b55c5850fe3a5f77671b0Chandler Carruth Indices.push_back(IRB.getInt(APInt(TD.getPointerSizeInBits(0), 0))); 17070e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } else if (StructType *STy = dyn_cast<StructType>(ElementTy)) { 17082fdb25b5a9c096f38bbd3f622c34ff9b2744e56dChandler Carruth if (STy->element_begin() == STy->element_end()) 17092fdb25b5a9c096f38bbd3f622c34ff9b2744e56dChandler Carruth break; // Nothing left to descend into. 17100e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth ElementTy = *STy->element_begin(); 17110e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices.push_back(IRB.getInt32(0)); 17120e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } else { 17130e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth break; 1714fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth } 17150e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth ++NumLayers; 17160e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } while (ElementTy != TargetTy); 17170e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (ElementTy != TargetTy) 17180e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices.erase(Indices.end() - NumLayers, Indices.end()); 17191e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17200e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return buildGEP(IRB, BasePtr, Indices, Prefix); 17210e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth} 17221e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17230e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// \brief Recursively compute indices for a natural GEP. 17240e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 17250e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// This is the recursive step for getNaturalGEPWithOffset that walks down the 17260e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// element types adding appropriate indices for the GEP. 17273574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmowstatic Value *getNaturalGEPRecursively(IRBuilder<> &IRB, const DataLayout &TD, 17280e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *Ptr, Type *Ty, APInt &Offset, 17290e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *TargetTy, 17300e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVectorImpl<Value *> &Indices, 17310e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const Twine &Prefix) { 17320e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Offset == 0) 17330e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return getNaturalGEPWithType(IRB, TD, Ptr, Ty, TargetTy, Indices, Prefix); 17341e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17350e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // We can't recurse through pointer types. 17360e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Ty->isPointerTy()) 17370e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; 17381e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17390e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // We try to analyze GEPs over vectors here, but note that these GEPs are 17400e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // extremely poorly defined currently. The long-term goal is to remove GEPing 17410e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // over a vector from the IR completely. 17420e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) { 1743e21708e4aaf741d0c9ccb5a5ddc75738fea7b61fNadav Rotem unsigned ElementSizeInBits = TD.getTypeSizeInBits(VecTy->getScalarType()); 17440e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (ElementSizeInBits % 8) 17450e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; // GEPs over non-multiple of 8 size vector elements are invalid. 17460e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth APInt ElementSize(Offset.getBitWidth(), ElementSizeInBits / 8); 174702bf98ab38355aafbe9a763caa019bc06ad9ce3dChandler Carruth APInt NumSkippedElements = Offset.sdiv(ElementSize); 17480e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (NumSkippedElements.ugt(VecTy->getNumElements())) 17490e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; 17500e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Offset -= NumSkippedElements * ElementSize; 17510e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices.push_back(IRB.getInt(NumSkippedElements)); 17520e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return getNaturalGEPRecursively(IRB, TD, Ptr, VecTy->getElementType(), 17530e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Offset, TargetTy, Indices, Prefix); 17540e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 17551e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17560e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { 17570e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *ElementTy = ArrTy->getElementType(); 17580e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth APInt ElementSize(Offset.getBitWidth(), TD.getTypeAllocSize(ElementTy)); 175902bf98ab38355aafbe9a763caa019bc06ad9ce3dChandler Carruth APInt NumSkippedElements = Offset.sdiv(ElementSize); 17600e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (NumSkippedElements.ugt(ArrTy->getNumElements())) 17610e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; 17621e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17630e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Offset -= NumSkippedElements * ElementSize; 17640e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices.push_back(IRB.getInt(NumSkippedElements)); 17650e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return getNaturalGEPRecursively(IRB, TD, Ptr, ElementTy, Offset, TargetTy, 17660e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices, Prefix); 17670e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 17681e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17690e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth StructType *STy = dyn_cast<StructType>(Ty); 17700e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!STy) 17710e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; 17721e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17730e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const StructLayout *SL = TD.getStructLayout(STy); 17740e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth uint64_t StructOffset = Offset.getZExtValue(); 17750e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (StructOffset >= SL->getSizeInBytes()) 17760e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; 17770e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth unsigned Index = SL->getElementContainingOffset(StructOffset); 17780e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Offset -= APInt(Offset.getBitWidth(), SL->getElementOffset(Index)); 17790e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *ElementTy = STy->getElementType(Index); 17800e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Offset.uge(TD.getTypeAllocSize(ElementTy))) 17810e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; // The offset points into alignment padding. 17821e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17830e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices.push_back(IRB.getInt32(Index)); 17840e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return getNaturalGEPRecursively(IRB, TD, Ptr, ElementTy, Offset, TargetTy, 17850e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices, Prefix); 17860e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth} 17871e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 17880e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// \brief Get a natural GEP from a base pointer to a particular offset and 17890e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// resulting in a particular type. 17900e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 17910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// The goal is to produce a "natural" looking GEP that works with the existing 17920e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// composite types to arrive at the appropriate offset and element type for 17930e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// a pointer. TargetTy is the element type the returned GEP should point-to if 17940e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// possible. We recurse by decreasing Offset, adding the appropriate index to 17950e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// Indices, and setting Ty to the result subtype. 17960e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 17970e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// If no natural GEP can be constructed, this function returns null. 17983574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmowstatic Value *getNaturalGEPWithOffset(IRBuilder<> &IRB, const DataLayout &TD, 17990e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *Ptr, APInt Offset, Type *TargetTy, 18000e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVectorImpl<Value *> &Indices, 18010e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const Twine &Prefix) { 18020e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth PointerType *Ty = cast<PointerType>(Ptr->getType()); 18031e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18040e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Don't consider any GEPs through an i8* as natural unless the TargetTy is 18050e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // an i8. 18060e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Ty == IRB.getInt8PtrTy() && TargetTy->isIntegerTy(8)) 18070e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; 18081e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18090e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *ElementTy = Ty->getElementType(); 18100e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!ElementTy->isSized()) 18110e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; // We can't GEP through an unsized element. 18120e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth APInt ElementSize(Offset.getBitWidth(), TD.getTypeAllocSize(ElementTy)); 18130e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (ElementSize == 0) 18140e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return 0; // Zero-length arrays can't help us build a natural GEP. 181502bf98ab38355aafbe9a763caa019bc06ad9ce3dChandler Carruth APInt NumSkippedElements = Offset.sdiv(ElementSize); 18161e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18170e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Offset -= NumSkippedElements * ElementSize; 18180e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices.push_back(IRB.getInt(NumSkippedElements)); 18190e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return getNaturalGEPRecursively(IRB, TD, Ptr, ElementTy, Offset, TargetTy, 18200e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices, Prefix); 18210e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth} 18221e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18230e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// \brief Compute an adjusted pointer from Ptr by Offset bytes where the 18240e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// resulting pointer has PointerTy. 18250e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 18260e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// This tries very hard to compute a "natural" GEP which arrives at the offset 18270e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// and produces the pointer type desired. Where it cannot, it will try to use 18280e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// the natural GEP to arrive at the offset and bitcast to the type. Where that 18290e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// fails, it will try to use an existing i8* and GEP to the byte offset and 18300e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// bitcast to the type. 18310e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 18320e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// The strategy for finding the more natural GEPs is to peel off layers of the 18330e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// pointer, walking back through bit casts and GEPs, searching for a base 18340e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// pointer from which we can compute a natural GEP with the desired 1835ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak/// properties. The algorithm tries to fold as many constant indices into 18360e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// a single GEP as possible, thus making each GEP more independent of the 18370e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// surrounding code. 18383574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmowstatic Value *getAdjustedPtr(IRBuilder<> &IRB, const DataLayout &TD, 18390e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *Ptr, APInt Offset, Type *PointerTy, 18400e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const Twine &Prefix) { 18410e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Even though we don't look through PHI nodes, we could be called on an 18420e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // instruction in an unreachable block, which may be on a cycle. 18430e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallPtrSet<Value *, 4> Visited; 18440e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Visited.insert(Ptr); 18450e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth SmallVector<Value *, 4> Indices; 18461e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18470e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // We may end up computing an offset pointer that has the wrong type. If we 18480e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // never are able to compute one directly that has the correct type, we'll 18490e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // fall back to it, so keep it around here. 18500e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *OffsetPtr = 0; 18511e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18520e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Remember any i8 pointer we come across to re-use if we need to do a raw 18530e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // byte offset. 18540e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Value *Int8Ptr = 0; 18550e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth APInt Int8PtrOffset(Offset.getBitWidth(), 0); 18561e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18570e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *TargetTy = PointerTy->getPointerElementType(); 18581e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18590e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth do { 18600e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // First fold any existing GEPs into the offset. 18610e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth while (GEPOperator *GEP = dyn_cast<GEPOperator>(Ptr)) { 18620e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth APInt GEPOffset(Offset.getBitWidth(), 0); 186398281a20503896349bd152e2dfe87435d3a6aadaNuno Lopes if (!GEP->accumulateConstantOffset(TD, GEPOffset)) 18640e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth break; 18650e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Offset += GEPOffset; 18660e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Ptr = GEP->getPointerOperand(); 18670e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!Visited.insert(Ptr)) 18680e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth break; 18690e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 18701e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18710e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // See if we can perform a natural GEP here. 18720e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices.clear(); 18730e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Value *P = getNaturalGEPWithOffset(IRB, TD, Ptr, Offset, TargetTy, 18740e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Indices, Prefix)) { 18750e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (P->getType() == PointerTy) { 18760e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Zap any offset pointer that we ended up computing in previous rounds. 18770e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (OffsetPtr && OffsetPtr->use_empty()) 18780e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Instruction *I = dyn_cast<Instruction>(OffsetPtr)) 18790e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth I->eraseFromParent(); 18800e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return P; 18810e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 18820e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!OffsetPtr) { 18830e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth OffsetPtr = P; 18840e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 18850e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 18861e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18870e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Stash this pointer if we've found an i8*. 18880e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Ptr->getType()->isIntegerTy(8)) { 18890e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Int8Ptr = Ptr; 18900e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Int8PtrOffset = Offset; 18910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 18921e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 18930e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Peel off a layer of the pointer and update the offset appropriately. 18940e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Operator::getOpcode(Ptr) == Instruction::BitCast) { 18950e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Ptr = cast<Operator>(Ptr)->getOperand(0); 18960e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(Ptr)) { 18970e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (GA->mayBeOverridden()) 18980e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth break; 18990e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Ptr = GA->getAliasee(); 19000e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } else { 19010e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth break; 19020e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 19030e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth assert(Ptr->getType()->isPointerTy() && "Unexpected operand type!"); 19040e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } while (Visited.insert(Ptr)); 19051e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 19060e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!OffsetPtr) { 19070e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!Int8Ptr) { 19080e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Int8Ptr = IRB.CreateBitCast(Ptr, IRB.getInt8PtrTy(), 19090e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Prefix + ".raw_cast"); 19100e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Int8PtrOffset = Offset; 19111e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth } 19120e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth 19130e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth OffsetPtr = Int8PtrOffset == 0 ? Int8Ptr : 19140e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth IRB.CreateInBoundsGEP(Int8Ptr, IRB.getInt(Int8PtrOffset), 19150e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Prefix + ".raw_idx"); 19161e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth } 19170e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Ptr = OffsetPtr; 19181e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 19190e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // On the off chance we were targeting i8*, guard the bitcast here. 19200e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (Ptr->getType() != PointerTy) 19210e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Ptr = IRB.CreateBitCast(Ptr, PointerTy, Prefix + ".cast"); 19221e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 19230e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return Ptr; 19240e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth} 19251e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 192611cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// \brief Test whether we can convert a value from the old to the new type. 192711cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// 192811cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// This predicate should be used to guard calls to convertValue in order to 192911cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// ensure that we only try to convert viable values. The strategy is that we 193011cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// will peel off single element struct and array wrappings to get to an 193111cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// underlying value, and convert that value. 193211cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruthstatic bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) { 193311cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (OldTy == NewTy) 193411cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return true; 193541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (IntegerType *OldITy = dyn_cast<IntegerType>(OldTy)) 193641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (IntegerType *NewITy = dyn_cast<IntegerType>(NewTy)) 193741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (NewITy->getBitWidth() >= OldITy->getBitWidth()) 193841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth return true; 193911cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (DL.getTypeSizeInBits(NewTy) != DL.getTypeSizeInBits(OldTy)) 194011cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return false; 194111cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType()) 194211cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return false; 194311cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth 194411cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (NewTy->isPointerTy() || OldTy->isPointerTy()) { 194511cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (NewTy->isPointerTy() && OldTy->isPointerTy()) 194611cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return true; 194711cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (NewTy->isIntegerTy() || OldTy->isIntegerTy()) 194811cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return true; 194911cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return false; 195011cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth } 195111cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth 195211cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return true; 195311cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth} 195411cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth 195511cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// \brief Generic routine to convert an SSA value to a value of a different 195611cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// type. 195711cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// 195811cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// This will try various different casting techniques, such as bitcasts, 195911cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// inttoptr, and ptrtoint casts. Use the \c canConvertValue predicate to test 196011cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth/// two types for viability with this routine. 196111cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruthstatic Value *convertValue(const DataLayout &DL, IRBuilder<> &IRB, Value *V, 196211cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth Type *Ty) { 196311cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth assert(canConvertValue(DL, V->getType(), Ty) && 196411cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth "Value not convertable to type"); 196511cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (V->getType() == Ty) 196611cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return V; 196741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (IntegerType *OldITy = dyn_cast<IntegerType>(V->getType())) 196841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (IntegerType *NewITy = dyn_cast<IntegerType>(Ty)) 196941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (NewITy->getBitWidth() > OldITy->getBitWidth()) 197041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth return IRB.CreateZExt(V, NewITy); 197111cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (V->getType()->isIntegerTy() && Ty->isPointerTy()) 197211cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return IRB.CreateIntToPtr(V, Ty); 197311cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth if (V->getType()->isPointerTy() && Ty->isIntegerTy()) 197411cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return IRB.CreatePtrToInt(V, Ty); 197511cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth 197611cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth return IRB.CreateBitCast(V, Ty); 197711cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth} 197811cb6ba5d0fe65c7249ef94c83df294b88d575f0Chandler Carruth 19790e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// \brief Test whether the given alloca partition can be promoted to a vector. 19800e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 19810e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// This is a quick test to check whether we can rewrite a particular alloca 19820e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// partition (and its newly formed alloca) into a vector alloca with only 19830e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// whole-vector loads and stores such that it could be promoted to a vector 19840e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// SSA value. We only can ensure this for a limited set of operations, and we 19850e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// don't want to do the rewrites unless we are confident that the result will 19860e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// be promotable, so we have an early test here. 19873574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmowstatic bool isVectorPromotionViable(const DataLayout &TD, 19880e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth Type *AllocaTy, 19890e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning &P, 19900e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth uint64_t PartitionBeginOffset, 19910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth uint64_t PartitionEndOffset, 19920e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning::const_use_iterator I, 19930e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth AllocaPartitioning::const_use_iterator E) { 19940e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth VectorType *Ty = dyn_cast<VectorType>(AllocaTy); 19950e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!Ty) 19960e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 19971e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 1998e21708e4aaf741d0c9ccb5a5ddc75738fea7b61fNadav Rotem uint64_t ElementSize = TD.getTypeSizeInBits(Ty->getScalarType()); 19991e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 20000e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // While the definition of LLVM vectors is bitpacked, we don't support sizes 20010e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // that aren't byte sized. 20020e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (ElementSize % 8) 20030e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 20040ea4c3d8c0d6b0815aa1381167d0f53baf186dbdBenjamin Kramer assert((TD.getTypeSizeInBits(Ty) % 8) == 0 && 20050ea4c3d8c0d6b0815aa1381167d0f53baf186dbdBenjamin Kramer "vector size not a multiple of element size?"); 20060e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth ElementSize /= 8; 20071e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 20080e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (; I != E; ++I) { 200941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth Use *U = I->getUse(); 201041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (!U) 20110e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth continue; // Skip dead use. 20121e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 20130e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth uint64_t BeginOffset = I->BeginOffset - PartitionBeginOffset; 20140e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth uint64_t BeginIndex = BeginOffset / ElementSize; 20150e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (BeginIndex * ElementSize != BeginOffset || 20160e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth BeginIndex >= Ty->getNumElements()) 20170e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 20180e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth uint64_t EndOffset = I->EndOffset - PartitionBeginOffset; 20190e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth uint64_t EndIndex = EndOffset / ElementSize; 20200e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (EndIndex * ElementSize != EndOffset || 20210e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth EndIndex > Ty->getNumElements()) 20220e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 20230e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth 202407df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth assert(EndIndex > BeginIndex && "Empty vector!"); 202507df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth uint64_t NumElements = EndIndex - BeginIndex; 202607df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth Type *PartitionTy 202707df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth = (NumElements == 1) ? Ty->getElementType() 202807df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth : VectorType::get(Ty->getElementType(), NumElements); 20290e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth 203041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { 20310e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (MI->isVolatile()) 20320e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 203341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(U->getUser())) { 20340e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const AllocaPartitioning::MemTransferOffsets &MTO 20350e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth = P.getMemTransferOffsets(*MTI); 20360e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!MTO.IsSplittable) 20370e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 20381e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth } 203941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth } else if (U->get()->getType()->getPointerElementType()->isStructTy()) { 20400e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Disable vector promotion when there are loads or stores of an FCA. 20410e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 204241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { 2043f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (LI->isVolatile()) 2044f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth return false; 204507df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth if (!canConvertValue(TD, PartitionTy, LI->getType())) 204607df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth return false; 204741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { 2048f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (SI->isVolatile()) 2049f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth return false; 205007df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth if (!canConvertValue(TD, SI->getValueOperand()->getType(), PartitionTy)) 205107df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth return false; 2052f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } else { 20530e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 20541e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth } 20550e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } 20560e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return true; 20570e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth} 20581e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 205981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth/// \brief Test whether the given alloca partition's integer operations can be 206081ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth/// widened to promotable ones. 20610e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth/// 206281ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth/// This is a quick test to check whether we can rewrite the integer loads and 206381ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth/// stores to a particular alloca into wider loads and stores and be able to 206481ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth/// promote the resulting alloca. 206581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruthstatic bool isIntegerWideningViable(const DataLayout &TD, 206681ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth Type *AllocaTy, 206781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth uint64_t AllocBeginOffset, 206881ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth AllocaPartitioning &P, 206981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth AllocaPartitioning::const_use_iterator I, 207081ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth AllocaPartitioning::const_use_iterator E) { 207181ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth uint64_t SizeInBits = TD.getTypeSizeInBits(AllocaTy); 20725bded7525b4eb1ae4ce3f7d05159de95d0f595f2Benjamin Kramer // Don't create integer types larger than the maximum bitwidth. 20735bded7525b4eb1ae4ce3f7d05159de95d0f595f2Benjamin Kramer if (SizeInBits > IntegerType::MAX_INT_BITS) 20745bded7525b4eb1ae4ce3f7d05159de95d0f595f2Benjamin Kramer return false; 207581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth 207681ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // Don't try to handle allocas with bit-padding. 207781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (SizeInBits != TD.getTypeStoreSizeInBits(AllocaTy)) 20780e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 20791e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 2080a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // We need to ensure that an integer type with the appropriate bitwidth can 2081a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // be converted to the alloca type, whatever that is. We don't want to force 2082a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // the alloca itself to have an integer type if there is a more suitable one. 2083a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits); 2084a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth if (!canConvertValue(TD, AllocaTy, IntTy) || 2085a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth !canConvertValue(TD, IntTy, AllocaTy)) 2086a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth return false; 2087a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth 208881ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth uint64_t Size = TD.getTypeStoreSize(AllocaTy); 208981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth 2090ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak // Check the uses to ensure the uses are (likely) promotable integer uses. 20910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // Also ensure that the alloca has a covering load or store. We don't want 2092ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak // to widen the integer operations only to fail to promote due to some other 209381ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // unsplittable entry (which we may make splittable later). 20940e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth bool WholeAllocaOp = false; 20950e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth for (; I != E; ++I) { 209641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth Use *U = I->getUse(); 209741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (!U) 20980e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth continue; // Skip dead use. 20991e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 210081ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth uint64_t RelBegin = I->BeginOffset - AllocBeginOffset; 210181ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth uint64_t RelEnd = I->EndOffset - AllocBeginOffset; 210281ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth 21030e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // We can't reasonably handle cases where the load or store extends past 21040e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth // the end of the aloca's type and into its padding. 210581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (RelEnd > Size) 21060e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 21071e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 210841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { 210981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (LI->isVolatile()) 21100e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 211181ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (RelBegin == 0 && RelEnd == Size) 21120e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth WholeAllocaOp = true; 211381ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) { 21143d9afa8e9714fb4d0fd0d210a35dcffbc6e33742Chandler Carruth if (ITy->getBitWidth() < TD.getTypeStoreSizeInBits(ITy)) 211581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth return false; 211681ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth continue; 211781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth } 211881ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // Non-integer loads need to be convertible from the alloca type so that 211981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // they are promotable. 212081ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (RelBegin != 0 || RelEnd != Size || 212181ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth !canConvertValue(TD, AllocaTy, LI->getType())) 212281ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth return false; 212341b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { 212481ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth Type *ValueTy = SI->getValueOperand()->getType(); 212581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (SI->isVolatile()) 21260e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 212781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (RelBegin == 0 && RelEnd == Size) 21280e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth WholeAllocaOp = true; 212981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) { 21303d9afa8e9714fb4d0fd0d210a35dcffbc6e33742Chandler Carruth if (ITy->getBitWidth() < TD.getTypeStoreSizeInBits(ITy)) 213181ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth return false; 213281ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth continue; 213381ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth } 213481ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // Non-integer stores need to be convertible to the alloca type so that 213581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // they are promotable. 213681ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (RelBegin != 0 || RelEnd != Size || 213781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth !canConvertValue(TD, ValueTy, AllocaTy)) 213881ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth return false; 213941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { 2140b0de1e31d11056037c4db3e2ecfe1547e85c3e1cChandler Carruth if (MI->isVolatile() || !isa<Constant>(MI->getLength())) 21410e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 214241b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(U->getUser())) { 21430e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth const AllocaPartitioning::MemTransferOffsets &MTO 21440e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth = P.getMemTransferOffsets(*MTI); 21450e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth if (!MTO.IsSplittable) 21460e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 21471e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth } 214841b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { 214981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (II->getIntrinsicID() != Intrinsic::lifetime_start && 215081ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth II->getIntrinsicID() != Intrinsic::lifetime_end) 215181ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth return false; 21520e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth } else { 21530e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return false; 21541e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth } 21551e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth } 21560e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth return WholeAllocaOp; 21570e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruth} 21581e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 21592360b7ad99eedecaae512373e7be49c2143550cfChandler Carruthstatic Value *extractInteger(const DataLayout &DL, IRBuilder<> &IRB, Value *V, 21602360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth IntegerType *Ty, uint64_t Offset, 21612360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth const Twine &Name) { 2162f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " start: " << *V << "\n"); 21632360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth IntegerType *IntTy = cast<IntegerType>(V->getType()); 21642360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && 21652360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth "Element extends past full value"); 21662360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth uint64_t ShAmt = 8*Offset; 21672360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth if (DL.isBigEndian()) 21682360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth ShAmt = 8*(DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset); 2169f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (ShAmt) { 21702360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = IRB.CreateLShr(V, ShAmt, Name + ".shift"); 2171f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " shifted: " << *V << "\n"); 2172f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } 21732360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(Ty->getBitWidth() <= IntTy->getBitWidth() && 21742360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth "Cannot extract to a larger integer!"); 2175f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (Ty != IntTy) { 21762360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = IRB.CreateTrunc(V, Ty, Name + ".trunc"); 2177f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " trunced: " << *V << "\n"); 2178f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } 21792360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth return V; 21802360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth} 21812360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth 21822360b7ad99eedecaae512373e7be49c2143550cfChandler Carruthstatic Value *insertInteger(const DataLayout &DL, IRBuilder<> &IRB, Value *Old, 21832360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Value *V, uint64_t Offset, const Twine &Name) { 21842360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth IntegerType *IntTy = cast<IntegerType>(Old->getType()); 21852360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth IntegerType *Ty = cast<IntegerType>(V->getType()); 21862360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(Ty->getBitWidth() <= IntTy->getBitWidth() && 21872360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth "Cannot insert a larger integer!"); 2188f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " start: " << *V << "\n"); 2189f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (Ty != IntTy) { 21902360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = IRB.CreateZExt(V, IntTy, Name + ".ext"); 2191f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " extended: " << *V << "\n"); 2192f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } 21932360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && 21942360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth "Element store outside of alloca store"); 21952360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth uint64_t ShAmt = 8*Offset; 21962360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth if (DL.isBigEndian()) 21972360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth ShAmt = 8*(DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset); 2198f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (ShAmt) { 21992360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = IRB.CreateShl(V, ShAmt, Name + ".shift"); 2200f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " shifted: " << *V << "\n"); 2201f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } 22022360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth 22032360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) { 22042360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt); 22052360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Old = IRB.CreateAnd(Old, Mask, Name + ".mask"); 2206f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " masked: " << *Old << "\n"); 22072360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = IRB.CreateOr(Old, V, Name + ".insert"); 2208f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " inserted: " << *V << "\n"); 22092360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth } 22102360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth return V; 22112360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth} 22122360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth 22135c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruthstatic Value *extractVector(IRBuilder<> &IRB, Value *V, 22145c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth unsigned BeginIndex, unsigned EndIndex, 22155c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth const Twine &Name) { 22165c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth VectorType *VecTy = cast<VectorType>(V->getType()); 22175c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth unsigned NumElements = EndIndex - BeginIndex; 22185c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth assert(NumElements <= VecTy->getNumElements() && "Too many elements!"); 22195c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth 22205c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth if (NumElements == VecTy->getNumElements()) 22215c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth return V; 22225c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth 22235c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth if (NumElements == 1) { 22245c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth V = IRB.CreateExtractElement(V, IRB.getInt32(BeginIndex), 22255c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth Name + ".extract"); 22265c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth DEBUG(dbgs() << " extract: " << *V << "\n"); 22275c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth return V; 22285c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth } 22295c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth 22305c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth SmallVector<Constant*, 8> Mask; 22315c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth Mask.reserve(NumElements); 22325c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth for (unsigned i = BeginIndex; i != EndIndex; ++i) 22335c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth Mask.push_back(IRB.getInt32(i)); 22345c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()), 22355c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth ConstantVector::get(Mask), 22365c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth Name + ".extract"); 22375c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth DEBUG(dbgs() << " shuffle: " << *V << "\n"); 22385c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth return V; 22395c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth} 22405c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth 22417f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruthstatic Value *insertVector(IRBuilder<> &IRB, Value *Old, Value *V, 22427f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth unsigned BeginIndex, const Twine &Name) { 22437f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth VectorType *VecTy = cast<VectorType>(Old->getType()); 22447f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth assert(VecTy && "Can only insert a vector into a vector"); 22457f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth 22467f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth VectorType *Ty = dyn_cast<VectorType>(V->getType()); 22477f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth if (!Ty) { 22487f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth // Single element to insert. 22497f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth V = IRB.CreateInsertElement(Old, V, IRB.getInt32(BeginIndex), 22507f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Name + ".insert"); 22517f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth DEBUG(dbgs() << " insert: " << *V << "\n"); 22527f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth return V; 22537f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth } 22547f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth 22557f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth assert(Ty->getNumElements() <= VecTy->getNumElements() && 22567f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth "Too many elements!"); 22577f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth if (Ty->getNumElements() == VecTy->getNumElements()) { 22587f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth assert(V->getType() == VecTy && "Vector type mismatch"); 22597f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth return V; 22607f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth } 22617f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth unsigned EndIndex = BeginIndex + Ty->getNumElements(); 22627f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth 22637f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth // When inserting a smaller vector into the larger to store, we first 22647f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth // use a shuffle vector to widen it with undef elements, and then 22657f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth // a second shuffle vector to select between the loaded vector and the 22667f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth // incoming vector. 22677f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth SmallVector<Constant*, 8> Mask; 22687f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Mask.reserve(VecTy->getNumElements()); 22697f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth for (unsigned i = 0; i != VecTy->getNumElements(); ++i) 22707f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth if (i >= BeginIndex && i < EndIndex) 22717f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Mask.push_back(IRB.getInt32(i - BeginIndex)); 22727f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth else 22737f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Mask.push_back(UndefValue::get(IRB.getInt32Ty())); 22747f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()), 22757f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth ConstantVector::get(Mask), 22767f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Name + ".expand"); 22777f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth DEBUG(dbgs() << " shuffle1: " << *V << "\n"); 22787f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth 22797f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Mask.clear(); 22807f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth for (unsigned i = 0; i != VecTy->getNumElements(); ++i) 22817f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth if (i >= BeginIndex && i < EndIndex) 22827f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Mask.push_back(IRB.getInt32(i)); 22837f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth else 22847f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Mask.push_back(IRB.getInt32(i + VecTy->getNumElements())); 22857f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth V = IRB.CreateShuffleVector(V, Old, ConstantVector::get(Mask), 22867f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Name + "insert"); 22877f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth DEBUG(dbgs() << " shuffle2: " << *V << "\n"); 22887f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth return V; 22897f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth} 22907f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth 22910e9da58af0e480af2fa87f8a61aa28b465fec280Chandler Carruthnamespace { 2292713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief Visitor to rewrite instructions using a partition of an alloca to 2293713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// use a new alloca. 2294713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 2295713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// Also implements the rewriting to vector-based accesses when the partition 2296713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// passes the isVectorPromotionViable predicate. Most of the rewriting logic 2297713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// lives here. 2298713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthclass AllocaPartitionRewriter : public InstVisitor<AllocaPartitionRewriter, 2299713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool> { 2300713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Befriend the base class so it can delegate to private visit methods. 2301713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth friend class llvm::InstVisitor<AllocaPartitionRewriter, bool>; 2302713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 23033574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow const DataLayout &TD; 2304713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning &P; 2305713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SROA &Pass; 2306713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaInst &OldAI, &NewAI; 2307713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset; 2308520eeaeffd5ef724ea5c68192f77331571f38060Chandler Carruth Type *NewAllocaTy; 2309713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2310713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If we are rewriting an alloca partition which can be written as pure 2311713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // vector operations, we stash extra information here. When VecTy is 2312ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak // non-null, we have some strict guarantees about the rewritten alloca: 2313713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // - The new alloca is exactly the size of the vector type here. 2314713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // - The accesses all either map to the entire vector or to a single 2315713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // element. 2316713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // - The set of accessing instructions is only one of those handled above 2317713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // in isVectorPromotionViable. Generally these are the same access kinds 2318713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // which are promotable via mem2reg. 2319713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth VectorType *VecTy; 2320713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *ElementTy; 2321713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t ElementSize; 2322713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2323bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth // This is a convenience and flag variable that will be null unless the new 232481ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // alloca's integer operations should be widened to this integer type due to 232581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth // passing isIntegerWideningViable above. If it is non-null, the desired 2326bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth // integer type will be stored here for easy access during rewriting. 232781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth IntegerType *IntTy; 2328bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth 2329713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // The offset of the partition user currently being rewritten. 2330713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t BeginOffset, EndOffset; 233141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth bool IsSplit; 233277c12700339496cfa18e3004aabb807ec1c5a1a2Chandler Carruth Use *OldUse; 2333713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Instruction *OldPtr; 2334713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2335713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // The name prefix to use when rewriting instructions for this alloca. 2336713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth std::string NamePrefix; 2337713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2338713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthpublic: 23393574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow AllocaPartitionRewriter(const DataLayout &TD, AllocaPartitioning &P, 2340713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning::iterator PI, 2341713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth SROA &Pass, AllocaInst &OldAI, AllocaInst &NewAI, 2342713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t NewBeginOffset, uint64_t NewEndOffset) 2343713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth : TD(TD), P(P), Pass(Pass), 2344713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth OldAI(OldAI), NewAI(NewAI), 2345713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth NewAllocaBeginOffset(NewBeginOffset), 2346713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth NewAllocaEndOffset(NewEndOffset), 2347520eeaeffd5ef724ea5c68192f77331571f38060Chandler Carruth NewAllocaTy(NewAI.getAllocatedType()), 234881ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth VecTy(), ElementTy(), ElementSize(), IntTy(), 234941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth BeginOffset(), EndOffset(), IsSplit(), OldUse(), OldPtr() { 2350713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2351713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2352713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief Visit the users of the alloca partition and rewrite them. 2353713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitUsers(AllocaPartitioning::const_use_iterator I, 2354713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning::const_use_iterator E) { 2355713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (isVectorPromotionViable(TD, NewAI.getAllocatedType(), P, 2356713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth NewAllocaBeginOffset, NewAllocaEndOffset, 2357713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth I, E)) { 2358713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++NumVectorized; 2359713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth VecTy = cast<VectorType>(NewAI.getAllocatedType()); 2360713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ElementTy = VecTy->getElementType(); 2361e21708e4aaf741d0c9ccb5a5ddc75738fea7b61fNadav Rotem assert((TD.getTypeSizeInBits(VecTy->getScalarType()) % 8) == 0 && 2362713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth "Only multiple-of-8 sized vector elements are viable"); 2363e21708e4aaf741d0c9ccb5a5ddc75738fea7b61fNadav Rotem ElementSize = TD.getTypeSizeInBits(VecTy->getScalarType()) / 8; 236481ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth } else if (isIntegerWideningViable(TD, NewAI.getAllocatedType(), 236581ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth NewAllocaBeginOffset, P, I, E)) { 236681ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth IntTy = Type::getIntNTy(NewAI.getContext(), 236781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth TD.getTypeSizeInBits(NewAI.getAllocatedType())); 2368713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2369713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool CanSROA = true; 2370713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (; I != E; ++I) { 237141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (!I->getUse()) 2372fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth continue; // Skip dead uses. 2373713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth BeginOffset = I->BeginOffset; 2374713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth EndOffset = I->EndOffset; 237541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth IsSplit = I->isSplit(); 237641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth OldUse = I->getUse(); 237741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth OldPtr = cast<Instruction>(OldUse->get()); 2378713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth NamePrefix = (Twine(NewAI.getName()) + "." + Twine(BeginOffset)).str(); 237941b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth CanSROA &= visit(cast<Instruction>(OldUse->getUser())); 2380713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2381713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (VecTy) { 2382713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(CanSROA); 2383713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth VecTy = 0; 2384713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ElementTy = 0; 2385713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ElementSize = 0; 2386713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 238781ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth if (IntTy) { 238881ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth assert(CanSROA); 238981ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth IntTy = 0; 239081ff90db4403f9b4ee3574f00ebdfba4a12177c9Chandler Carruth } 2391713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return CanSROA; 2392713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2393713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2394713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthprivate: 2395713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Every instruction which can end up as a user must have a rewrite rule. 2396713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitInstruction(Instruction &I) { 2397713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n"); 2398713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth llvm_unreachable("No rewrite rule for this instruction!"); 2399713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2400713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2401713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Twine getName(const Twine &Suffix) { 2402713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return NamePrefix + Suffix; 2403713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2404713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2405713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *getAdjustedAllocaPtr(IRBuilder<> &IRB, Type *PointerTy) { 2406713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(BeginOffset >= NewAllocaBeginOffset); 2407426c2bf5cdd2173e4a33aea8cb92cf684a724f4bChandler Carruth APInt Offset(TD.getPointerSizeInBits(), BeginOffset - NewAllocaBeginOffset); 2408713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return getAdjustedPtr(IRB, TD, &NewAI, Offset, PointerTy, getName("")); 2409713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2410713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2411f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// \brief Compute suitable alignment to access an offset into the new alloca. 2412f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth unsigned getOffsetAlign(uint64_t Offset) { 2413673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth unsigned NewAIAlign = NewAI.getAlignment(); 2414673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth if (!NewAIAlign) 2415673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth NewAIAlign = TD.getABITypeAlignment(NewAI.getAllocatedType()); 2416673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth return MinAlign(NewAIAlign, Offset); 2417673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth } 2418f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth 2419f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// \brief Compute suitable alignment to access this partition of the new 2420f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// alloca. 2421f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth unsigned getPartitionAlign() { 2422f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth return getOffsetAlign(BeginOffset - NewAllocaBeginOffset); 2423673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth } 2424673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth 2425f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// \brief Compute suitable alignment to access a type at an offset of the 2426f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// new alloca. 2427f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// 2428f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// \returns zero if the type's ABI alignment is a suitable alignment, 2429f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// otherwise returns the maximal suitable alignment. 2430f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth unsigned getOffsetTypeAlign(Type *Ty, uint64_t Offset) { 2431f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth unsigned Align = getOffsetAlign(Offset); 2432f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth return Align == TD.getABITypeAlignment(Ty) ? 0 : Align; 2433f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth } 2434f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth 2435f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// \brief Compute suitable alignment to access a type at the beginning of 2436f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// this partition of the new alloca. 2437f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// 2438f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth /// See \c getOffsetTypeAlign for details; this routine delegates to it. 2439f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth unsigned getPartitionTypeAlign(Type *Ty) { 2440f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth return getOffsetTypeAlign(Ty, BeginOffset - NewAllocaBeginOffset); 2441673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth } 2442673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth 244307df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth unsigned getIndex(uint64_t Offset) { 2444713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(VecTy && "Can only call getIndex when rewriting a vector"); 2445713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t RelOffset = Offset - NewAllocaBeginOffset; 2446713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds"); 2447713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint32_t Index = RelOffset / ElementSize; 2448713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(Index * ElementSize == RelOffset); 244907df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth return Index; 2450713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2451713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2452713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth void deleteIfTriviallyDead(Value *V) { 2453713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Instruction *I = cast<Instruction>(V); 2454713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (isInstructionTriviallyDead(I)) 2455f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(I); 2456713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2457713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 24588ab1efd5fcff755b4ccc86d6d3da52cfe7cf0b9dChandler Carruth Value *rewriteVectorizedLoadInst(IRBuilder<> &IRB) { 24598ab1efd5fcff755b4ccc86d6d3da52cfe7cf0b9dChandler Carruth unsigned BeginIndex = getIndex(BeginOffset); 24608ab1efd5fcff755b4ccc86d6d3da52cfe7cf0b9dChandler Carruth unsigned EndIndex = getIndex(EndOffset); 24618ab1efd5fcff755b4ccc86d6d3da52cfe7cf0b9dChandler Carruth assert(EndIndex > BeginIndex && "Empty vector!"); 24625c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth 24635c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 24645c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth getName(".load")); 24655c531eb8b121b83ad14ac5cf94c8fd9c50ef6f5bChandler Carruth return extractVector(IRB, V, BeginIndex, EndIndex, getName(".vec")); 24668ab1efd5fcff755b4ccc86d6d3da52cfe7cf0b9dChandler Carruth } 24678ab1efd5fcff755b4ccc86d6d3da52cfe7cf0b9dChandler Carruth 2468f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Value *rewriteIntegerLoad(IRBuilder<> &IRB, LoadInst &LI) { 24692360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(IntTy && "We cannot insert an integer to the alloca"); 2470bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth assert(!LI.isVolatile()); 24712360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 24722360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth getName(".load")); 24732360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = convertValue(TD, IRB, V, IntTy); 24742360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset"); 24752360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth uint64_t Offset = BeginOffset - NewAllocaBeginOffset; 2476f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (Offset > 0 || EndOffset < NewAllocaEndOffset) 2477f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth V = extractInteger(TD, IRB, V, cast<IntegerType>(LI.getType()), Offset, 2478f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth getName(".extract")); 2479f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth return V; 2480bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth } 2481bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth 2482713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitLoadInst(LoadInst &LI) { 2483713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " original: " << LI << "\n"); 2484713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *OldOp = LI.getOperand(0); 2485713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(OldOp == OldPtr); 2486713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2487a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth uint64_t Size = EndOffset - BeginOffset; 2488176792990e1b8df4894d0dabddb735abbb254e7cChandler Carruth 24899497005d38b1c0d5183374493c08982eb8586eadJakub Staszak IRBuilder<> IRB(&LI); 249041b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), Size * 8) 249141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth : LI.getType(); 2492f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth bool IsPtrAdjusted = false; 2493f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Value *V; 2494f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (VecTy) { 24958ab1efd5fcff755b4ccc86d6d3da52cfe7cf0b9dChandler Carruth V = rewriteVectorizedLoadInst(IRB); 2496f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } else if (IntTy && LI.getType()->isIntegerTy()) { 2497f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth V = rewriteIntegerLoad(IRB, LI); 2498f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } else if (BeginOffset == NewAllocaBeginOffset && 2499f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth canConvertValue(TD, NewAllocaTy, LI.getType())) { 2500f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 2501f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth LI.isVolatile(), getName(".load")); 2502f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } else { 2503f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Type *LTy = TargetTy->getPointerTo(); 2504f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth V = IRB.CreateAlignedLoad(getAdjustedAllocaPtr(IRB, LTy), 2505f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth getPartitionTypeAlign(TargetTy), 2506f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth LI.isVolatile(), getName(".load")); 2507f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth IsPtrAdjusted = true; 2508f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } 2509f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth V = convertValue(TD, IRB, V, TargetTy); 2510f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth 251141b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (IsSplit) { 2512a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth assert(!LI.isVolatile()); 2513a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth assert(LI.getType()->isIntegerTy() && 2514a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth "Only integer type loads and stores are split"); 251541b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth assert(Size < TD.getTypeStoreSize(LI.getType()) && 251641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth "Split load isn't smaller than original load"); 2517a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth assert(LI.getType()->getIntegerBitWidth() == 2518a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth TD.getTypeStoreSizeInBits(LI.getType()) && 2519a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth "Non-byte-multiple bit width"); 2520a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // Move the insertion point just past the load so that we can refer to it. 2521a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth IRB.SetInsertPoint(llvm::next(BasicBlock::iterator(&LI))); 2522a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // Create a placeholder value with the same type as LI to use as the 2523a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // basis for the new value. This allows us to replace the uses of LI with 2524a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // the computed value, and then replace the placeholder with LI, leaving 2525a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth // LI only used for this computation. 2526a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth Value *Placeholder 25275801ff93e3b81ee18fdd57f59d0e776ac3b27384Jakub Staszak = new LoadInst(UndefValue::get(LI.getType()->getPointerTo())); 2528a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth V = insertInteger(TD, IRB, Placeholder, V, BeginOffset, 2529a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth getName(".insert")); 2530a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth LI.replaceAllUsesWith(V); 2531a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth Placeholder->replaceAllUsesWith(&LI); 25325801ff93e3b81ee18fdd57f59d0e776ac3b27384Jakub Staszak delete Placeholder; 2533f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } else { 2534f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth LI.replaceAllUsesWith(V); 2535520eeaeffd5ef724ea5c68192f77331571f38060Chandler Carruth } 2536520eeaeffd5ef724ea5c68192f77331571f38060Chandler Carruth 2537f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(&LI); 2538713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth deleteIfTriviallyDead(OldOp); 2539f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " to: " << *V << "\n"); 2540f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth return !LI.isVolatile() && !IsPtrAdjusted; 2541fd22019ec36e8335254430e61a1e5a3f8a681f6aEvan Cheng } 2542fd22019ec36e8335254430e61a1e5a3f8a681f6aEvan Cheng 2543f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth bool rewriteVectorizedStoreInst(IRBuilder<> &IRB, Value *V, 2544f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth StoreInst &SI, Value *OldOp) { 254507df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth unsigned BeginIndex = getIndex(BeginOffset); 254607df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth unsigned EndIndex = getIndex(EndOffset); 254707df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth assert(EndIndex > BeginIndex && "Empty vector!"); 254807df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth unsigned NumElements = EndIndex - BeginIndex; 254907df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth assert(NumElements <= VecTy->getNumElements() && "Too many elements!"); 255007df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth Type *PartitionTy 255107df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth = (NumElements == 1) ? ElementTy 255207df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth : VectorType::get(ElementTy, NumElements); 255307df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth if (V->getType() != PartitionTy) 255407df765e65204203f0185a7a243e5ec3a5c4b21cChandler Carruth V = convertValue(TD, IRB, V, PartitionTy); 2555d6e4397a5b68836d9dae1d4a30da6f54ece51219Chandler Carruth 2556d6e4397a5b68836d9dae1d4a30da6f54ece51219Chandler Carruth // Mix in the existing elements. 25577f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 25587f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth getName(".load")); 25597f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth V = insertVector(IRB, Old, V, BeginIndex, getName(".vec")); 2560d6e4397a5b68836d9dae1d4a30da6f54ece51219Chandler Carruth 256181b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); 2562f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(&SI); 2563713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2564713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth (void)Store; 2565713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " to: " << *Store << "\n"); 2566713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return true; 2567713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2568713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2569f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth bool rewriteIntegerStore(IRBuilder<> &IRB, Value *V, StoreInst &SI) { 25702360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(IntTy && "We cannot extract an integer from the alloca"); 2571bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth assert(!SI.isVolatile()); 25722360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth if (TD.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) { 25732360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 25742360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth getName(".oldload")); 25752360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Old = convertValue(TD, IRB, Old, IntTy); 25762360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset"); 25772360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth uint64_t Offset = BeginOffset - NewAllocaBeginOffset; 25782360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = insertInteger(TD, IRB, Old, SI.getValueOperand(), Offset, 25792360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth getName(".insert")); 25802360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth } 25812360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth V = convertValue(TD, IRB, V, NewAllocaTy); 25822360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); 2583f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(&SI); 2584bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth (void)Store; 2585bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth DEBUG(dbgs() << " to: " << *Store << "\n"); 2586bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth return true; 2587bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth } 2588bc4021f31eaa97ee52655828da3e3de14a39e4a6Chandler Carruth 2589713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitStoreInst(StoreInst &SI) { 2590713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " original: " << SI << "\n"); 2591713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *OldOp = SI.getOperand(1); 2592713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(OldOp == OldPtr); 2593713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IRBuilder<> IRB(&SI); 2594713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2595f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Value *V = SI.getValueOperand(); 2596520eeaeffd5ef724ea5c68192f77331571f38060Chandler Carruth 2597b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth // Strip all inbounds GEPs and pointer casts to try to dig out any root 2598b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth // alloca that should be re-examined after promoting this alloca. 2599f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (V->getType()->isPointerTy()) 2600f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets())) 2601b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth Pass.PostPromotionWorklist.insert(AI); 2602b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth 2603f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth uint64_t Size = EndOffset - BeginOffset; 2604f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (Size < TD.getTypeStoreSize(V->getType())) { 2605f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth assert(!SI.isVolatile()); 260641b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth assert(IsSplit && "A seemingly split store isn't splittable"); 2607f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth assert(V->getType()->isIntegerTy() && 2608f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth "Only integer type loads and stores are split"); 2609f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth assert(V->getType()->getIntegerBitWidth() == 2610f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth TD.getTypeStoreSizeInBits(V->getType()) && 2611f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth "Non-byte-multiple bit width"); 2612f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), Size * 8); 2613f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth V = extractInteger(TD, IRB, V, NarrowTy, BeginOffset, 2614f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth getName(".extract")); 2615520eeaeffd5ef724ea5c68192f77331571f38060Chandler Carruth } 2616520eeaeffd5ef724ea5c68192f77331571f38060Chandler Carruth 2617f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (VecTy) 2618f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth return rewriteVectorizedStoreInst(IRB, V, SI, OldOp); 2619f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (IntTy && V->getType()->isIntegerTy()) 2620f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth return rewriteIntegerStore(IRB, V, SI); 2621713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2622f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth StoreInst *NewSI; 2623f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth if (BeginOffset == NewAllocaBeginOffset && 2624f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth canConvertValue(TD, V->getType(), NewAllocaTy)) { 2625f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth V = convertValue(TD, IRB, V, NewAllocaTy); 2626f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth NewSI = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), 2627f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth SI.isVolatile()); 2628f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } else { 2629f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Value *NewPtr = getAdjustedAllocaPtr(IRB, V->getType()->getPointerTo()); 2630f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth NewSI = IRB.CreateAlignedStore(V, NewPtr, 2631f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth getPartitionTypeAlign(V->getType()), 2632f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth SI.isVolatile()); 2633f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth } 2634f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth (void)NewSI; 2635f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(&SI); 2636713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth deleteIfTriviallyDead(OldOp); 2637f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth 2638f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DEBUG(dbgs() << " to: " << *NewSI << "\n"); 2639f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth return NewSI->getPointerOperand() == &NewAI && !SI.isVolatile(); 2640713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2641713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2642225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// \brief Compute an integer value from splatting an i8 across the given 2643225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// number of bytes. 2644225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// 2645225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// Note that this routine assumes an i8 is a byte. If that isn't true, don't 2646225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// call this routine. 2647ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak /// FIXME: Heed the advice above. 2648225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// 2649225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// \param V The i8 value to splat. 2650225d25de490386dd6398842c19f567b921a97db6Chandler Carruth /// \param Size The number of bytes in the output (assuming i8 is one byte) 2651225d25de490386dd6398842c19f567b921a97db6Chandler Carruth Value *getIntegerSplat(IRBuilder<> &IRB, Value *V, unsigned Size) { 2652225d25de490386dd6398842c19f567b921a97db6Chandler Carruth assert(Size > 0 && "Expected a positive number of bytes."); 2653225d25de490386dd6398842c19f567b921a97db6Chandler Carruth IntegerType *VTy = cast<IntegerType>(V->getType()); 2654225d25de490386dd6398842c19f567b921a97db6Chandler Carruth assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte"); 2655225d25de490386dd6398842c19f567b921a97db6Chandler Carruth if (Size == 1) 2656225d25de490386dd6398842c19f567b921a97db6Chandler Carruth return V; 2657225d25de490386dd6398842c19f567b921a97db6Chandler Carruth 2658225d25de490386dd6398842c19f567b921a97db6Chandler Carruth Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size*8); 2659225d25de490386dd6398842c19f567b921a97db6Chandler Carruth V = IRB.CreateMul(IRB.CreateZExt(V, SplatIntTy, getName(".zext")), 2660225d25de490386dd6398842c19f567b921a97db6Chandler Carruth ConstantExpr::getUDiv( 2661225d25de490386dd6398842c19f567b921a97db6Chandler Carruth Constant::getAllOnesValue(SplatIntTy), 2662225d25de490386dd6398842c19f567b921a97db6Chandler Carruth ConstantExpr::getZExt( 2663225d25de490386dd6398842c19f567b921a97db6Chandler Carruth Constant::getAllOnesValue(V->getType()), 2664225d25de490386dd6398842c19f567b921a97db6Chandler Carruth SplatIntTy)), 2665225d25de490386dd6398842c19f567b921a97db6Chandler Carruth getName(".isplat")); 2666225d25de490386dd6398842c19f567b921a97db6Chandler Carruth return V; 2667225d25de490386dd6398842c19f567b921a97db6Chandler Carruth } 2668225d25de490386dd6398842c19f567b921a97db6Chandler Carruth 266917c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth /// \brief Compute a vector splat for a given element value. 267017c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth Value *getVectorSplat(IRBuilder<> &IRB, Value *V, unsigned NumElements) { 26716c3074958370bf25dc6e4e4b757f0c083e245dbeBenjamin Kramer V = IRB.CreateVectorSplat(NumElements, V, NamePrefix); 267217c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth DEBUG(dbgs() << " splat: " << *V << "\n"); 267317c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth return V; 267417c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth } 267517c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth 2676713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitMemSetInst(MemSetInst &II) { 2677713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " original: " << II << "\n"); 2678713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IRBuilder<> IRB(&II); 2679713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(II.getRawDest() == OldPtr); 2680713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2681713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If the memset has a variable size, it cannot be split, just adjust the 2682713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // pointer to the new alloca. 2683713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!isa<Constant>(II.getLength())) { 2684713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.setDest(getAdjustedAllocaPtr(IRB, II.getRawDest()->getType())); 2685d0ac06d18812dd1c20ad55f9324c550609138593Chandler Carruth Type *CstTy = II.getAlignmentCst()->getType(); 2686f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth II.setAlignment(ConstantInt::get(CstTy, getPartitionAlign())); 2687d0ac06d18812dd1c20ad55f9324c550609138593Chandler Carruth 2688713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth deleteIfTriviallyDead(OldPtr); 2689713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; 2690713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2691713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2692713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Record this instruction for deletion. 2693f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(&II); 2694713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2695713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *AllocaTy = NewAI.getAllocatedType(); 2696713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *ScalarTy = AllocaTy->getScalarType(); 2697713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2698713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If this doesn't map cleanly onto the alloca type, and that type isn't 2699713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // a single value type, just emit a memset. 270094fc64c42fe27aa16dd5611eb3657ff0297801c8Chandler Carruth if (!VecTy && !IntTy && 270194fc64c42fe27aa16dd5611eb3657ff0297801c8Chandler Carruth (BeginOffset != NewAllocaBeginOffset || 270294fc64c42fe27aa16dd5611eb3657ff0297801c8Chandler Carruth EndOffset != NewAllocaEndOffset || 270394fc64c42fe27aa16dd5611eb3657ff0297801c8Chandler Carruth !AllocaTy->isSingleValueType() || 270417c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth !TD.isLegalInteger(TD.getTypeSizeInBits(ScalarTy)) || 270517c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth TD.getTypeSizeInBits(ScalarTy)%8 != 0)) { 2706713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *SizeTy = II.getLength()->getType(); 2707713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Constant *Size = ConstantInt::get(SizeTy, EndOffset - BeginOffset); 2708713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth CallInst *New 2709713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth = IRB.CreateMemSet(getAdjustedAllocaPtr(IRB, 2710713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.getRawDest()->getType()), 2711f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth II.getValue(), Size, getPartitionAlign(), 2712713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.isVolatile()); 2713713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth (void)New; 2714713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " to: " << *New << "\n"); 2715713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; 2716713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2717713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2718713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If we can represent this as a simple value, we have to build the actual 2719713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // value to store, which requires expanding the byte present in memset to 2720713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // a sensible representation for the alloca type. This is essentially 272117c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth // splatting the byte to a sufficiently wide integer, splatting it across 272217c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth // any desired vector width, and bitcasting to the final type. 27230ea4c3d8c0d6b0815aa1381167d0f53baf186dbdBenjamin Kramer Value *V; 2724713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 272517c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth if (VecTy) { 272617c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth // If this is a memset of a vectorized alloca, insert it. 272717c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth assert(ElementTy == ScalarTy); 272817c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth 272917c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth unsigned BeginIndex = getIndex(BeginOffset); 273017c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth unsigned EndIndex = getIndex(EndOffset); 273117c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth assert(EndIndex > BeginIndex && "Empty vector!"); 273217c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth unsigned NumElements = EndIndex - BeginIndex; 273317c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth assert(NumElements <= VecTy->getNumElements() && "Too many elements!"); 273417c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth 273517c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth Value *Splat = getIntegerSplat(IRB, II.getValue(), 273617c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth TD.getTypeSizeInBits(ElementTy)/8); 27378bbff2348d378192b332db38394498d83ed4feebChandler Carruth Splat = convertValue(TD, IRB, Splat, ElementTy); 27388bbff2348d378192b332db38394498d83ed4feebChandler Carruth if (NumElements > 1) 273917c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth Splat = getVectorSplat(IRB, Splat, NumElements); 274017c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth 27417f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 27427f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth getName(".oldload")); 27437f7d201d737ecb354abd683d63ab8abbf83158c3Chandler Carruth V = insertVector(IRB, Old, Splat, BeginIndex, getName(".vec")); 274417c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth } else if (IntTy) { 274517c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth // If this is a memset on an alloca where we can widen stores, insert the 274617c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth // set integer. 274794fc64c42fe27aa16dd5611eb3657ff0297801c8Chandler Carruth assert(!II.isVolatile()); 2748713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 27490ea4c3d8c0d6b0815aa1381167d0f53baf186dbdBenjamin Kramer uint64_t Size = EndOffset - BeginOffset; 275017c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth V = getIntegerSplat(IRB, II.getValue(), Size); 275117c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth 275217c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth if (IntTy && (BeginOffset != NewAllocaBeginOffset || 275317c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth EndOffset != NewAllocaBeginOffset)) { 275417c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 275517c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth getName(".oldload")); 275617c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth Old = convertValue(TD, IRB, Old, IntTy); 275717c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset"); 275817c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth uint64_t Offset = BeginOffset - NewAllocaBeginOffset; 275917c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth V = insertInteger(TD, IRB, Old, V, Offset, getName(".insert")); 276017c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth } else { 276117c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth assert(V->getType() == IntTy && 276217c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth "Wrong type for an alloca wide integer!"); 276317c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth } 27645814a88ce9ebe415015b37477c841304fa8b7b1aChandler Carruth V = convertValue(TD, IRB, V, AllocaTy); 276517c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth } else { 276617c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth // Established these invariants above. 276717c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth assert(BeginOffset == NewAllocaBeginOffset); 276817c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth assert(EndOffset == NewAllocaEndOffset); 276917c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth 277017c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth V = getIntegerSplat(IRB, II.getValue(), 277117c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth TD.getTypeSizeInBits(ScalarTy)/8); 277217c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy)) 277317c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth V = getVectorSplat(IRB, V, AllocaVecTy->getNumElements()); 27745814a88ce9ebe415015b37477c841304fa8b7b1aChandler Carruth 27755814a88ce9ebe415015b37477c841304fa8b7b1aChandler Carruth V = convertValue(TD, IRB, V, AllocaTy); 277617c84ea594c6f10cb13c84ebe765b54f234c82efChandler Carruth } 277794fc64c42fe27aa16dd5611eb3657ff0297801c8Chandler Carruth 27785814a88ce9ebe415015b37477c841304fa8b7b1aChandler Carruth Value *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), 277981b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth II.isVolatile()); 2780713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth (void)New; 2781713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " to: " << *New << "\n"); 2782713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return !II.isVolatile(); 2783713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2784713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2785713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitMemTransferInst(MemTransferInst &II) { 2786713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Rewriting of memory transfer instructions can be a bit tricky. We break 2787713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // them into two categories: split intrinsics and unsplit intrinsics. 2788713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2789713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " original: " << II << "\n"); 2790713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IRBuilder<> IRB(&II); 2791713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2792713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(II.getRawSource() == OldPtr || II.getRawDest() == OldPtr); 2793713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool IsDest = II.getRawDest() == OldPtr; 2794713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2795713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const AllocaPartitioning::MemTransferOffsets &MTO 2796713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth = P.getMemTransferOffsets(II); 2797713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2798673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth // Compute the relative offset within the transfer. 2799426c2bf5cdd2173e4a33aea8cb92cf684a724f4bChandler Carruth unsigned IntPtrWidth = TD.getPointerSizeInBits(); 2800673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth APInt RelOffset(IntPtrWidth, BeginOffset - (IsDest ? MTO.DestBegin 2801673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth : MTO.SourceBegin)); 2802673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth 2803673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth unsigned Align = II.getAlignment(); 2804673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth if (Align > 1) 2805673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth Align = MinAlign(RelOffset.zextOrTrunc(64).getZExtValue(), 2806f710fb14ee951fe533b94eb1046d422e94d4f1f3Chandler Carruth MinAlign(II.getAlignment(), getPartitionAlign())); 2807673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth 2808713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // For unsplit intrinsics, we simply modify the source and destination 2809713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // pointers in place. This isn't just an optimization, it is a matter of 2810713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // correctness. With unsplit intrinsics we may be dealing with transfers 2811713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // within a single alloca before SROA ran, or with transfers that have 2812713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // a variable length. We may also be dealing with memmove instead of 2813713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // memcpy, and so simply updating the pointers is the necessary for us to 2814713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // update both source and dest of a single call. 2815713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!MTO.IsSplittable) { 2816713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *OldOp = IsDest ? II.getRawDest() : II.getRawSource(); 2817713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (IsDest) 2818713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.setDest(getAdjustedAllocaPtr(IRB, II.getRawDest()->getType())); 2819713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth else 2820713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.setSource(getAdjustedAllocaPtr(IRB, II.getRawSource()->getType())); 2821713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2822d0ac06d18812dd1c20ad55f9324c550609138593Chandler Carruth Type *CstTy = II.getAlignmentCst()->getType(); 2823673850aa2dceb8fe1b7a20b72339bf803af8609fChandler Carruth II.setAlignment(ConstantInt::get(CstTy, Align)); 2824d0ac06d18812dd1c20ad55f9324c550609138593Chandler Carruth 2825713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " to: " << II << "\n"); 2826713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth deleteIfTriviallyDead(OldOp); 2827713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; 2828713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2829713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // For split transfer intrinsics we have an incredibly useful assurance: 2830713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // the source and destination do not reside within the same alloca, and at 2831713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // least one of them does not escape. This means that we can replace 2832713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // memmove with memcpy, and we don't need to worry about all manner of 2833713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // downsides to splitting and transforming the operations. 2834713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2835713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If this doesn't map cleanly onto the alloca type, and that type isn't 2836713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // a single value type, just emit a memcpy. 2837713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool EmitMemCpy 2838d2cd73f6a5103f5de221a67410e35e6bee211fa3Chandler Carruth = !VecTy && !IntTy && (BeginOffset != NewAllocaBeginOffset || 2839d2cd73f6a5103f5de221a67410e35e6bee211fa3Chandler Carruth EndOffset != NewAllocaEndOffset || 2840d2cd73f6a5103f5de221a67410e35e6bee211fa3Chandler Carruth !NewAI.getAllocatedType()->isSingleValueType()); 2841713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2842713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If we're just going to emit a memcpy, the alloca hasn't changed, and the 2843713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // size hasn't been shrunk based on analysis of the viable range, this is 2844713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // a no-op. 2845713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (EmitMemCpy && &OldAI == &NewAI) { 2846713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t OrigBegin = IsDest ? MTO.DestBegin : MTO.SourceBegin; 2847713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t OrigEnd = IsDest ? MTO.DestEnd : MTO.SourceEnd; 2848713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Ensure the start lines up. 2849713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(BeginOffset == OrigBegin); 2850d0807690de6c735af84e35dd76a30420eaded16bBenjamin Kramer (void)OrigBegin; 2851713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2852713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Rewrite the size as needed. 2853713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (EndOffset != OrigEnd) 2854713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.setLength(ConstantInt::get(II.getLength()->getType(), 2855713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth EndOffset - BeginOffset)); 2856713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; 2857713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2858713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Record this instruction for deletion. 2859f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(&II); 2860713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2861713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Strip all inbounds GEPs and pointer casts to try to dig out any root 2862713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // alloca that should be re-examined after rewriting this instruction. 286399a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest(); 2864713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (AllocaInst *AI 2865713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth = dyn_cast<AllocaInst>(OtherPtr->stripInBoundsOffsets())) 2866b3dca3f50e649caa29f97736f0fa4b2fe1097034Chandler Carruth Pass.Worklist.insert(AI); 2867713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2868713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (EmitMemCpy) { 286999a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Type *OtherPtrTy = IsDest ? II.getRawSource()->getType() 287099a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth : II.getRawDest()->getType(); 287199a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth 287299a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth // Compute the other pointer, folding as much as possible to produce 287399a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth // a single, simple GEP in most cases. 287499a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth OtherPtr = getAdjustedPtr(IRB, TD, OtherPtr, RelOffset, OtherPtrTy, 287599a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth getName("." + OtherPtr->getName())); 287699a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth 2877713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *OurPtr 2878713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth = getAdjustedAllocaPtr(IRB, IsDest ? II.getRawDest()->getType() 2879713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth : II.getRawSource()->getType()); 2880713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *SizeTy = II.getLength()->getType(); 2881713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Constant *Size = ConstantInt::get(SizeTy, EndOffset - BeginOffset); 2882713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2883713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth CallInst *New = IRB.CreateMemCpy(IsDest ? OurPtr : OtherPtr, 2884713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IsDest ? OtherPtr : OurPtr, 288581b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth Size, Align, II.isVolatile()); 2886713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth (void)New; 2887713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " to: " << *New << "\n"); 2888713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; 2889713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2890713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2891322e9ba2cb92330a345bd6e606c64a7b16bd929fChandler Carruth // Note that we clamp the alignment to 1 here as a 0 alignment for a memcpy 2892322e9ba2cb92330a345bd6e606c64a7b16bd929fChandler Carruth // is equivalent to 1, but that isn't true if we end up rewriting this as 2893322e9ba2cb92330a345bd6e606c64a7b16bd929fChandler Carruth // a load or store. 2894322e9ba2cb92330a345bd6e606c64a7b16bd929fChandler Carruth if (!Align) 2895322e9ba2cb92330a345bd6e606c64a7b16bd929fChandler Carruth Align = 1; 2896322e9ba2cb92330a345bd6e606c64a7b16bd929fChandler Carruth 289799a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth bool IsWholeAlloca = BeginOffset == NewAllocaBeginOffset && 289899a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth EndOffset == NewAllocaEndOffset; 289999a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth uint64_t Size = EndOffset - BeginOffset; 290099a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth unsigned BeginIndex = VecTy ? getIndex(BeginOffset) : 0; 290199a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth unsigned EndIndex = VecTy ? getIndex(EndOffset) : 0; 290299a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth unsigned NumElements = EndIndex - BeginIndex; 290399a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth IntegerType *SubIntTy 290499a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth = IntTy ? Type::getIntNTy(IntTy->getContext(), Size*8) : 0; 290599a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth 290699a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Type *OtherPtrTy = NewAI.getType(); 290799a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth if (VecTy && !IsWholeAlloca) { 290899a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth if (NumElements == 1) 290999a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth OtherPtrTy = VecTy->getElementType(); 291099a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth else 291199a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth OtherPtrTy = VectorType::get(VecTy->getElementType(), NumElements); 291299a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth 291399a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth OtherPtrTy = OtherPtrTy->getPointerTo(); 291499a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth } else if (IntTy && !IsWholeAlloca) { 291599a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth OtherPtrTy = SubIntTy->getPointerTo(); 291699a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth } 291799a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth 291899a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Value *SrcPtr = getAdjustedPtr(IRB, TD, OtherPtr, RelOffset, OtherPtrTy, 291999a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth getName("." + OtherPtr->getName())); 2920713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *DstPtr = &NewAI; 2921713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!IsDest) 2922713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth std::swap(SrcPtr, DstPtr); 2923713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2924713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *Src; 292599a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth if (VecTy && !IsWholeAlloca && !IsDest) { 292699a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 292799a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth getName(".load")); 292899a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Src = extractVector(IRB, Src, BeginIndex, EndIndex, getName(".vec")); 2929d2cd73f6a5103f5de221a67410e35e6bee211fa3Chandler Carruth } else if (IntTy && !IsWholeAlloca && !IsDest) { 29302360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 29312360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth getName(".load")); 29322360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Src = convertValue(TD, IRB, Src, IntTy); 29332360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset"); 29342360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth uint64_t Offset = BeginOffset - NewAllocaBeginOffset; 29352360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Src = extractInteger(TD, IRB, Src, SubIntTy, Offset, getName(".extract")); 2936713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } else { 293781b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth Src = IRB.CreateAlignedLoad(SrcPtr, Align, II.isVolatile(), 293881b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth getName(".copyload")); 2939713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2940713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 294199a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth if (VecTy && !IsWholeAlloca && IsDest) { 294299a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 294399a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth getName(".oldload")); 294499a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth Src = insertVector(IRB, Old, Src, BeginIndex, getName(".vec")); 294599a54942ae0fb6fdca03e91b2e492e9738fa4436Chandler Carruth } else if (IntTy && !IsWholeAlloca && IsDest) { 29462360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), 29472360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth getName(".oldload")); 29482360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Old = convertValue(TD, IRB, Old, IntTy); 29492360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset"); 29502360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth uint64_t Offset = BeginOffset - NewAllocaBeginOffset; 29512360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Src = insertInteger(TD, IRB, Old, Src, Offset, getName(".insert")); 29522360b7ad99eedecaae512373e7be49c2143550cfChandler Carruth Src = convertValue(TD, IRB, Src, NewAllocaTy); 2953d2cd73f6a5103f5de221a67410e35e6bee211fa3Chandler Carruth } 2954d2cd73f6a5103f5de221a67410e35e6bee211fa3Chandler Carruth 295581b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth StoreInst *Store = cast<StoreInst>( 295681b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth IRB.CreateAlignedStore(Src, DstPtr, Align, II.isVolatile())); 295781b001a2209be99e42aceed74f411c7cefc4d022Chandler Carruth (void)Store; 2958713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " to: " << *Store << "\n"); 2959713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return !II.isVolatile(); 2960713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2961713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2962713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitIntrinsicInst(IntrinsicInst &II) { 2963713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(II.getIntrinsicID() == Intrinsic::lifetime_start || 2964713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth II.getIntrinsicID() == Intrinsic::lifetime_end); 2965713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " original: " << II << "\n"); 2966713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IRBuilder<> IRB(&II); 2967713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(II.getArgOperand(1) == OldPtr); 2968713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2969713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Record this instruction for deletion. 2970f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth Pass.DeadInsts.insert(&II); 2971713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2972713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ConstantInt *Size 2973713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth = ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()), 2974713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth EndOffset - BeginOffset); 2975713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *Ptr = getAdjustedAllocaPtr(IRB, II.getArgOperand(1)->getType()); 2976713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *New; 2977713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (II.getIntrinsicID() == Intrinsic::lifetime_start) 2978713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New = IRB.CreateLifetimeStart(Ptr, Size); 2979713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth else 2980713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth New = IRB.CreateLifetimeEnd(Ptr, Size); 2981713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2982f1af1feeee0f0ec797410762c006211f9c1e2a0fEdwin Vane (void)New; 2983713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " to: " << *New << "\n"); 2984713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return true; 2985713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 2986713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2987713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitPHINode(PHINode &PN) { 2988713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " original: " << PN << "\n"); 29891e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 2990713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // We would like to compute a new pointer in only one place, but have it be 2991713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // as local as possible to the PHI. To do that, we re-use the location of 2992713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // the old pointer, which necessarily must be in the right position to 2993713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // dominate the PHI. 2994713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IRBuilder<> PtrBuilder(cast<Instruction>(OldPtr)); 2995713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 2996713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *NewPtr = getAdjustedAllocaPtr(PtrBuilder, OldPtr->getType()); 29971e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth // Replace the operands which were using the old pointer. 29982a13242b208b80f2d20cdb3067ccb82edb7d1d0fBenjamin Kramer std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr); 2999713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 30001e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth DEBUG(dbgs() << " to: " << PN << "\n"); 30011e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth deleteIfTriviallyDead(OldPtr); 30021e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth return false; 3003713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3004713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3005713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool visitSelectInst(SelectInst &SI) { 3006713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " original: " << SI << "\n"); 3007713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IRBuilder<> IRB(&SI); 3008713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3009713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Find the operand we need to rewrite here. 3010713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool IsTrueVal = SI.getTrueValue() == OldPtr; 3011713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (IsTrueVal) 3012713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(SI.getFalseValue() != OldPtr && "Pointer is both operands!"); 3013713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth else 3014713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(SI.getFalseValue() == OldPtr && "Pointer isn't an operand!"); 3015713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 30161e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth Value *NewPtr = getAdjustedAllocaPtr(IRB, OldPtr->getType()); 30171e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth SI.setOperand(IsTrueVal ? 1 : 2, NewPtr); 30181e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth DEBUG(dbgs() << " to: " << SI << "\n"); 3019713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth deleteIfTriviallyDead(OldPtr); 30201e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth return false; 3021713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3022713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3023713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth}; 3024713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3025713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3026c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruthnamespace { 3027c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth/// \brief Visitor to rewrite aggregate loads and stores as scalar. 3028c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth/// 3029c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth/// This pass aggressively rewrites all aggregate loads and stores on 3030c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth/// a particular pointer (or any pointer derived from it which we can identify) 3031c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth/// with scalar loads and stores. 3032c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruthclass AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> { 3033c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth // Befriend the base class so it can delegate to private visit methods. 3034c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth friend class llvm::InstVisitor<AggLoadStoreRewriter, bool>; 3035c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 30363574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow const DataLayout &TD; 3037c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3038c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// Queue of pointer uses to analyze and potentially rewrite. 3039c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth SmallVector<Use *, 8> Queue; 3040c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3041c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// Set to prevent us from cycling with phi nodes and loops. 3042c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth SmallPtrSet<User *, 8> Visited; 3043c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3044c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// The current pointer use being rewritten. This is used to dig up the used 3045c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// value (as opposed to the user). 3046c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth Use *U; 3047c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3048c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruthpublic: 30493574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow AggLoadStoreRewriter(const DataLayout &TD) : TD(TD) {} 3050c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3051c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// Rewrite loads and stores through a pointer and all pointers derived from 3052c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// it. 3053c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool rewrite(Instruction &I) { 3054c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth DEBUG(dbgs() << " Rewriting FCA loads and stores...\n"); 3055c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth enqueueUsers(I); 3056c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool Changed = false; 3057c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth while (!Queue.empty()) { 3058c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth U = Queue.pop_back_val(); 3059c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth Changed |= visit(cast<Instruction>(U->getUser())); 3060c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3061c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return Changed; 3062c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3063c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3064c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruthprivate: 3065c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// Enqueue all the users of the given instruction for further processing. 3066c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth /// This uses a set to de-duplicate users. 3067c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth void enqueueUsers(Instruction &I) { 3068c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth for (Value::use_iterator UI = I.use_begin(), UE = I.use_end(); UI != UE; 3069c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth ++UI) 3070c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth if (Visited.insert(*UI)) 3071c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth Queue.push_back(&UI.getUse()); 3072c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3073c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3074c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth // Conservative default is to not rewrite anything. 3075c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool visitInstruction(Instruction &I) { return false; } 3076c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 30776e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// \brief Generic recursive split emission class. 3078371d5d86bd70756d2066692114fec599c33baf92Benjamin Kramer template <typename Derived> 30796e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer class OpSplitter { 30806e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer protected: 30816e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// The builder used to form new instructions. 30826e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer IRBuilder<> IRB; 30836e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// The indices which to be used with insert- or extractvalue to select the 30846e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// appropriate value within the aggregate. 30856e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer SmallVector<unsigned, 4> Indices; 30866e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// The indices to a GEP instruction which will move Ptr to the correct slot 30876e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// within the aggregate. 30886e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer SmallVector<Value *, 4> GEPIndices; 30896e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// The base pointer of the original op, used as a base for GEPing the 30906e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// split operations. 30916e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Value *Ptr; 30926e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer 30936e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// Initialize the splitter with an insertion point, Ptr and start with a 30946e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// single zero GEP index. 30956e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer OpSplitter(Instruction *InsertionPoint, Value *Ptr) 3096371d5d86bd70756d2066692114fec599c33baf92Benjamin Kramer : IRB(InsertionPoint), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr) {} 30976e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer 30986e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer public: 30996e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// \brief Generic recursive split emission routine. 31006e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// 31016e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// This method recursively splits an aggregate op (load or store) into 31026e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// scalar or vector ops. It splits recursively until it hits a single value 31036e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// and emits that single value operation via the template argument. 31046e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// 31056e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// The logic of this routine relies on GEPs and insertvalue and 31066e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// extractvalue all operating with the same fundamental index list, merely 31076e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// formatted differently (GEPs need actual values). 31086e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// 31096e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// \param Ty The type being split recursively into smaller ops. 31106e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// \param Agg The aggregate value being built up or stored, depending on 31116e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// whether this is splitting a load or a store respectively. 31126e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) { 31136e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer if (Ty->isSingleValueType()) 3114371d5d86bd70756d2066692114fec599c33baf92Benjamin Kramer return static_cast<Derived *>(this)->emitFunc(Ty, Agg, Name); 31156e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer 31166e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 31176e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer unsigned OldSize = Indices.size(); 31186e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer (void)OldSize; 31196e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size; 31206e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer ++Idx) { 31216e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer assert(Indices.size() == OldSize && "Did not return to the old size"); 31226e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Indices.push_back(Idx); 31236e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer GEPIndices.push_back(IRB.getInt32(Idx)); 31246e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx)); 31256e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer GEPIndices.pop_back(); 31266e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Indices.pop_back(); 31276e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer } 31286e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer return; 3129c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3130c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 31316e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer if (StructType *STy = dyn_cast<StructType>(Ty)) { 31326e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer unsigned OldSize = Indices.size(); 31336e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer (void)OldSize; 31346e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size; 31356e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer ++Idx) { 31366e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer assert(Indices.size() == OldSize && "Did not return to the old size"); 31376e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Indices.push_back(Idx); 31386e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer GEPIndices.push_back(IRB.getInt32(Idx)); 31396e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx)); 31406e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer GEPIndices.pop_back(); 31416e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Indices.pop_back(); 31426e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer } 31436e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer return; 3144c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3145c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 31466e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer llvm_unreachable("Only arrays and structs are aggregate loadable types"); 31476e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer } 31486e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer }; 3149c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3150371d5d86bd70756d2066692114fec599c33baf92Benjamin Kramer struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> { 31516e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr) 31523b682bdf671e5dd0eb70c0498ba18e5c08874fc9Benjamin Kramer : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr) {} 31536e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer 31546e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// Emit a leaf load of a single value. This is called at the leaves of the 31556e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// recursive emission to actually load values. 3156371d5d86bd70756d2066692114fec599c33baf92Benjamin Kramer void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) { 31576e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer assert(Ty->isSingleValueType()); 31586e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer // Load the single value and insert it using the indices. 315950573b1c274d290a81ab76331104206ea0b09bdeJakub Staszak Value *GEP = IRB.CreateInBoundsGEP(Ptr, GEPIndices, Name + ".gep"); 316050573b1c274d290a81ab76331104206ea0b09bdeJakub Staszak Value *Load = IRB.CreateLoad(GEP, Name + ".load"); 31616e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert"); 31626e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer DEBUG(dbgs() << " to: " << *Load << "\n"); 31636e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer } 31646e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer }; 3165c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3166c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool visitLoadInst(LoadInst &LI) { 3167c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth assert(LI.getPointerOperand() == *U); 3168c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth if (!LI.isSimple() || LI.getType()->isSingleValueType()) 3169c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return false; 3170c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3171c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth // We have an aggregate being loaded, split it apart. 3172c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth DEBUG(dbgs() << " original: " << LI << "\n"); 31736e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer LoadOpSplitter Splitter(&LI, *U); 3174c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth Value *V = UndefValue::get(LI.getType()); 31756e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca"); 3176c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth LI.replaceAllUsesWith(V); 3177c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth LI.eraseFromParent(); 3178c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return true; 3179c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3180c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3181371d5d86bd70756d2066692114fec599c33baf92Benjamin Kramer struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> { 31826e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr) 31833b682bdf671e5dd0eb70c0498ba18e5c08874fc9Benjamin Kramer : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr) {} 31846e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer 31856e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// Emit a leaf store of a single value. This is called at the leaves of the 31866e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer /// recursive emission to actually produce stores. 3187371d5d86bd70756d2066692114fec599c33baf92Benjamin Kramer void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) { 31886e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer assert(Ty->isSingleValueType()); 31896e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer // Extract the single value and store it using the indices. 31906e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Value *Store = IRB.CreateStore( 31916e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer IRB.CreateExtractValue(Agg, Indices, Name + ".extract"), 31926e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer IRB.CreateInBoundsGEP(Ptr, GEPIndices, Name + ".gep")); 31936e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer (void)Store; 31946e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer DEBUG(dbgs() << " to: " << *Store << "\n"); 31956e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer } 31966e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer }; 3197c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3198c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool visitStoreInst(StoreInst &SI) { 3199c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth if (!SI.isSimple() || SI.getPointerOperand() != *U) 3200c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return false; 3201c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth Value *V = SI.getValueOperand(); 3202c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth if (V->getType()->isSingleValueType()) 3203c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return false; 3204c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3205c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth // We have an aggregate being stored, split it apart. 3206c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth DEBUG(dbgs() << " original: " << SI << "\n"); 32076e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer StoreOpSplitter Splitter(&SI, *U); 32086e67b254ae2cf2411cb2d1ae41bf0fbc05079f40Benjamin Kramer Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca"); 3209c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth SI.eraseFromParent(); 3210c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return true; 3211c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3212c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3213c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool visitBitCastInst(BitCastInst &BC) { 3214c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth enqueueUsers(BC); 3215c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return false; 3216c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3217c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3218c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool visitGetElementPtrInst(GetElementPtrInst &GEPI) { 3219c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth enqueueUsers(GEPI); 3220c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return false; 3221c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3222c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3223c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool visitPHINode(PHINode &PN) { 3224c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth enqueueUsers(PN); 3225c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return false; 3226c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3227c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3228c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool visitSelectInst(SelectInst &SI) { 3229c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth enqueueUsers(SI); 3230c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return false; 3231c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth } 3232c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth}; 3233c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth} 3234c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 323507525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth/// \brief Strip aggregate type wrapping. 323607525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth/// 323707525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth/// This removes no-op aggregate types wrapping an underlying type. It will 323807525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth/// strip as many layers of types as it can without changing either the type 323907525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth/// size or the allocated size. 324007525a6be6bce604f3b528c91973ac4e66742266Chandler Carruthstatic Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) { 324107525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth if (Ty->isSingleValueType()) 324207525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth return Ty; 324307525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth 324407525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth uint64_t AllocSize = DL.getTypeAllocSize(Ty); 324507525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth uint64_t TypeSize = DL.getTypeSizeInBits(Ty); 324607525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth 324707525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth Type *InnerTy; 324807525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { 324907525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth InnerTy = ArrTy->getElementType(); 325007525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 325107525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth const StructLayout *SL = DL.getStructLayout(STy); 325207525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth unsigned Index = SL->getElementContainingOffset(0); 325307525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth InnerTy = STy->getElementType(Index); 325407525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth } else { 325507525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth return Ty; 325607525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth } 325707525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth 325807525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth if (AllocSize > DL.getTypeAllocSize(InnerTy) || 325907525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth TypeSize > DL.getTypeSizeInBits(InnerTy)) 326007525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth return Ty; 326107525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth 326207525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth return stripAggregateTypeWrapping(DL, InnerTy); 326307525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth} 326407525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth 3265713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief Try to find a partition of the aggregate type passed in for a given 3266713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// offset and size. 3267713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 3268713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// This recurses through the aggregate type and tries to compute a subtype 3269713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// based on the offset and size. When the offset and size span a sub-section 32706b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// of an array, it will even compute a new array type for that sub-section, 32716b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// and the same for structs. 32726b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// 32736b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// Note that this routine is very strict and tries to find a partition of the 32746b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// type which produces the *exact* right offset and size. It is not forgiving 32756b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// when the size or offset cause either end of type-based partition to be off. 32766b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// Also, this is a best-effort routine. It is reasonable to give up and not 32776b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth/// return a type if necessary. 32783574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmowstatic Type *getTypePartition(const DataLayout &TD, Type *Ty, 3279713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t Offset, uint64_t Size) { 3280713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Offset == 0 && TD.getTypeAllocSize(Ty) == Size) 328107525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth return stripAggregateTypeWrapping(TD, Ty); 3282a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth if (Offset > TD.getTypeAllocSize(Ty) || 3283a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth (TD.getTypeAllocSize(Ty) - Offset) < Size) 3284a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth return 0; 3285713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3286713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (SequentialType *SeqTy = dyn_cast<SequentialType>(Ty)) { 3287713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // We can't partition pointers... 3288713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (SeqTy->isPointerTy()) 3289713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3290713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3291713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *ElementTy = SeqTy->getElementType(); 3292713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t ElementSize = TD.getTypeAllocSize(ElementTy); 3293713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t NumSkippedElements = Offset / ElementSize; 3294713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (ArrayType *ArrTy = dyn_cast<ArrayType>(SeqTy)) 3295713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (NumSkippedElements >= ArrTy->getNumElements()) 3296713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3297713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (VectorType *VecTy = dyn_cast<VectorType>(SeqTy)) 3298713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (NumSkippedElements >= VecTy->getNumElements()) 3299713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3300713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Offset -= NumSkippedElements * ElementSize; 3301713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3302713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // First check if we need to recurse. 3303713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Offset > 0 || Size < ElementSize) { 3304713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Bail if the partition ends in a different array element. 3305713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if ((Offset + Size) > ElementSize) 3306713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3307713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Recurse through the element type trying to peel off offset bytes. 3308713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return getTypePartition(TD, ElementTy, Offset, Size); 3309713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3310713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(Offset == 0); 3311713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3312713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Size == ElementSize) 331307525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth return stripAggregateTypeWrapping(TD, ElementTy); 3314713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(Size > ElementSize); 3315713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t NumElements = Size / ElementSize; 3316713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (NumElements * ElementSize != Size) 3317713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3318713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return ArrayType::get(ElementTy, NumElements); 3319713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3320713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3321713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth StructType *STy = dyn_cast<StructType>(Ty); 3322713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!STy) 3323713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3324713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3325713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const StructLayout *SL = TD.getStructLayout(STy); 33266b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth if (Offset >= SL->getSizeInBytes()) 3327713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3328713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t EndOffset = Offset + Size; 3329713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (EndOffset > SL->getSizeInBytes()) 3330713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3331713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3332713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth unsigned Index = SL->getElementContainingOffset(Offset); 3333713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Offset -= SL->getElementOffset(Index); 3334713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3335713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *ElementTy = STy->getElementType(Index); 3336713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t ElementSize = TD.getTypeAllocSize(ElementTy); 3337713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Offset >= ElementSize) 3338713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; // The offset points into alignment padding. 3339713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3340713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // See if any partition must be contained by the element. 3341713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Offset > 0 || Size < ElementSize) { 3342713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if ((Offset + Size) > ElementSize) 3343713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; 3344713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return getTypePartition(TD, ElementTy, Offset, Size); 3345713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3346713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(Offset == 0); 3347713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3348713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Size == ElementSize) 334907525a6be6bce604f3b528c91973ac4e66742266Chandler Carruth return stripAggregateTypeWrapping(TD, ElementTy); 3350713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3351713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth StructType::element_iterator EI = STy->element_begin() + Index, 3352713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth EE = STy->element_end(); 3353713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (EndOffset < SL->getSizeInBytes()) { 3354713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth unsigned EndIndex = SL->getElementContainingOffset(EndOffset); 3355713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Index == EndIndex) 3356713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return 0; // Within a single element and its padding. 33576b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth 33586b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth // Don't try to form "natural" types if the elements don't line up with the 33596b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth // expected size. 33606b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth // FIXME: We could potentially recurse down through the last element in the 33616b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth // sub-struct to find a natural end point. 33626b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth if (SL->getElementOffset(EndIndex) != EndOffset) 33636b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth return 0; 33646b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth 3365713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(Index < EndIndex); 3366713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth EE = STy->element_begin() + EndIndex; 3367713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3368713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3369713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Try to build up a sub-structure. 33702a13242b208b80f2d20cdb3067ccb82edb7d1d0fBenjamin Kramer StructType *SubTy = StructType::get(STy->getContext(), makeArrayRef(EI, EE), 3371713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth STy->isPacked()); 3372713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const StructLayout *SubSL = TD.getStructLayout(SubTy); 33736b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth if (Size != SubSL->getSizeInBytes()) 33746b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth return 0; // The sub-struct doesn't have quite the size needed. 3375713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 33766b547a2af560fac18f7f7c9b82cd685fee2dde10Chandler Carruth return SubTy; 3377713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3378713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3379713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief Rewrite an alloca partition's users. 3380713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 3381713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// This routine drives both of the rewriting goals of the SROA pass. It tries 3382713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// to rewrite uses of an alloca partition to be conducive for SSA value 3383713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// promotion. If the partition needs a new, more refined alloca, this will 3384713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// build that new alloca, preserving as much type information as possible, and 3385713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// rewrite the uses of the old alloca to point at the new one and have the 3386713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// appropriate new offsets. It also evaluates how successful the rewrite was 3387713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// at enabling promotion and if it was successful queues the alloca to be 3388713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// promoted. 3389713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthbool SROA::rewriteAllocaPartition(AllocaInst &AI, 3390713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning &P, 3391713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning::iterator PI) { 3392713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth uint64_t AllocaSize = PI->EndOffset - PI->BeginOffset; 3393fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth bool IsLive = false; 3394fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth for (AllocaPartitioning::use_iterator UI = P.use_begin(PI), 3395fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth UE = P.use_end(PI); 3396fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth UI != UE && !IsLive; ++UI) 339741b55f5556d1332934cefa7c14862313eb87fa29Chandler Carruth if (UI->getUse()) 3398fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth IsLive = true; 3399fdb15850e6f2d395b128bde0f2fff6b20c96adc6Chandler Carruth if (!IsLive) 3400713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; // No live uses left of this partition. 3401713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 34021e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth DEBUG(dbgs() << "Speculating PHIs and selects in partition " 34031e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth << "[" << PI->BeginOffset << "," << PI->EndOffset << ")\n"); 34041e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 34051e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth PHIOrSelectSpeculator Speculator(*TD, P, *this); 34061e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth DEBUG(dbgs() << " speculating "); 34071e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth DEBUG(P.print(dbgs(), PI, "")); 3408a346f462716b4eb32d27ffe20fc8696473f225f6Chandler Carruth Speculator.visitUsers(PI); 34091e1b16c48515d6d54525ab279e4acb5ddd05c83dChandler Carruth 3410713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Try to compute a friendly type for this partition of the alloca. This 3411713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // won't always succeed, in which case we fall back to a legal integer type 3412713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // or an i8 array of an appropriate size. 3413713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Type *AllocaTy = 0; 3414713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Type *PartitionTy = P.getCommonType(PI)) 3415713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (TD->getTypeAllocSize(PartitionTy) >= AllocaSize) 3416713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaTy = PartitionTy; 3417713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!AllocaTy) 3418713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Type *PartitionTy = getTypePartition(*TD, AI.getAllocatedType(), 3419713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth PI->BeginOffset, AllocaSize)) 3420713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaTy = PartitionTy; 3421713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if ((!AllocaTy || 3422713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth (AllocaTy->isArrayTy() && 3423713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaTy->getArrayElementType()->isIntegerTy())) && 3424713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth TD->isLegalInteger(AllocaSize * 8)) 3425713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaTy = Type::getIntNTy(*C, AllocaSize * 8); 3426713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!AllocaTy) 3427713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaTy = ArrayType::get(Type::getInt8Ty(*C), AllocaSize); 3428b3dd9a13e8454fe069fafc1b259b64851f70e6e7Chandler Carruth assert(TD->getTypeAllocSize(AllocaTy) >= AllocaSize); 3429713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3430713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Check for the case where we're going to rewrite to a new alloca of the 3431713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // exact same type as the original, and with the same access offsets. In that 3432713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // case, re-use the existing alloca, but still run through the rewriter to 3433ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak // perform phi and select speculation. 3434713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaInst *NewAI; 3435713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (AllocaTy == AI.getAllocatedType()) { 3436713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(PI->BeginOffset == 0 && 3437713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth "Non-zero begin offset but same alloca type"); 3438713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth assert(PI == P.begin() && "Begin offset is zero on later partition"); 3439713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth NewAI = &AI; 3440713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } else { 3441b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth unsigned Alignment = AI.getAlignment(); 3442b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth if (!Alignment) { 3443b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth // The minimum alignment which users can rely on when the explicit 3444b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth // alignment is omitted or zero is that required by the ABI for this 3445b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth // type. 3446b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth Alignment = TD->getABITypeAlignment(AI.getAllocatedType()); 3447b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth } 3448b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth Alignment = MinAlign(Alignment, PI->BeginOffset); 3449b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth // If we will get at least this much alignment from the type alone, leave 3450b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth // the alloca's alignment unconstrained. 3451b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth if (Alignment <= TD->getABITypeAlignment(AllocaTy)) 3452b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth Alignment = 0; 3453b67c9a5b0295faaab40c6c64c433ab679d225a70Chandler Carruth NewAI = new AllocaInst(AllocaTy, 0, Alignment, 3454713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AI.getName() + ".sroa." + Twine(PI - P.begin()), 3455713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth &AI); 3456713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++NumNewAllocas; 3457713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3458713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3459713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << "Rewriting alloca partition " 3460713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << "[" << PI->BeginOffset << "," << PI->EndOffset << ") to: " 3461713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth << *NewAI << "\n"); 3462713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3463b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth // Track the high watermark of the post-promotion worklist. We will reset it 3464b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth // to this point if the alloca is not in fact scheduled for promotion. 3465b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth unsigned PPWOldSize = PostPromotionWorklist.size(); 3466b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth 3467713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitionRewriter Rewriter(*TD, P, PI, *this, AI, *NewAI, 3468713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth PI->BeginOffset, PI->EndOffset); 3469713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " rewriting "); 3470713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(P.print(dbgs(), PI, "")); 3471b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth bool Promotable = Rewriter.visitUsers(P.use_begin(PI), P.use_end(PI)); 3472b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth if (Promotable) { 3473713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " and queuing for promotion\n"); 3474713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth PromotableAllocas.push_back(NewAI); 3475713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } else if (NewAI != &AI) { 3476713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // If we can't promote the alloca, iterate on it to check for new 3477713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // refinements exposed by splitting the current alloca. Don't iterate on an 3478713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // alloca which didn't actually change and didn't get promoted. 3479713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Worklist.insert(NewAI); 3480713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3481b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth 3482b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth // Drop any post-promotion work items if promotion didn't happen. 3483b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth if (!Promotable) 3484b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth while (PostPromotionWorklist.size() > PPWOldSize) 3485b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth PostPromotionWorklist.pop_back(); 3486b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth 3487713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return true; 3488713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3489713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3490713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief Walks the partitioning of an alloca rewriting uses of each partition. 3491713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthbool SROA::splitAlloca(AllocaInst &AI, AllocaPartitioning &P) { 3492713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool Changed = false; 3493713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (AllocaPartitioning::iterator PI = P.begin(), PE = P.end(); PI != PE; 3494713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++PI) 3495713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Changed |= rewriteAllocaPartition(AI, P, PI); 3496713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3497713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return Changed; 3498713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3499713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3500713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// \brief Analyze an alloca for SROA. 3501713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// 3502713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// This analyzes the alloca to ensure we can reason about it, builds 3503713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// a partitioning of the alloca, and then hands it off to be split and 3504713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth/// rewritten as needed. 3505713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthbool SROA::runOnAlloca(AllocaInst &AI) { 3506713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << "SROA alloca: " << AI << "\n"); 3507713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++NumAllocasAnalyzed; 3508713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3509713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Special case dead allocas, as they're trivial. 3510713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (AI.use_empty()) { 3511713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AI.eraseFromParent(); 3512713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return true; 3513713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3514713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3515713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Skip alloca forms that this analysis can't handle. 3516713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (AI.isArrayAllocation() || !AI.getAllocatedType()->isSized() || 3517713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth TD->getTypeAllocSize(AI.getAllocatedType()) == 0) 3518713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; 3519713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3520c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth bool Changed = false; 3521c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3522c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth // First, split any FCA loads and stores touching this alloca to promote 3523c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth // better splitting and promotion opportunities. 3524c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth AggLoadStoreRewriter AggRewriter(*TD); 3525c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth Changed |= AggRewriter.rewrite(AI); 3526c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth 3527713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Build the partition set using a recursive instruction-visiting builder. 3528713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AllocaPartitioning P(*TD, AI); 3529713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(P.print(dbgs())); 3530713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (P.isEscaped()) 3531c370acdf9698b7eee11f3d8e3732f1d72cd25943Chandler Carruth return Changed; 3532713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3533713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Delete all the dead users of this alloca before splitting and rewriting it. 3534713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (AllocaPartitioning::dead_user_iterator DI = P.dead_user_begin(), 3535713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DE = P.dead_user_end(); 3536713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DI != DE; ++DI) { 3537713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Changed = true; 3538713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth (*DI)->replaceAllUsesWith(UndefValue::get((*DI)->getType())); 3539f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DeadInsts.insert(*DI); 3540713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3541713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (AllocaPartitioning::dead_op_iterator DO = P.dead_op_begin(), 3542713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DE = P.dead_op_end(); 3543713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DO != DE; ++DO) { 3544713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Value *OldV = **DO; 3545713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Clobber the use with an undef value. 3546713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth **DO = UndefValue::get(OldV->getType()); 3547713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Instruction *OldI = dyn_cast<Instruction>(OldV)) 3548713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (isInstructionTriviallyDead(OldI)) { 3549713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Changed = true; 3550f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DeadInsts.insert(OldI); 3551713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3552713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3553713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3554fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth // No partitions to split. Leave the dead alloca for a later pass to clean up. 3555fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth if (P.begin() == P.end()) 3556fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth return Changed; 3557fca3f4021ae9a561edb6d2fcb4a282b6f25ab144Chandler Carruth 3558713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return splitAlloca(AI, P) || Changed; 3559713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3560713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 35618615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// \brief Delete the dead instructions accumulated in this run. 35628615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// 35638615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// Recursively deletes the dead instructions we've accumulated. This is done 35648615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// at the very end to maximize locality of the recursive delete and to 35658615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// minimize the problems of invalidated instruction pointers as such pointers 35668615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// are used heavily in the intermediate stages of the algorithm. 35678615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// 35688615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// We also record the alloca instructions deleted here so that they aren't 35698615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth/// subsequently handed to mem2reg to promote. 35708615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruthvoid SROA::deleteDeadInstructions(SmallPtrSet<AllocaInst*, 4> &DeletedAllocas) { 3571713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth while (!DeadInsts.empty()) { 3572713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Instruction *I = DeadInsts.pop_back_val(); 3573713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n"); 3574713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3575a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth I->replaceAllUsesWith(UndefValue::get(I->getType())); 3576a2b88163af30f59d12ae0172565f3406bdbf6c45Chandler Carruth 3577713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI) 3578713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (Instruction *U = dyn_cast<Instruction>(*OI)) { 3579713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth // Zero out the operand and see if it becomes trivially dead. 3580713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth *OI = 0; 3581713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (isInstructionTriviallyDead(U)) 3582f5837aacd4b04e551fb194f4095006b7e6f2b991Chandler Carruth DeadInsts.insert(U); 3583713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3584713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3585713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) 3586713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DeletedAllocas.insert(AI); 3587713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3588713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth ++NumDeleted; 3589713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth I->eraseFromParent(); 3590713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3591713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3592713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 35931c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// \brief Promote the allocas, using the best available technique. 35941c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// 35951c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// This attempts to promote whatever allocas have been identified as viable in 35961c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// the PromotableAllocas list. If that list is empty, there is nothing to do. 35971c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// If there is a domtree available, we attempt to promote using the full power 35981c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// of mem2reg. Otherwise, we build and use the AllocaPromoter above which is 35991c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth/// based on the SSAUpdater utilities. This function returns whether any 3600ba6f722d6a80efeacb69c12f9322d858affb4d2bJakub Staszak/// promotion occurred. 36011c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruthbool SROA::promoteAllocas(Function &F) { 36021c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (PromotableAllocas.empty()) 36031c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth return false; 36041c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 36051c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth NumPromoted += PromotableAllocas.size(); 36061c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 36071c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (DT && !ForceSSAUpdater) { 36081c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DEBUG(dbgs() << "Promoting allocas with mem2reg...\n"); 36091c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth PromoteMemToReg(PromotableAllocas, *DT); 36101c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth PromotableAllocas.clear(); 36111c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth return true; 36121c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 36131c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 36141c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DEBUG(dbgs() << "Promoting allocas with SSAUpdater...\n"); 36151c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth SSAUpdater SSA; 36161c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DIBuilder DIB(*F.getParent()); 36171c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth SmallVector<Instruction*, 64> Insts; 36181c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 36191c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth for (unsigned Idx = 0, Size = PromotableAllocas.size(); Idx != Size; ++Idx) { 36201c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth AllocaInst *AI = PromotableAllocas[Idx]; 36211c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth for (Value::use_iterator UI = AI->use_begin(), UE = AI->use_end(); 36221c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth UI != UE;) { 36231c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Instruction *I = cast<Instruction>(*UI++); 36241c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth // FIXME: Currently the SSAUpdater infrastructure doesn't reason about 36251c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth // lifetime intrinsics and so we strip them (and the bitcasts+GEPs 36261c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth // leading to them) here. Eventually it should use them to optimize the 36271c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth // scalar values produced. 36281c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (isa<BitCastInst>(I) || isa<GetElementPtrInst>(I)) { 36291c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth assert(onlyUsedByLifetimeMarkers(I) && 36301c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth "Found a bitcast used outside of a lifetime marker."); 36311c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth while (!I->use_empty()) 36321c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth cast<Instruction>(*I->use_begin())->eraseFromParent(); 36331c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth I->eraseFromParent(); 36341c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth continue; 36351c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 36361c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { 36371c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth assert(II->getIntrinsicID() == Intrinsic::lifetime_start || 36381c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth II->getIntrinsicID() == Intrinsic::lifetime_end); 36391c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth II->eraseFromParent(); 36401c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth continue; 36411c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 36421c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 36431c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Insts.push_back(I); 36441c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 36451c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth AllocaPromoter(Insts, SSA, *AI, DIB).run(Insts); 36461c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth Insts.clear(); 36471c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth } 36481c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 36491c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth PromotableAllocas.clear(); 36501c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth return true; 36511c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth} 36521c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth 3653713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthnamespace { 3654713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth /// \brief A predicate to test whether an alloca belongs to a set. 3655713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth class IsAllocaInSet { 3656713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth typedef SmallPtrSet<AllocaInst *, 4> SetType; 3657713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth const SetType &Set; 3658713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3659713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth public: 366075eac5f0ebff4d0ffe10ce6bc8f2867c5f15315bChandler Carruth typedef AllocaInst *argument_type; 366175eac5f0ebff4d0ffe10ce6bc8f2867c5f15315bChandler Carruth 3662713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth IsAllocaInSet(const SetType &Set) : Set(Set) {} 366375eac5f0ebff4d0ffe10ce6bc8f2867c5f15315bChandler Carruth bool operator()(AllocaInst *AI) const { return Set.count(AI); } 3664713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth }; 3665713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3666713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3667713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthbool SROA::runOnFunction(Function &F) { 3668713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << "SROA function: " << F.getName() << "\n"); 3669713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth C = &F.getContext(); 36703574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow TD = getAnalysisIfAvailable<DataLayout>(); 3671713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (!TD) { 3672713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth DEBUG(dbgs() << " Skipping SROA -- no target data!\n"); 3673713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return false; 3674713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 36751c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth DT = getAnalysisIfAvailable<DominatorTree>(); 3676713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3677713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth BasicBlock &EntryBB = F.getEntryBlock(); 3678713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth for (BasicBlock::iterator I = EntryBB.begin(), E = llvm::prior(EntryBB.end()); 3679713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth I != E; ++I) 3680713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) 3681713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth Worklist.insert(AI); 3682713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3683713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth bool Changed = false; 36848615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth // A set of deleted alloca instruction pointers which should be removed from 36858615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth // the list of promotable allocas. 36868615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth SmallPtrSet<AllocaInst *, 4> DeletedAllocas; 36878615cd236ed9f9540b4a3ad05dd0a1d202a3f6b4Chandler Carruth 3688b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth do { 3689b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth while (!Worklist.empty()) { 3690b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth Changed |= runOnAlloca(*Worklist.pop_back_val()); 3691b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth deleteDeadInstructions(DeletedAllocas); 3692b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth 3693b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth // Remove the deleted allocas from various lists so that we don't try to 3694b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth // continue processing them. 3695b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth if (!DeletedAllocas.empty()) { 3696b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth Worklist.remove_if(IsAllocaInSet(DeletedAllocas)); 3697b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth PostPromotionWorklist.remove_if(IsAllocaInSet(DeletedAllocas)); 3698b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth PromotableAllocas.erase(std::remove_if(PromotableAllocas.begin(), 3699b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth PromotableAllocas.end(), 3700b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth IsAllocaInSet(DeletedAllocas)), 3701b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth PromotableAllocas.end()); 3702b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth DeletedAllocas.clear(); 3703b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth } 3704713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth } 3705713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3706b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth Changed |= promoteAllocas(F); 3707b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth 3708b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth Worklist = PostPromotionWorklist; 3709b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth PostPromotionWorklist.clear(); 3710b2d98c29170d99fb43de0244a7f4f93a8893c208Chandler Carruth } while (!Worklist.empty()); 3711713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3712713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth return Changed; 3713713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3714713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth 3715713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruthvoid SROA::getAnalysisUsage(AnalysisUsage &AU) const { 37161c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth if (RequiresDomTree) 37171c8db50a9aca4229ebee75ed9cfbf8b222292787Chandler Carruth AU.addRequired<DominatorTree>(); 3718713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth AU.setPreservesCFG(); 3719713aa9431d84805c5b7ddce57159d14ee94cd548Chandler Carruth} 3720