1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "CGLoopInfo.h"
11#include "clang/AST/ASTContext.h"
12#include "clang/AST/Attr.h"
13#include "clang/Sema/LoopHint.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/InstrTypes.h"
17#include "llvm/IR/Instructions.h"
18#include "llvm/IR/Metadata.h"
19using namespace clang::CodeGen;
20using namespace llvm;
21
22static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
23                              llvm::DebugLoc Location) {
24
25  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
26      Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
27      Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
28      Attrs.UnrollEnable == LoopAttributes::Unspecified &&
29      Attrs.DistributeEnable == LoopAttributes::Unspecified &&
30      !Location)
31    return nullptr;
32
33  SmallVector<Metadata *, 4> Args;
34  // Reserve operand 0 for loop id self reference.
35  auto TempNode = MDNode::getTemporary(Ctx, None);
36  Args.push_back(TempNode.get());
37
38  // If we have a valid debug location for the loop, add it.
39  if (Location)
40    Args.push_back(Location.getAsMDNode());
41
42  // Setting vectorize.width
43  if (Attrs.VectorizeWidth > 0) {
44    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
45                        ConstantAsMetadata::get(ConstantInt::get(
46                            Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
47    Args.push_back(MDNode::get(Ctx, Vals));
48  }
49
50  // Setting interleave.count
51  if (Attrs.InterleaveCount > 0) {
52    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
53                        ConstantAsMetadata::get(ConstantInt::get(
54                            Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
55    Args.push_back(MDNode::get(Ctx, Vals));
56  }
57
58  // Setting interleave.count
59  if (Attrs.UnrollCount > 0) {
60    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
61                        ConstantAsMetadata::get(ConstantInt::get(
62                            Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
63    Args.push_back(MDNode::get(Ctx, Vals));
64  }
65
66  // Setting vectorize.enable
67  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
68    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
69                        ConstantAsMetadata::get(ConstantInt::get(
70                            Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
71                                                   LoopAttributes::Enable)))};
72    Args.push_back(MDNode::get(Ctx, Vals));
73  }
74
75  // Setting unroll.full or unroll.disable
76  if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
77    std::string Name;
78    if (Attrs.UnrollEnable == LoopAttributes::Enable)
79      Name = "llvm.loop.unroll.enable";
80    else if (Attrs.UnrollEnable == LoopAttributes::Full)
81      Name = "llvm.loop.unroll.full";
82    else
83      Name = "llvm.loop.unroll.disable";
84    Metadata *Vals[] = {MDString::get(Ctx, Name)};
85    Args.push_back(MDNode::get(Ctx, Vals));
86  }
87
88  if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
89    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
90                        ConstantAsMetadata::get(ConstantInt::get(
91                            Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
92                                                   LoopAttributes::Enable)))};
93    Args.push_back(MDNode::get(Ctx, Vals));
94  }
95
96  // Set the first operand to itself.
97  MDNode *LoopID = MDNode::get(Ctx, Args);
98  LoopID->replaceOperandWith(0, LoopID);
99  return LoopID;
100}
101
102LoopAttributes::LoopAttributes(bool IsParallel)
103    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
104      UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
105      InterleaveCount(0), UnrollCount(0),
106      DistributeEnable(LoopAttributes::Unspecified) {}
107
108void LoopAttributes::clear() {
109  IsParallel = false;
110  VectorizeWidth = 0;
111  InterleaveCount = 0;
112  UnrollCount = 0;
113  VectorizeEnable = LoopAttributes::Unspecified;
114  UnrollEnable = LoopAttributes::Unspecified;
115}
116
117LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
118                   llvm::DebugLoc Location)
119    : LoopID(nullptr), Header(Header), Attrs(Attrs) {
120  LoopID = createMetadata(Header->getContext(), Attrs, Location);
121}
122
123void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) {
124  Active.push_back(LoopInfo(Header, StagedAttrs, Location));
125  // Clear the attributes so nested loops do not inherit them.
126  StagedAttrs.clear();
127}
128
129void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
130                         ArrayRef<const clang::Attr *> Attrs,
131                         llvm::DebugLoc Location) {
132
133  // Identify loop hint attributes from Attrs.
134  for (const auto *Attr : Attrs) {
135    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
136    const OpenCLUnrollHintAttr *OpenCLHint =
137        dyn_cast<OpenCLUnrollHintAttr>(Attr);
138
139    // Skip non loop hint attributes
140    if (!LH && !OpenCLHint) {
141      continue;
142    }
143
144    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
145    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
146    unsigned ValueInt = 1;
147    // Translate opencl_unroll_hint attribute argument to
148    // equivalent LoopHintAttr enums.
149    // OpenCL v2.0 s6.11.5:
150    // 0 - full unroll (no argument).
151    // 1 - disable unroll.
152    // other positive integer n - unroll by n.
153    if (OpenCLHint) {
154      ValueInt = OpenCLHint->getUnrollHint();
155      if (ValueInt == 0) {
156        State = LoopHintAttr::Full;
157      } else if (ValueInt != 1) {
158        Option = LoopHintAttr::UnrollCount;
159        State = LoopHintAttr::Numeric;
160      }
161    } else if (LH) {
162      auto *ValueExpr = LH->getValue();
163      if (ValueExpr) {
164        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
165        ValueInt = ValueAPS.getSExtValue();
166      }
167
168      Option = LH->getOption();
169      State = LH->getState();
170    }
171    switch (State) {
172    case LoopHintAttr::Disable:
173      switch (Option) {
174      case LoopHintAttr::Vectorize:
175        // Disable vectorization by specifying a width of 1.
176        setVectorizeWidth(1);
177        break;
178      case LoopHintAttr::Interleave:
179        // Disable interleaving by speciyfing a count of 1.
180        setInterleaveCount(1);
181        break;
182      case LoopHintAttr::Unroll:
183        setUnrollState(LoopAttributes::Disable);
184        break;
185      case LoopHintAttr::Distribute:
186        setDistributeState(false);
187        break;
188      case LoopHintAttr::UnrollCount:
189      case LoopHintAttr::VectorizeWidth:
190      case LoopHintAttr::InterleaveCount:
191        llvm_unreachable("Options cannot be disabled.");
192        break;
193      }
194      break;
195    case LoopHintAttr::Enable:
196      switch (Option) {
197      case LoopHintAttr::Vectorize:
198      case LoopHintAttr::Interleave:
199        setVectorizeEnable(true);
200        break;
201      case LoopHintAttr::Unroll:
202        setUnrollState(LoopAttributes::Enable);
203        break;
204      case LoopHintAttr::Distribute:
205        setDistributeState(true);
206        break;
207      case LoopHintAttr::UnrollCount:
208      case LoopHintAttr::VectorizeWidth:
209      case LoopHintAttr::InterleaveCount:
210        llvm_unreachable("Options cannot enabled.");
211        break;
212      }
213      break;
214    case LoopHintAttr::AssumeSafety:
215      switch (Option) {
216      case LoopHintAttr::Vectorize:
217      case LoopHintAttr::Interleave:
218        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
219        setParallel(true);
220        setVectorizeEnable(true);
221        break;
222      case LoopHintAttr::Unroll:
223      case LoopHintAttr::UnrollCount:
224      case LoopHintAttr::VectorizeWidth:
225      case LoopHintAttr::InterleaveCount:
226      case LoopHintAttr::Distribute:
227        llvm_unreachable("Options cannot be used to assume mem safety.");
228        break;
229      }
230      break;
231    case LoopHintAttr::Full:
232      switch (Option) {
233      case LoopHintAttr::Unroll:
234        setUnrollState(LoopAttributes::Full);
235        break;
236      case LoopHintAttr::Vectorize:
237      case LoopHintAttr::Interleave:
238      case LoopHintAttr::UnrollCount:
239      case LoopHintAttr::VectorizeWidth:
240      case LoopHintAttr::InterleaveCount:
241      case LoopHintAttr::Distribute:
242        llvm_unreachable("Options cannot be used with 'full' hint.");
243        break;
244      }
245      break;
246    case LoopHintAttr::Numeric:
247      switch (Option) {
248      case LoopHintAttr::VectorizeWidth:
249        setVectorizeWidth(ValueInt);
250        break;
251      case LoopHintAttr::InterleaveCount:
252        setInterleaveCount(ValueInt);
253        break;
254      case LoopHintAttr::UnrollCount:
255        setUnrollCount(ValueInt);
256        break;
257      case LoopHintAttr::Unroll:
258      case LoopHintAttr::Vectorize:
259      case LoopHintAttr::Interleave:
260      case LoopHintAttr::Distribute:
261        llvm_unreachable("Options cannot be assigned a value.");
262        break;
263      }
264      break;
265    }
266  }
267
268  /// Stage the attributes.
269  push(Header, Location);
270}
271
272void LoopInfoStack::pop() {
273  assert(!Active.empty() && "No active loops to pop");
274  Active.pop_back();
275}
276
277void LoopInfoStack::InsertHelper(Instruction *I) const {
278  if (!hasInfo())
279    return;
280
281  const LoopInfo &L = getInfo();
282  if (!L.getLoopID())
283    return;
284
285  if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) {
286    for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i)
287      if (TI->getSuccessor(i) == L.getHeader()) {
288        TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
289        break;
290      }
291    return;
292  }
293
294  if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory())
295    I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID());
296}
297