RSForEachExpand.cpp revision 806075b3a54af826fea78490fb213d8a0784138e 
1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "bcc/Assert.h"
18#include "bcc/Renderscript/RSTransforms.h"
19
20#include <cstdlib>
21
22#include <llvm/IR/DerivedTypes.h>
23#include <llvm/IR/Function.h>
24#include <llvm/IR/Instructions.h>
25#include <llvm/IR/IRBuilder.h>
26#include <llvm/IR/Module.h>
27#include <llvm/Pass.h>
28#include <llvm/Support/raw_ostream.h>
29#include <llvm/IR/DataLayout.h>
30#include <llvm/IR/Type.h>
31#include <llvm/Transforms/Utils/BasicBlockUtils.h>
32
33#include "bcc/Config/Config.h"
34#include "bcc/Renderscript/RSInfo.h"
35#include "bcc/Support/Log.h"
36
37using namespace bcc;
38
39namespace {
40
41/* RSForEachExpandPass - This pass operates on functions that are able to be
42 * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the
43 * ForEach-able function to be invoked over the appropriate data cells of the
44 * input/output allocations (adjusting other relevant parameters as we go). We
45 * support doing this for any ForEach-able compute kernels. The new function
46 * name is the original function name followed by ".expand". Note that we
47 * still generate code for the original function.
48 */
49class RSForEachExpandPass : public llvm::ModulePass {
50private:
51  static char ID;
52
53  llvm::Module *M;
54  llvm::LLVMContext *C;
55
56  const RSInfo::ExportForeachFuncListTy &mFuncs;
57
58  // Turns on optimization of allocation stride values.
59  bool mEnableStepOpt;
60
61  uint32_t getRootSignature(llvm::Function *F) {
62    const llvm::NamedMDNode *ExportForEachMetadata =
63        M->getNamedMetadata("#rs_export_foreach");
64
65    if (!ExportForEachMetadata) {
66      llvm::SmallVector<llvm::Type*, 8> RootArgTys;
67      for (llvm::Function::arg_iterator B = F->arg_begin(),
68                                        E = F->arg_end();
69           B != E;
70           ++B) {
71        RootArgTys.push_back(B->getType());
72      }
73
74      // For pre-ICS bitcode, we may not have signature information. In that
75      // case, we use the size of the RootArgTys to select the number of
76      // arguments.
77      return (1 << RootArgTys.size()) - 1;
78    }
79
80    if (ExportForEachMetadata->getNumOperands() == 0) {
81      return 0;
82    }
83
84    bccAssert(ExportForEachMetadata->getNumOperands() > 0);
85
86    // We only handle the case for legacy root() functions here, so this is
87    // hard-coded to look at only the first such function.
88    llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0);
89    if (SigNode != NULL && SigNode->getNumOperands() == 1) {
90      llvm::Value *SigVal = SigNode->getOperand(0);
91      if (SigVal->getValueID() == llvm::Value::MDStringVal) {
92        llvm::StringRef SigString =
93            static_cast<llvm::MDString*>(SigVal)->getString();
94        uint32_t Signature = 0;
95        if (SigString.getAsInteger(10, Signature)) {
96          ALOGE("Non-integer signature value '%s'", SigString.str().c_str());
97          return 0;
98        }
99        return Signature;
100      }
101    }
102
103    return 0;
104  }
105
106  // Get the actual value we should use to step through an allocation.
107  // DL - Target Data size/layout information.
108  // T - Type of allocation (should be a pointer).
109  // OrigStep - Original step increment (root.expand() input from driver).
110  llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *T,
111                            llvm::Value *OrigStep) {
112    bccAssert(DL);
113    bccAssert(T);
114    bccAssert(OrigStep);
115    llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(T);
116    llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C);
117    if (mEnableStepOpt && T != VoidPtrTy && PT) {
118      llvm::Type *ET = PT->getElementType();
119      uint64_t ETSize = DL->getTypeAllocSize(ET);
120      llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C);
121      return llvm::ConstantInt::get(Int32Ty, ETSize);
122    } else {
123      return OrigStep;
124    }
125  }
126
127  static bool hasIn(uint32_t Signature) {
128    return Signature & 0x01;
129  }
130
131  static bool hasOut(uint32_t Signature) {
132    return Signature & 0x02;
133  }
134
135  static bool hasUsrData(uint32_t Signature) {
136    return Signature & 0x04;
137  }
138
139  static bool hasX(uint32_t Signature) {
140    return Signature & 0x08;
141  }
142
143  static bool hasY(uint32_t Signature) {
144    return Signature & 0x10;
145  }
146
147  static bool isKernel(uint32_t Signature) {
148    return Signature & 0x20;
149  }
150
151  /// @brief Returns the type of the ForEach stub parameter structure.
152  ///
153  /// Renderscript uses a single structure in which all parameters are passed
154  /// to keep the signature of the expanded function independent of the
155  /// parameters passed to it.
156  llvm::Type *getForeachStubTy() {
157    llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C);
158    llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C);
159    llvm::Type *SizeTy = Int32Ty;
160    /* Defined in frameworks/base/libs/rs/rs_hal.h:
161     *
162     * struct RsForEachStubParamStruct {
163     *   const void *in;
164     *   void *out;
165     *   const void *usr;
166     *   size_t usr_len;
167     *   uint32_t x;
168     *   uint32_t y;
169     *   uint32_t z;
170     *   uint32_t lod;
171     *   enum RsAllocationCubemapFace face;
172     *   uint32_t ar[16];
173     * };
174     */
175    llvm::SmallVector<llvm::Type*, 9> StructTys;
176    StructTys.push_back(VoidPtrTy);  // const void *in
177    StructTys.push_back(VoidPtrTy);  // void *out
178    StructTys.push_back(VoidPtrTy);  // const void *usr
179    StructTys.push_back(SizeTy);     // size_t usr_len
180    StructTys.push_back(Int32Ty);    // uint32_t x
181    StructTys.push_back(Int32Ty);    // uint32_t y
182    StructTys.push_back(Int32Ty);    // uint32_t z
183    StructTys.push_back(Int32Ty);    // uint32_t lod
184    StructTys.push_back(Int32Ty);    // enum RsAllocationCubemapFace
185    StructTys.push_back(llvm::ArrayType::get(Int32Ty, 16));  // uint32_t ar[16]
186
187    return llvm::StructType::create(StructTys, "RsForEachStubParamStruct");
188  }
189
190  /// @brief Create skeleton of the expanded function.
191  ///
192  /// This creates a function with the following signature:
193  ///
194  ///   void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2,
195  ///         uint32_t instep, uint32_t outstep)
196  ///
197  llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) {
198    llvm::Type *ForEachStubPtrTy = getForeachStubTy()->getPointerTo();
199    llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C);
200
201    llvm::SmallVector<llvm::Type*, 8> ParamTys;
202    ParamTys.push_back(ForEachStubPtrTy);  // const RsForEachStubParamStruct *p
203    ParamTys.push_back(Int32Ty);           // uint32_t x1
204    ParamTys.push_back(Int32Ty);           // uint32_t x2
205    ParamTys.push_back(Int32Ty);           // uint32_t instep
206    ParamTys.push_back(Int32Ty);           // uint32_t outstep
207
208    llvm::FunctionType *FT =
209        llvm::FunctionType::get(llvm::Type::getVoidTy(*C), ParamTys, false);
210    llvm::Function *F =
211        llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage,
212                               OldName + ".expand", M);
213
214    llvm::Function::arg_iterator AI = F->arg_begin();
215
216    AI->setName("p");
217    AI++;
218    AI->setName("x1");
219    AI++;
220    AI->setName("x2");
221    AI++;
222    AI->setName("arg_instep");
223    AI++;
224    AI->setName("arg_outstep");
225    AI++;
226
227    assert(AI == F->arg_end());
228
229    llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*C, "Begin", F);
230    llvm::IRBuilder<> Builder(Begin);
231    Builder.CreateRetVoid();
232
233    return F;
234  }
235
236public:
237  RSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs,
238                      bool pEnableStepOpt)
239      : ModulePass(ID), M(NULL), C(NULL), mFuncs(pForeachFuncs),
240        mEnableStepOpt(pEnableStepOpt) {
241  }
242
243  /* Performs the actual optimization on a selected function. On success, the
244   * Module will contain a new function of the name "<NAME>.expand" that
245   * invokes <NAME>() in a loop with the appropriate parameters.
246   */
247  bool ExpandFunction(llvm::Function *F, uint32_t Signature) {
248    ALOGV("Expanding ForEach-able Function %s", F->getName().str().c_str());
249
250    if (!Signature) {
251      Signature = getRootSignature(F);
252      if (!Signature) {
253        // We couldn't determine how to expand this function based on its
254        // function signature.
255        return false;
256      }
257    }
258
259    llvm::DataLayout DL(M);
260
261    llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C);
262    llvm::Function *ExpandedFunc = createEmptyExpandedFunction(F->getName());
263
264    // Create and name the actual arguments to this expanded function.
265    llvm::SmallVector<llvm::Argument*, 8> ArgVec;
266    for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(),
267                                      E = ExpandedFunc->arg_end();
268         B != E;
269         ++B) {
270      ArgVec.push_back(B);
271    }
272
273    if (ArgVec.size() != 5) {
274      ALOGE("Incorrect number of arguments to function: %zu",
275            ArgVec.size());
276      return false;
277    }
278    llvm::Value *Arg_p = ArgVec[0];
279    llvm::Value *Arg_x1 = ArgVec[1];
280    llvm::Value *Arg_x2 = ArgVec[2];
281    llvm::Value *Arg_instep = ArgVec[3];
282    llvm::Value *Arg_outstep = ArgVec[4];
283
284    llvm::Value *InStep = NULL;
285    llvm::Value *OutStep = NULL;
286
287    // Construct the actual function body.
288    llvm::BasicBlock *Begin = &ExpandedFunc->getEntryBlock();
289    llvm::IRBuilder<> Builder(Begin->begin());
290
291    // uint32_t X = x1;
292    llvm::AllocaInst *AX = Builder.CreateAlloca(Int32Ty, 0, "AX");
293    Builder.CreateStore(Arg_x1, AX);
294
295    // Collect and construct the arguments for the kernel().
296    // Note that we load any loop-invariant arguments before entering the Loop.
297    llvm::Function::arg_iterator Args = F->arg_begin();
298
299    llvm::Type *InTy = NULL;
300    llvm::AllocaInst *AIn = NULL;
301    if (hasIn(Signature)) {
302      InTy = Args->getType();
303      AIn = Builder.CreateAlloca(InTy, 0, "AIn");
304      InStep = getStepValue(&DL, InTy, Arg_instep);
305      InStep->setName("instep");
306      Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad(
307          Builder.CreateStructGEP(Arg_p, 0)), InTy), AIn);
308      Args++;
309    }
310
311    llvm::Type *OutTy = NULL;
312    llvm::AllocaInst *AOut = NULL;
313    if (hasOut(Signature)) {
314      OutTy = Args->getType();
315      AOut = Builder.CreateAlloca(OutTy, 0, "AOut");
316      OutStep = getStepValue(&DL, OutTy, Arg_outstep);
317      OutStep->setName("outstep");
318      Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad(
319          Builder.CreateStructGEP(Arg_p, 1)), OutTy), AOut);
320      Args++;
321    }
322
323    llvm::Value *UsrData = NULL;
324    if (hasUsrData(Signature)) {
325      llvm::Type *UsrDataTy = Args->getType();
326      UsrData = Builder.CreatePointerCast(Builder.CreateLoad(
327          Builder.CreateStructGEP(Arg_p, 2)), UsrDataTy);
328      UsrData->setName("UsrData");
329      Args++;
330    }
331
332    if (hasX(Signature)) {
333      Args++;
334    }
335
336    llvm::Value *Y = NULL;
337    if (hasY(Signature)) {
338      Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y");
339      Args++;
340    }
341
342    bccAssert(Args == F->arg_end());
343
344    llvm::BasicBlock *Loop = llvm::BasicBlock::Create(*C, "Loop", ExpandedFunc);
345
346    // if (x1 < x2) goto Loop; else goto Exit;
347    llvm::Value *Cond = Builder.CreateICmpSLT(Arg_x1, Arg_x2);
348
349    llvm::BasicBlock *Exit = llvm::SplitBlock(Builder.GetInsertBlock(),
350                                              Builder.GetInsertPoint(), this);
351    Exit->setName("Exit");
352    Begin->getTerminator()->eraseFromParent();
353    Builder.SetInsertPoint(Begin);
354    Builder.CreateCondBr(Cond, Loop, Exit);
355
356    // Loop:
357    Builder.SetInsertPoint(Loop);
358
359    // Populate the actual call to kernel().
360    llvm::SmallVector<llvm::Value*, 8> RootArgs;
361
362    llvm::Value *InPtr = NULL;
363    llvm::Value *OutPtr = NULL;
364
365    if (AIn) {
366      InPtr = Builder.CreateLoad(AIn, "InPtr");
367      RootArgs.push_back(InPtr);
368    }
369
370    if (AOut) {
371      OutPtr = Builder.CreateLoad(AOut, "OutPtr");
372      RootArgs.push_back(OutPtr);
373    }
374
375    if (UsrData) {
376      RootArgs.push_back(UsrData);
377    }
378
379    // We always have to load X, since it is used to iterate through the loop.
380    llvm::Value *X = Builder.CreateLoad(AX, "X");
381    if (hasX(Signature)) {
382      RootArgs.push_back(X);
383    }
384
385    if (Y) {
386      RootArgs.push_back(Y);
387    }
388
389    Builder.CreateCall(F, RootArgs);
390
391    if (InPtr) {
392      // InPtr += instep
393      llvm::Value *NewIn = Builder.CreateIntToPtr(Builder.CreateNUWAdd(
394          Builder.CreatePtrToInt(InPtr, Int32Ty), InStep), InTy);
395      Builder.CreateStore(NewIn, AIn);
396    }
397
398    if (OutPtr) {
399      // OutPtr += outstep
400      llvm::Value *NewOut = Builder.CreateIntToPtr(Builder.CreateNUWAdd(
401          Builder.CreatePtrToInt(OutPtr, Int32Ty), OutStep), OutTy);
402      Builder.CreateStore(NewOut, AOut);
403    }
404
405    // X++;
406    llvm::Value *XPlusOne =
407        Builder.CreateNUWAdd(X, llvm::ConstantInt::get(Int32Ty, 1));
408    Builder.CreateStore(XPlusOne, AX);
409
410    // If (X < x2) goto Loop; else goto Exit;
411    Cond = Builder.CreateICmpSLT(XPlusOne, Arg_x2);
412    Builder.CreateCondBr(Cond, Loop, Exit);
413
414    return true;
415  }
416
417  /* Expand a pass-by-value kernel.
418   */
419  bool ExpandKernel(llvm::Function *F, uint32_t Signature) {
420    bccAssert(isKernel(Signature));
421    ALOGV("Expanding kernel Function %s", F->getName().str().c_str());
422
423    // TODO: Refactor this to share functionality with ExpandFunction.
424    llvm::DataLayout DL(M);
425
426    llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C);
427    llvm::Function *ExpandedFunc = createEmptyExpandedFunction(F->getName());
428
429    // Create and name the actual arguments to this expanded function.
430    llvm::SmallVector<llvm::Argument*, 8> ArgVec;
431    for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(),
432                                      E = ExpandedFunc->arg_end();
433         B != E;
434         ++B) {
435      ArgVec.push_back(B);
436    }
437
438    if (ArgVec.size() != 5) {
439      ALOGE("Incorrect number of arguments to function: %zu",
440            ArgVec.size());
441      return false;
442    }
443    llvm::Value *Arg_p = ArgVec[0];
444    llvm::Value *Arg_x1 = ArgVec[1];
445    llvm::Value *Arg_x2 = ArgVec[2];
446    llvm::Value *Arg_instep = ArgVec[3];
447    llvm::Value *Arg_outstep = ArgVec[4];
448
449    llvm::Value *InStep = NULL;
450    llvm::Value *OutStep = NULL;
451
452    // Construct the actual function body.
453    llvm::BasicBlock *Begin = &ExpandedFunc->getEntryBlock();
454    llvm::IRBuilder<> Builder(Begin->begin());
455
456    // uint32_t X = x1;
457    llvm::AllocaInst *AX = Builder.CreateAlloca(Int32Ty, 0, "AX");
458    Builder.CreateStore(Arg_x1, AX);
459
460    // Collect and construct the arguments for the kernel().
461    // Note that we load any loop-invariant arguments before entering the Loop.
462    llvm::Function::arg_iterator Args = F->arg_begin();
463
464    llvm::Type *OutTy = NULL;
465    llvm::AllocaInst *AOut = NULL;
466    bool PassOutByReference = false;
467    if (hasOut(Signature)) {
468      llvm::Type *OutBaseTy = F->getReturnType();
469      if (OutBaseTy->isVoidTy()) {
470        PassOutByReference = true;
471        OutTy = Args->getType();
472        Args++;
473      } else {
474        OutTy = OutBaseTy->getPointerTo();
475        // We don't increment Args, since we are using the actual return type.
476      }
477      AOut = Builder.CreateAlloca(OutTy, 0, "AOut");
478      OutStep = getStepValue(&DL, OutTy, Arg_outstep);
479      OutStep->setName("outstep");
480      Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad(
481          Builder.CreateStructGEP(Arg_p, 1)), OutTy), AOut);
482    }
483
484    llvm::Type *InBaseTy = NULL;
485    llvm::Type *InTy = NULL;
486    llvm::AllocaInst *AIn = NULL;
487    if (hasIn(Signature)) {
488      InBaseTy = Args->getType();
489      InTy =InBaseTy->getPointerTo();
490      AIn = Builder.CreateAlloca(InTy, 0, "AIn");
491      InStep = getStepValue(&DL, InTy, Arg_instep);
492      InStep->setName("instep");
493      Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad(
494          Builder.CreateStructGEP(Arg_p, 0)), InTy), AIn);
495      Args++;
496    }
497
498    // No usrData parameter on kernels.
499    bccAssert(!hasUsrData(Signature));
500
501    if (hasX(Signature)) {
502      Args++;
503    }
504
505    llvm::Value *Y = NULL;
506    if (hasY(Signature)) {
507      Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y");
508      Args++;
509    }
510
511    bccAssert(Args == F->arg_end());
512
513    llvm::BasicBlock *Loop = llvm::BasicBlock::Create(*C, "Loop", ExpandedFunc);
514
515    // if (x1 < x2) goto Loop; else goto Exit;
516    llvm::Value *Cond = Builder.CreateICmpSLT(Arg_x1, Arg_x2);
517
518    llvm::BasicBlock *Exit = llvm::SplitBlock(Builder.GetInsertBlock(),
519                                              Builder.GetInsertPoint(), this);
520    Exit->setName("Exit");
521    Begin->getTerminator()->eraseFromParent();
522    Builder.SetInsertPoint(Begin);
523    Builder.CreateCondBr(Cond, Loop, Exit);
524
525    // Loop:
526    Builder.SetInsertPoint(Loop);
527
528    // Populate the actual call to kernel().
529    llvm::SmallVector<llvm::Value*, 8> RootArgs;
530
531    llvm::Value *InPtr = NULL;
532    llvm::Value *In = NULL;
533    llvm::Value *OutPtr = NULL;
534
535    if (PassOutByReference) {
536      OutPtr = Builder.CreateLoad(AOut, "OutPtr");
537      RootArgs.push_back(OutPtr);
538    }
539
540    if (AIn) {
541      InPtr = Builder.CreateLoad(AIn, "InPtr");
542      In = Builder.CreateLoad(InPtr, "In");
543      RootArgs.push_back(In);
544    }
545
546    // We always have to load X, since it is used to iterate through the loop.
547    llvm::Value *X = Builder.CreateLoad(AX, "X");
548    if (hasX(Signature)) {
549      RootArgs.push_back(X);
550    }
551
552    if (Y) {
553      RootArgs.push_back(Y);
554    }
555
556    llvm::Value *RetVal = Builder.CreateCall(F, RootArgs);
557
558    if (AOut && !PassOutByReference) {
559      OutPtr = Builder.CreateLoad(AOut, "OutPtr");
560      Builder.CreateStore(RetVal, OutPtr);
561    }
562
563    if (InPtr) {
564      // InPtr += instep
565      llvm::Value *NewIn = Builder.CreateIntToPtr(Builder.CreateNUWAdd(
566          Builder.CreatePtrToInt(InPtr, Int32Ty), InStep), InTy);
567      Builder.CreateStore(NewIn, AIn);
568    }
569
570    if (OutPtr) {
571      // OutPtr += outstep
572      llvm::Value *NewOut = Builder.CreateIntToPtr(Builder.CreateNUWAdd(
573          Builder.CreatePtrToInt(OutPtr, Int32Ty), OutStep), OutTy);
574      Builder.CreateStore(NewOut, AOut);
575    }
576
577    // X++;
578    llvm::Value *XPlusOne =
579        Builder.CreateNUWAdd(X, llvm::ConstantInt::get(Int32Ty, 1));
580    Builder.CreateStore(XPlusOne, AX);
581
582    // If (X < x2) goto Loop; else goto Exit;
583    Cond = Builder.CreateICmpSLT(XPlusOne, Arg_x2);
584    Builder.CreateCondBr(Cond, Loop, Exit);
585
586    return true;
587  }
588
589  virtual bool runOnModule(llvm::Module &M) {
590    bool Changed = false;
591    this->M = &M;
592    C = &M.getContext();
593
594    for (RSInfo::ExportForeachFuncListTy::const_iterator
595             func_iter = mFuncs.begin(), func_end = mFuncs.end();
596         func_iter != func_end; func_iter++) {
597      const char *name = func_iter->first;
598      uint32_t signature = func_iter->second;
599      llvm::Function *kernel = M.getFunction(name);
600      if (kernel && isKernel(signature)) {
601        Changed |= ExpandKernel(kernel, signature);
602      }
603      else if (kernel && kernel->getReturnType()->isVoidTy()) {
604        Changed |= ExpandFunction(kernel, signature);
605      }
606    }
607
608    return Changed;
609  }
610
611  virtual const char *getPassName() const {
612    return "ForEach-able Function Expansion";
613  }
614
615}; // end RSForEachExpandPass
616
617} // end anonymous namespace
618
619char RSForEachExpandPass::ID = 0;
620
621namespace bcc {
622
623llvm::ModulePass *
624createRSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs,
625                          bool pEnableStepOpt){
626  return new RSForEachExpandPass(pForeachFuncs, pEnableStepOpt);
627}
628
629} // end namespace bcc
630