RSKernelExpand.cpp revision 33cda5cf335afc6aa2dbe02062bc9e6649e1f87c
1db169187dea4602e4ad32058762d23d474753fd0Stephen Hines/*
2db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Copyright 2012, The Android Open Source Project
3db169187dea4602e4ad32058762d23d474753fd0Stephen Hines *
4db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Licensed under the Apache License, Version 2.0 (the "License");
5db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * you may not use this file except in compliance with the License.
6db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * You may obtain a copy of the License at
7db169187dea4602e4ad32058762d23d474753fd0Stephen Hines *
8db169187dea4602e4ad32058762d23d474753fd0Stephen Hines *     http://www.apache.org/licenses/LICENSE-2.0
9db169187dea4602e4ad32058762d23d474753fd0Stephen Hines *
10db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Unless required by applicable law or agreed to in writing, software
11db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * distributed under the License is distributed on an "AS IS" BASIS,
12db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * See the License for the specific language governing permissions and
14db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * limitations under the License.
15db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */
16db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
176e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines#include "bcc/Assert.h"
18e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSTransforms.h"
197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao
207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao#include <cstdlib>
2133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross#include <functional>
227a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao
23b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DerivedTypes.h>
24b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Function.h>
25b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Instructions.h>
26b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/IRBuilder.h>
2718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser#include <llvm/IR/MDBuilder.h>
28b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Module.h>
29c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Pass.h>
307ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines#include <llvm/Support/raw_ostream.h>
31b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DataLayout.h>
32cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser#include <llvm/IR/Function.h>
33b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Type.h>
34806075b3a54af826fea78490fb213d8a0784138eTobias Grosser#include <llvm/Transforms/Utils/BasicBlockUtils.h>
35c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang
36c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include "bcc/Config/Config.h"
37ef73a242762bcd8113b9b65ceccbe7d909b5acbcZonr Chang#include "bcc/Support/Log.h"
38db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
39d88177580db4ddedf680854c51db333c97eabc59Stephen Hines#include "bcinfo/MetadataExtractor.h"
40d88177580db4ddedf680854c51db333c97eabc59Stephen Hines
415010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes#define NUM_EXPANDED_FUNCTION_PARAMS 4
42bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
437a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaousing namespace bcc;
447a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao
45db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace {
467a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao
4714588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chienstatic const bool gEnableRsTbaa = true;
489c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines
497a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao/* RSForEachExpandPass - This pass operates on functions that are able to be
507a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the
517a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * ForEach-able function to be invoked over the appropriate data cells of the
527a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * input/output allocations (adjusting other relevant parameters as we go). We
537a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * support doing this for any ForEach-able compute kernels. The new function
547a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * name is the original function name followed by ".expand". Note that we
557a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * still generate code for the original function.
567a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao */
577a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoclass RSForEachExpandPass : public llvm::ModulePass {
5833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grosspublic:
59db169187dea4602e4ad32058762d23d474753fd0Stephen Hines  static char ID;
60db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
6133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grossprivate:
6233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
63bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  llvm::Module *Module;
64bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  llvm::LLVMContext *Context;
65bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
66bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  /*
67bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes   * Pointer to LLVM type information for the ForEachStubType and the function
68bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes   * signature for expanded kernels.  These must be re-calculated for each
69bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes   * module the pass is run on.
70bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes   */
71bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  llvm::StructType   *ForEachStubType;
72bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  llvm::FunctionType *ExpandedFunctionType;
73db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
7425eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines  uint32_t mExportForEachCount;
7525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines  const char **mExportForEachNameList;
7625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines  const uint32_t *mExportForEachSignatureList;
77cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines
782b04086acbef6520ae2c54a868b1271abf053122Stephen Hines  // Turns on optimization of allocation stride values.
792b04086acbef6520ae2c54a868b1271abf053122Stephen Hines  bool mEnableStepOpt;
802b04086acbef6520ae2c54a868b1271abf053122Stephen Hines
81bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  uint32_t getRootSignature(llvm::Function *Function) {
82db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    const llvm::NamedMDNode *ExportForEachMetadata =
83bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes        Module->getNamedMetadata("#rs_export_foreach");
84db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
85db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    if (!ExportForEachMetadata) {
86db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      llvm::SmallVector<llvm::Type*, 8> RootArgTys;
87bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      for (llvm::Function::arg_iterator B = Function->arg_begin(),
88bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes                                        E = Function->arg_end();
89db169187dea4602e4ad32058762d23d474753fd0Stephen Hines           B != E;
90db169187dea4602e4ad32058762d23d474753fd0Stephen Hines           ++B) {
91db169187dea4602e4ad32058762d23d474753fd0Stephen Hines        RootArgTys.push_back(B->getType());
92db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      }
93db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
94db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      // For pre-ICS bitcode, we may not have signature information. In that
95db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      // case, we use the size of the RootArgTys to select the number of
96db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      // arguments.
97db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      return (1 << RootArgTys.size()) - 1;
98db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
99db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
1007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    if (ExportForEachMetadata->getNumOperands() == 0) {
1017ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines      return 0;
1027ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    }
1037ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
1046e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines    bccAssert(ExportForEachMetadata->getNumOperands() > 0);
105db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
106cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines    // We only handle the case for legacy root() functions here, so this is
107cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines    // hard-coded to look at only the first such function.
108db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0);
109900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    if (SigNode != nullptr && SigNode->getNumOperands() == 1) {
110db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      llvm::Value *SigVal = SigNode->getOperand(0);
111db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      if (SigVal->getValueID() == llvm::Value::MDStringVal) {
112db169187dea4602e4ad32058762d23d474753fd0Stephen Hines        llvm::StringRef SigString =
113db169187dea4602e4ad32058762d23d474753fd0Stephen Hines            static_cast<llvm::MDString*>(SigVal)->getString();
114db169187dea4602e4ad32058762d23d474753fd0Stephen Hines        uint32_t Signature = 0;
115db169187dea4602e4ad32058762d23d474753fd0Stephen Hines        if (SigString.getAsInteger(10, Signature)) {
116db169187dea4602e4ad32058762d23d474753fd0Stephen Hines          ALOGE("Non-integer signature value '%s'", SigString.str().c_str());
117db169187dea4602e4ad32058762d23d474753fd0Stephen Hines          return 0;
118db169187dea4602e4ad32058762d23d474753fd0Stephen Hines        }
119db169187dea4602e4ad32058762d23d474753fd0Stephen Hines        return Signature;
120db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      }
121db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
122db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
123db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    return 0;
124db169187dea4602e4ad32058762d23d474753fd0Stephen Hines  }
125db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
126429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray  bool isStepOptSupported(llvm::Type *AllocType) {
127429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
128429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType);
129429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context);
130429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
131429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    if (mEnableStepOpt) {
132429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray      return false;
133429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    }
134429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
135429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    if (AllocType == VoidPtrTy) {
136429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray      return false;
137429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    }
138429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
139429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    if (!PT) {
140429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray      return false;
141429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    }
142429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
143429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    // remaining conditions are 64-bit only
144429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    if (VoidPtrTy->getPrimitiveSizeInBits() == 32) {
145429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray      return true;
146429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    }
147429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
148429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    // coerce suggests an upconverted struct type, which we can't support
149429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    if (AllocType->getStructName().find("coerce") != llvm::StringRef::npos) {
150429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray      return false;
151429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    }
152429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
153429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    // 2xi64 and i128 suggest an upconverted struct type, which are also unsupported
154429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    llvm::Type *V2xi64Ty = llvm::VectorType::get(llvm::Type::getInt64Ty(*Context), 2);
155429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    llvm::Type *Int128Ty = llvm::Type::getIntNTy(*Context, 128);
156429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    if (AllocType == V2xi64Ty || AllocType == Int128Ty) {
157429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray      return false;
158429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    }
159429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
160429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    return true;
161429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray  }
162429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray
1632b04086acbef6520ae2c54a868b1271abf053122Stephen Hines  // Get the actual value we should use to step through an allocation.
1647b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser  //
1657b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser  // Normally the value we use to step through an allocation is given to us by
1667b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser  // the driver. However, for certain primitive data types, we can derive an
1677b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser  // integer constant for the step value. We use this integer constant whenever
1687b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser  // possible to allow further compiler optimizations to take place.
1697b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser  //
170b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines  // DL - Target Data size/layout information.
1712b04086acbef6520ae2c54a868b1271abf053122Stephen Hines  // T - Type of allocation (should be a pointer).
1722b04086acbef6520ae2c54a868b1271abf053122Stephen Hines  // OrigStep - Original step increment (root.expand() input from driver).
173bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *AllocType,
1742b04086acbef6520ae2c54a868b1271abf053122Stephen Hines                            llvm::Value *OrigStep) {
175b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines    bccAssert(DL);
176bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    bccAssert(AllocType);
1772b04086acbef6520ae2c54a868b1271abf053122Stephen Hines    bccAssert(OrigStep);
178bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType);
179429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray    if (isStepOptSupported(AllocType)) {
1802b04086acbef6520ae2c54a868b1271abf053122Stephen Hines      llvm::Type *ET = PT->getElementType();
181b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines      uint64_t ETSize = DL->getTypeAllocSize(ET);
182bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
1832b04086acbef6520ae2c54a868b1271abf053122Stephen Hines      return llvm::ConstantInt::get(Int32Ty, ETSize);
1842b04086acbef6520ae2c54a868b1271abf053122Stephen Hines    } else {
1852b04086acbef6520ae2c54a868b1271abf053122Stephen Hines      return OrigStep;
1862b04086acbef6520ae2c54a868b1271abf053122Stephen Hines    }
1872b04086acbef6520ae2c54a868b1271abf053122Stephen Hines  }
1882b04086acbef6520ae2c54a868b1271abf053122Stephen Hines
189e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_INS         0
190e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_INESTRIDES  1
191e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_OUT         2
192e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_Y           3
193e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_Z           4
194e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_LID         5
195e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_USR         6
196e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_DIMX        7
197e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_DIMY        8
198e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_DIMZ        9
199e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_SLOT       10
200097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes
201097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes  /// Builds the types required by the pass for the given context.
202bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  void buildTypes(void) {
203bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    // Create the RsForEachStubParam struct.
204bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
205097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    llvm::Type *VoidPtrTy    = llvm::Type::getInt8PtrTy(*Context);
206097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    llvm::Type *VoidPtrPtrTy = VoidPtrTy->getPointerTo();
207097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    llvm::Type *Int32Ty      = llvm::Type::getInt32Ty(*Context);
208097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    llvm::Type *Int32PtrTy   = Int32Ty->getPointerTo();
209097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes
210097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    /* Defined in frameworks/base/libs/rs/cpu_ref/rsCpuCore.h:
211db169187dea4602e4ad32058762d23d474753fd0Stephen Hines     *
212097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes     * struct RsForEachKernelStruct{
21333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross     *   const void **ins;
21433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross     *   uint32_t *inEStrides;
215db169187dea4602e4ad32058762d23d474753fd0Stephen Hines     *   void *out;
216db169187dea4602e4ad32058762d23d474753fd0Stephen Hines     *   uint32_t y;
217db169187dea4602e4ad32058762d23d474753fd0Stephen Hines     *   uint32_t z;
218097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes     *   uint32_t lid;
219097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes     *   const void *usr;
220097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes     *   uint32_t dimX;
221097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes     *   uint32_t dimY;
222097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes     *   uint32_t dimZ;
223097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes     *   uint32_t slot;
224db169187dea4602e4ad32058762d23d474753fd0Stephen Hines     * };
225db169187dea4602e4ad32058762d23d474753fd0Stephen Hines     */
226097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    llvm::SmallVector<llvm::Type*, 12> StructTypes;
227e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    StructTypes.push_back(VoidPtrPtrTy); // const void **ins
228e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    StructTypes.push_back(Int32PtrTy);   // uint32_t *inEStrides
229097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(VoidPtrTy);    // void *out
230097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(Int32Ty);      // uint32_t y
231097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(Int32Ty);      // uint32_t z
232097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(Int32Ty);      // uint32_t lid
233097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(VoidPtrTy);    // const void *usr
234097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(Int32Ty);      // uint32_t dimX
235097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(Int32Ty);      // uint32_t dimY
236097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(Int32Ty);      // uint32_t dimZ
237097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes    StructTypes.push_back(Int32Ty);      // uint32_t slot
238881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
239bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    ForEachStubType =
240bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      llvm::StructType::create(StructTypes, "RsForEachStubParamStruct");
241bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
242bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    // Create the function type for expanded kernels.
243bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
244bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Type *ForEachStubPtrTy = ForEachStubType->getPointerTo();
245bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
246bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::SmallVector<llvm::Type*, 8> ParamTypes;
247bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    ParamTypes.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p
248bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    ParamTypes.push_back(Int32Ty);          // uint32_t x1
249bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    ParamTypes.push_back(Int32Ty);          // uint32_t x2
250bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    ParamTypes.push_back(Int32Ty);          // uint32_t outstep
251bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
252e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    ExpandedFunctionType =
253e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes        llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), ParamTypes,
254e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes                                false);
2558ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser  }
2568ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser
257357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  /// @brief Create skeleton of the expanded function.
258357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  ///
259357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  /// This creates a function with the following signature:
260357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  ///
261357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  ///   void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2,
2625010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes  ///         uint32_t outstep)
263357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  ///
264357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) {
265bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Function *ExpandedFunction =
266bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      llvm::Function::Create(ExpandedFunctionType,
267bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes                             llvm::GlobalValue::ExternalLinkage,
268bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes                             OldName + ".expand", Module);
269bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
270bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS);
271bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
272bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Function::arg_iterator AI = ExpandedFunction->arg_begin();
273bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
274bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    (AI++)->setName("p");
275bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    (AI++)->setName("x1");
276bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    (AI++)->setName("x2");
277bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    (AI++)->setName("arg_outstep");
278bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
279bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin",
280bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes                                                       ExpandedFunction);
281806075b3a54af826fea78490fb213d8a0784138eTobias Grosser    llvm::IRBuilder<> Builder(Begin);
282806075b3a54af826fea78490fb213d8a0784138eTobias Grosser    Builder.CreateRetVoid();
283806075b3a54af826fea78490fb213d8a0784138eTobias Grosser
284bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    return ExpandedFunction;
285357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser  }
286357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser
287e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// @brief Create an empty loop
288e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  ///
289e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// Create a loop of the form:
290e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  ///
291e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// for (i = LowerBound; i < UpperBound; i++)
292e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  ///   ;
293e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  ///
294e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// After the loop has been created, the builder is set such that
295e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// instructions can be added to the loop body.
296e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  ///
297e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// @param Builder The builder to use to build this loop. The current
298e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  ///                position of the builder is the position the loop
299e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  ///                will be inserted.
300e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// @param LowerBound The first value of the loop iterator
301e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// @param UpperBound The maximal value of the loop iterator
302e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// @param LoopIV A reference that will be set to the loop iterator.
303e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  /// @return The BasicBlock that will be executed after the loop.
304e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder,
305e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser                               llvm::Value *LowerBound,
306e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser                               llvm::Value *UpperBound,
307e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser                               llvm::PHINode **LoopIV) {
308e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    assert(LowerBound->getType() == UpperBound->getType());
309e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser
310e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB;
311e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    llvm::Value *Cond, *IVNext;
312e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    llvm::PHINode *IV;
313e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser
314e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    CondBB = Builder.GetInsertBlock();
315e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), this);
316bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    HeaderBB = llvm::BasicBlock::Create(*Context, "Loop", CondBB->getParent());
317e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser
318e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    // if (LowerBound < Upperbound)
319e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    //   goto LoopHeader
320e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    // else
321e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    //   goto AfterBB
322e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    CondBB->getTerminator()->eraseFromParent();
323e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    Builder.SetInsertPoint(CondBB);
324e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser    Cond = Builder.CreateICmpULT(LowerBound, UpperBound);
325e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    Builder.CreateCondBr(Cond, HeaderBB, AfterBB);
326e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser
327e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ]
328e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    // iv.next = iv + 1
329e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    // if (iv.next < Upperbound)
330e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    //   goto LoopHeader
331e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    // else
332e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    //   goto AfterBB
333e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    Builder.SetInsertPoint(HeaderBB);
334e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    IV = Builder.CreatePHI(LowerBound->getType(), 2, "X");
335e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    IV->addIncoming(LowerBound, CondBB);
336e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1));
337e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    IV->addIncoming(IVNext, HeaderBB);
338e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser    Cond = Builder.CreateICmpULT(IVNext, UpperBound);
339e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    Builder.CreateCondBr(Cond, HeaderBB, AfterBB);
340e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    AfterBB->setName("Exit");
341e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    Builder.SetInsertPoint(HeaderBB->getFirstNonPHI());
342e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    *LoopIV = IV;
343e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser    return AfterBB;
344e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser  }
345e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser
3468ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosserpublic:
34733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  RSForEachExpandPass(bool pEnableStepOpt = true)
348900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes      : ModulePass(ID), Module(nullptr), Context(nullptr),
349bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes        mEnableStepOpt(pEnableStepOpt) {
350bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
3518ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser  }
3528ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser
353c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines  virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
354c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines    // This pass does not use any other analysis passes, but it does
355c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines    // add/wrap the existing functions in the module (thus altering the CFG).
356c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines  }
357c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines
35833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  // Build contribution to outgoing argument list for calling a
35933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  // ForEach-able function, based on the special parameters of that
36033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  // function.
36133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  //
36233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  // Signature - metadata bits for the signature of the ForEach-able function
36333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  // X, Arg_p - values derived directly from expanded function,
36433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  //            suitable for computing arguments for the ForEach-able function
36533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  // CalleeArgs - contribution is accumulated here
36633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  // Bump - invoked once for each contributed outgoing argument
36733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  void ExpandSpecialArguments(uint32_t Signature,
36833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                              llvm::Value *X,
36933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                              llvm::Value *Arg_p,
37033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                              llvm::IRBuilder<> &Builder,
37133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                              llvm::SmallVector<llvm::Value*, 8> &CalleeArgs,
37233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                              std::function<void ()> Bump) {
37333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
37433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    if (bcinfo::MetadataExtractor::hasForEachSignatureCtxt(Signature)) {
37533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      CalleeArgs.push_back(Arg_p);
37633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      Bump();
37733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    }
37833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
37933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) {
38033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      CalleeArgs.push_back(X);
38133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      Bump();
38233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    }
38333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
38433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) {
38533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      llvm::Value *Y = Builder.CreateLoad(
38633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                         Builder.CreateStructGEP(Arg_p, PARAM_FIELD_Y), "Y");
38733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      CalleeArgs.push_back(Y);
38833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      Bump();
38933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    }
39033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
39133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    if (bcinfo::MetadataExtractor::hasForEachSignatureZ(Signature)) {
39233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      llvm::Value *Z = Builder.CreateLoad(
39333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                         Builder.CreateStructGEP(Arg_p, PARAM_FIELD_Z), "Z");
39433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      CalleeArgs.push_back(Z);
39533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross      Bump();
39633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    }
39733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross  }
39833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
3998ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser  /* Performs the actual optimization on a selected function. On success, the
4008ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser   * Module will contain a new function of the name "<NAME>.expand" that
4018ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser   * invokes <NAME>() in a loop with the appropriate parameters.
4028ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser   */
403bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  bool ExpandFunction(llvm::Function *Function, uint32_t Signature) {
404bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    ALOGV("Expanding ForEach-able Function %s",
405bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes          Function->getName().str().c_str());
4068ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser
4078ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser    if (!Signature) {
408bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      Signature = getRootSignature(Function);
4098ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser      if (!Signature) {
4108ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser        // We couldn't determine how to expand this function based on its
4118ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser        // function signature.
4128ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser        return false;
4138ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser      }
4148ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser    }
4158ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser
416bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::DataLayout DL(Module);
4178ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser
418bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Function *ExpandedFunction =
419bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      createEmptyExpandedFunction(Function->getName());
420db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
421bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    /*
422bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes     * Extract the expanded function's parameters.  It is guaranteed by
423bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes     * createEmptyExpandedFunction that there will be five parameters.
424bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes     */
42533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
42633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS);
42733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
428bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Function::arg_iterator ExpandedFunctionArgIter =
429bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      ExpandedFunction->arg_begin();
430db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
431bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Value *Arg_p       = &*(ExpandedFunctionArgIter++);
432bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Value *Arg_x1      = &*(ExpandedFunctionArgIter++);
433bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Value *Arg_x2      = &*(ExpandedFunctionArgIter++);
4345010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes    llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter);
435bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
436900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *InStep  = nullptr;
437900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *OutStep = nullptr;
438db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
439db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    // Construct the actual function body.
440bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin());
441db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
442cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines    // Collect and construct the arguments for the kernel().
443db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    // Note that we load any loop-invariant arguments before entering the Loop.
444bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Function::arg_iterator FunctionArgIter = Function->arg_begin();
445db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
446900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Type  *InTy      = nullptr;
447900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *InBasePtr = nullptr;
448d88177580db4ddedf680854c51db333c97eabc59Stephen Hines    if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) {
449e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::Value    *InsMember  = Builder.CreateStructGEP(Arg_p,
450e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes                                                           PARAM_FIELD_INS);
451e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::LoadInst *InsBasePtr = Builder.CreateLoad(InsMember, "inputs_base");
452e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes
453e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::Value *InStepsMember =
454e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes        Builder.CreateStructGEP(Arg_p, PARAM_FIELD_INESTRIDES);
455e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::LoadInst *InStepsBase = Builder.CreateLoad(InStepsMember,
456e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes                                                       "insteps_base");
457e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes
458e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::Value *IndexVal = Builder.getInt32(0);
459e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes
460e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::Value    *InStepAddr = Builder.CreateGEP(InStepsBase, IndexVal);
461e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::LoadInst *InStepArg  = Builder.CreateLoad(InStepAddr,
462e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes                                                      "instep_addr");
463e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes
464bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      InTy = (FunctionArgIter++)->getType();
465e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      InStep = getStepValue(&DL, InTy, InStepArg);
466e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes
4672b04086acbef6520ae2c54a868b1271abf053122Stephen Hines      InStep->setName("instep");
468e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes
469e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::Value *InputAddr = Builder.CreateGEP(InsBasePtr, IndexVal);
470e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      InBasePtr = Builder.CreateLoad(InputAddr, "input_base");
471db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
472db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
473900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Type *OutTy = nullptr;
474900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *OutBasePtr = nullptr;
475d88177580db4ddedf680854c51db333c97eabc59Stephen Hines    if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) {
476bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      OutTy = (FunctionArgIter++)->getType();
477b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines      OutStep = getStepValue(&DL, OutTy, Arg_outstep);
4782b04086acbef6520ae2c54a868b1271abf053122Stephen Hines      OutStep->setName("outstep");
479097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes      OutBasePtr = Builder.CreateLoad(
480097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes                     Builder.CreateStructGEP(Arg_p, PARAM_FIELD_OUT));
481db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
482db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
483900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *UsrData = nullptr;
484d88177580db4ddedf680854c51db333c97eabc59Stephen Hines    if (bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)) {
485bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      llvm::Type *UsrDataTy = (FunctionArgIter++)->getType();
486db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      UsrData = Builder.CreatePointerCast(Builder.CreateLoad(
487097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes          Builder.CreateStructGEP(Arg_p, PARAM_FIELD_USR)), UsrDataTy);
488db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      UsrData->setName("UsrData");
489db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
490db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
49133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    llvm::PHINode *IV;
49233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    createLoop(Builder, Arg_x1, Arg_x2, &IV);
493097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes
49433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    llvm::SmallVector<llvm::Value*, 8> CalleeArgs;
49533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs,
49633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                           [&FunctionArgIter]() { FunctionArgIter++; });
497db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
498bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    bccAssert(FunctionArgIter == Function->arg_end());
499db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
500cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines    // Populate the actual call to kernel().
501db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    llvm::SmallVector<llvm::Value*, 8> RootArgs;
502db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
503900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *InPtr  = nullptr;
504900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *OutPtr = nullptr;
505db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
506ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser    // Calculate the current input and output pointers
50702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    //
508ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser    // We always calculate the input/output pointers with a GEP operating on i8
50902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    // values and only cast at the very end to OutTy. This is because the step
51002f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    // between two values is given in bytes.
51102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    //
51202f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    // TODO: We could further optimize the output by using a GEP operation of
51302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    // type 'OutTy' in cases where the element type of the allocation allows.
51402f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    if (OutBasePtr) {
51502f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser      llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1);
51602f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser      OutOffset = Builder.CreateMul(OutOffset, OutStep);
51702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser      OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset);
51802f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser      OutPtr = Builder.CreatePointerCast(OutPtr, OutTy);
51902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    }
520bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
521ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser    if (InBasePtr) {
522ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser      llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1);
523ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser      InOffset = Builder.CreateMul(InOffset, InStep);
524ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser      InPtr = Builder.CreateGEP(InBasePtr, InOffset);
525ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser      InPtr = Builder.CreatePointerCast(InPtr, InTy);
526ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser    }
52702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser
528ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser    if (InPtr) {
5297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines      RootArgs.push_back(InPtr);
530db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
531db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
53202f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser    if (OutPtr) {
5337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines      RootArgs.push_back(OutPtr);
534db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
535db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
536db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    if (UsrData) {
537db169187dea4602e4ad32058762d23d474753fd0Stephen Hines      RootArgs.push_back(UsrData);
538db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
539db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
54033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    RootArgs.append(CalleeArgs.begin(), CalleeArgs.end());
541db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
542bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    Builder.CreateCall(Function, RootArgs);
543db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
5447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    return true;
5457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines  }
5467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
5477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines  /* Expand a pass-by-value kernel.
5487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines   */
549bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  bool ExpandKernel(llvm::Function *Function, uint32_t Signature) {
550d88177580db4ddedf680854c51db333c97eabc59Stephen Hines    bccAssert(bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature));
551bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    ALOGV("Expanding kernel Function %s", Function->getName().str().c_str());
5527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
5537ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    // TODO: Refactor this to share functionality with ExpandFunction.
554bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::DataLayout DL(Module);
5557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
556bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Function *ExpandedFunction =
557bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      createEmptyExpandedFunction(Function->getName());
5587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
559bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    /*
560bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes     * Extract the expanded function's parameters.  It is guaranteed by
561bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes     * createEmptyExpandedFunction that there will be five parameters.
562bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes     */
563881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
564881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS);
565881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
566bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Function::arg_iterator ExpandedFunctionArgIter =
567bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      ExpandedFunction->arg_begin();
568bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
569bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Value *Arg_p       = &*(ExpandedFunctionArgIter++);
570bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Value *Arg_x1      = &*(ExpandedFunctionArgIter++);
571bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Value *Arg_x2      = &*(ExpandedFunctionArgIter++);
5725010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes    llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter);
5737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
5747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    // Construct the actual function body.
575bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin());
5767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
57718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // Create TBAA meta-data.
57818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    llvm::MDNode *TBAARenderScript, *TBAAAllocation, *TBAAPointer;
579bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::MDBuilder MDHelper(*Context);
58014588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien
58118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    TBAARenderScript = MDHelper.createTBAARoot("RenderScript TBAA");
582e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    TBAAAllocation = MDHelper.createTBAAScalarTypeNode("allocation",
583e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes                                                       TBAARenderScript);
584e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    TBAAAllocation = MDHelper.createTBAAStructTagNode(TBAAAllocation,
585e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes                                                      TBAAAllocation, 0);
586e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    TBAAPointer = MDHelper.createTBAAScalarTypeNode("pointer",
587e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes                                                    TBAARenderScript);
58814588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien    TBAAPointer = MDHelper.createTBAAStructTagNode(TBAAPointer, TBAAPointer, 0);
58918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
59050f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray    llvm::MDNode *AliasingDomain, *AliasingScope;
59150f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray    AliasingDomain = MDHelper.createAnonymousAliasScopeDomain("RS argument scope domain");
59250f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray    AliasingScope = MDHelper.createAnonymousAliasScope(AliasingDomain, "RS argument scope");
59350f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray
594881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    /*
595881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes     * Collect and construct the arguments for the kernel().
596881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes     *
597881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes     * Note that we load any loop-invariant arguments before entering the Loop.
598881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes     */
599881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    size_t NumInputs = Function->arg_size();
6007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
601881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // No usrData parameter on kernels.
602881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    bccAssert(
603881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        !bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature));
604881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
605881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    llvm::Function::arg_iterator ArgIter = Function->arg_begin();
606881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
607881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // Check the return type
608bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::Type     *OutTy            = nullptr;
609bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::Value    *OutStep          = nullptr;
610bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::LoadInst *OutBasePtr       = nullptr;
611bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::Value    *CastedOutBasePtr = nullptr;
612881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
613e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    bool PassOutByPointer = false;
614881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
615d88177580db4ddedf680854c51db333c97eabc59Stephen Hines    if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) {
616bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      llvm::Type *OutBaseTy = Function->getReturnType();
617881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
61874a4b08235990916911b8fe758d656c1171faf26Stephen Hines      if (OutBaseTy->isVoidTy()) {
619e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes        PassOutByPointer = true;
620881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        OutTy = ArgIter->getType();
621881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
622881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        ArgIter++;
623881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        --NumInputs;
62474a4b08235990916911b8fe758d656c1171faf26Stephen Hines      } else {
62574a4b08235990916911b8fe758d656c1171faf26Stephen Hines        // We don't increment Args, since we are using the actual return type.
626881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        OutTy = OutBaseTy->getPointerTo();
62774a4b08235990916911b8fe758d656c1171faf26Stephen Hines      }
628881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
629b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines      OutStep = getStepValue(&DL, OutTy, Arg_outstep);
63074a4b08235990916911b8fe758d656c1171faf26Stephen Hines      OutStep->setName("outstep");
631097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes      OutBasePtr = Builder.CreateLoad(
632097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes                     Builder.CreateStructGEP(Arg_p, PARAM_FIELD_OUT));
633097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes
6349c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines      if (gEnableRsTbaa) {
6359c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines        OutBasePtr->setMetadata("tbaa", TBAAPointer);
6369c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines      }
63750f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray
63850f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray      OutBasePtr->setMetadata("alias.scope", AliasingScope);
63950f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray
640bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray      CastedOutBasePtr = Builder.CreatePointerCast(OutBasePtr, OutTy, "casted_out");
64174a4b08235990916911b8fe758d656c1171faf26Stephen Hines    }
64274a4b08235990916911b8fe758d656c1171faf26Stephen Hines
64333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    llvm::PHINode *IV;
64433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    createLoop(Builder, Arg_x1, Arg_x2, &IV);
64533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
64633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    llvm::SmallVector<llvm::Value*, 8> CalleeArgs;
64733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs,
64833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross                           [&NumInputs]() { --NumInputs; });
64933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross
650bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::SmallVector<llvm::Type*,  8> InTypes;
651bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::SmallVector<llvm::Value*, 8> InSteps;
652bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::SmallVector<llvm::Value*, 8> InBasePtrs;
653bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    llvm::SmallVector<bool,         8> InIsStructPointer;
654881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
655e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    if (NumInputs > 0) {
656e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::Value *InsMember = Builder.CreateStructGEP(Arg_p, PARAM_FIELD_INS);
657e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::LoadInst *InsBasePtr = Builder.CreateLoad(InsMember, "inputs_base");
6587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
659e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::Value *InStepsMember =
660e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes        Builder.CreateStructGEP(Arg_p, PARAM_FIELD_INESTRIDES);
661e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      llvm::LoadInst *InStepsBase = Builder.CreateLoad(InStepsMember,
662881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes                                                         "insteps_base");
663881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
664881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes      for (size_t InputIndex = 0; InputIndex < NumInputs;
665881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes           ++InputIndex, ArgIter++) {
666881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
667881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          llvm::Value *IndexVal = Builder.getInt32(InputIndex);
668881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
669881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          llvm::Value    *InStepAddr = Builder.CreateGEP(InStepsBase, IndexVal);
670881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          llvm::LoadInst *InStepArg  = Builder.CreateLoad(InStepAddr,
671881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes                                                          "instep_addr");
672881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
673326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          llvm::Type *InType = ArgIter->getType();
674326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes
675326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes        /*
676326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         * AArch64 calling dictate that structs of sufficient size get passed by
677326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         * pointer instead of passed by value.  This, combined with the fact
678326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         * that we don't allow kernels to operate on pointer data means that if
679326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         * we see a kernel with a pointer parameter we know that it is struct
680326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         * input that has been promoted.  As such we don't need to convert its
681326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         * type to a pointer.  Later we will need to know to avoid a load, so we
682326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         * save this information in InIsStructPointer.
683326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes         */
684326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          if (!InType->isPointerTy()) {
685326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes            InType = InType->getPointerTo();
686326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes            InIsStructPointer.push_back(false);
687326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          } else {
688326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes            InIsStructPointer.push_back(true);
689326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          }
690326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes
691881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          llvm::Value *InStep = getStepValue(&DL, InType, InStepArg);
692881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
693881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          InStep->setName("instep");
694881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
695881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          llvm::Value    *InputAddr = Builder.CreateGEP(InsBasePtr, IndexVal);
696881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          llvm::LoadInst *InBasePtr = Builder.CreateLoad(InputAddr,
697881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes                                                         "input_base");
698bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray          llvm::Value    *CastInBasePtr = Builder.CreatePointerCast(InBasePtr,
699bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray                                                                    InType, "casted_in");
700881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          if (gEnableRsTbaa) {
701881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes            InBasePtr->setMetadata("tbaa", TBAAPointer);
702881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          }
703881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
70450f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray          InBasePtr->setMetadata("alias.scope", AliasingScope);
70550f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray
706881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          InTypes.push_back(InType);
707881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes          InSteps.push_back(InStep);
708bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray          InBasePtrs.push_back(CastInBasePtr);
709881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes      }
710881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    }
7117ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
7127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    // Populate the actual call to kernel().
7137ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    llvm::SmallVector<llvm::Value*, 8> RootArgs;
7147ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
7154102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser    // Calculate the current input and output pointers
7167b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser    //
7177b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser    //
718881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // We always calculate the input/output pointers with a GEP operating on i8
719881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // values combined with a multiplication and only cast at the very end to
720881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // OutTy.  This is to account for dynamic stepping sizes when the value
721881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // isn't apparent at compile time.  In the (very common) case when we know
722881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // the step size at compile time, due to haveing complete type information
723881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // this multiplication will optmized out and produces code equivalent to a
724881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // a GEP on a pointer of the correct type.
725881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
726881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // Output
727881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
728900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes    llvm::Value *OutPtr = nullptr;
729bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray    if (CastedOutBasePtr) {
7307b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser      llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1);
731881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
732bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray      OutPtr    = Builder.CreateGEP(CastedOutBasePtr, OutOffset);
733bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
734e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes      if (PassOutByPointer) {
735881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        RootArgs.push_back(OutPtr);
736881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes      }
7374102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser    }
7387b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser
739881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    // Inputs
74074a4b08235990916911b8fe758d656c1171faf26Stephen Hines
741881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes    if (NumInputs > 0) {
742881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes      llvm::Value *Offset = Builder.CreateSub(IV, Arg_x1);
743881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
744881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes      for (size_t Index = 0; Index < NumInputs; ++Index) {
745bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray        llvm::Value *InPtr    = Builder.CreateGEP(InBasePtrs[Index], Offset);
746326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes        llvm::Value *Input;
747326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes
748326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes        if (InIsStructPointer[Index]) {
749326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          Input = InPtr;
750326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes
751326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes        } else {
752326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          llvm::LoadInst *InputLoad = Builder.CreateLoad(InPtr, "input");
753326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes
754326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          if (gEnableRsTbaa) {
755326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes            InputLoad->setMetadata("tbaa", TBAAAllocation);
756326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          }
757881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
75850f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray          InputLoad->setMetadata("alias.scope", AliasingScope);
75950f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray
760326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes          Input = InputLoad;
761881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        }
762881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes
763881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes        RootArgs.push_back(Input);
7649c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines      }
7657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    }
7667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
76733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross    RootArgs.append(CalleeArgs.begin(), CalleeArgs.end());
7687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
769bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::Value *RetVal = Builder.CreateCall(Function, RootArgs);
7707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
771e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes    if (OutPtr && !PassOutByPointer) {
77218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser      llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr);
7739c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines      if (gEnableRsTbaa) {
7749c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines        Store->setMetadata("tbaa", TBAAAllocation);
7759c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines      }
77650f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray      Store->setMetadata("alias.scope", AliasingScope);
7777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines    }
7787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines
779db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    return true;
780db169187dea4602e4ad32058762d23d474753fd0Stephen Hines  }
781db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
78218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// @brief Checks if pointers to allocation internals are exposed
78318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  ///
78418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// This function verifies if through the parameters passed to the kernel
78518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// or through calls to the runtime library the script gains access to
78618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// pointers pointing to data within a RenderScript Allocation.
78718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// If we know we control all loads from and stores to data within
78818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// RenderScript allocations and if we know the run-time internal accesses
78918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// are all annotated with RenderScript TBAA metadata, only then we
79018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// can safely use TBAA to distinguish between generic and from-allocation
79118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// pointers.
792bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  bool allocPointersExposed(llvm::Module &Module) {
79318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // Old style kernel function can expose pointers to elements within
79418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // allocations.
79518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // TODO: Extend analysis to allow simple cases of old-style kernels.
79625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines    for (size_t i = 0; i < mExportForEachCount; ++i) {
79725eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines      const char *Name = mExportForEachNameList[i];
79825eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines      uint32_t Signature = mExportForEachSignatureList[i];
799bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      if (Module.getFunction(Name) &&
800d88177580db4ddedf680854c51db333c97eabc59Stephen Hines          !bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)) {
80118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser        return true;
80218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser      }
80318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    }
80418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
80518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // Check for library functions that expose a pointer to an Allocation or
80618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // that are not yet annotated with RenderScript-specific tbaa information.
80718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    static std::vector<std::string> Funcs;
80818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
80918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // rsGetElementAt(...)
81018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z14rsGetElementAt13rs_allocationj");
81118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z14rsGetElementAt13rs_allocationjj");
81218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z14rsGetElementAt13rs_allocationjjj");
81318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // rsSetElementAt()
81418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvj");
81518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjj");
81618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjjj");
81718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // rsGetElementAtYuv_uchar_Y()
81818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z25rsGetElementAtYuv_uchar_Y13rs_allocationjj");
81918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // rsGetElementAtYuv_uchar_U()
82018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z25rsGetElementAtYuv_uchar_U13rs_allocationjj");
82118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    // rsGetElementAtYuv_uchar_V()
82218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    Funcs.push_back("_Z25rsGetElementAtYuv_uchar_V13rs_allocationjj");
82318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
82418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    for (std::vector<std::string>::iterator FI = Funcs.begin(),
82518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser                                            FE = Funcs.end();
82618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser         FI != FE; ++FI) {
827bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      llvm::Function *Function = Module.getFunction(*FI);
82818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
829bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      if (!Function) {
83018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser        ALOGE("Missing run-time function '%s'", FI->c_str());
83118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser        return true;
83218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser      }
83318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
834bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      if (Function->getNumUses() > 0) {
83518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser        return true;
83618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser      }
83718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    }
83818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
83918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    return false;
84018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  }
84118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
84218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// @brief Connect RenderScript TBAA metadata to C/C++ metadata
84318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  ///
84418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  /// The TBAA metadata used to annotate loads/stores from RenderScript
845e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes  /// Allocations is generated in a separate TBAA tree with a
846e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes  /// "RenderScript TBAA" root node. LLVM does assume may-alias for all nodes in
847e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes  /// unrelated alias analysis trees. This function makes the RenderScript TBAA
848e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes  /// a subtree of the normal C/C++ TBAA tree aside of normal C/C++ types. With
849e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes  /// the connected trees every access to an Allocation is resolved to
850e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes  /// must-alias if compared to a normal C/C++ access.
851bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  void connectRenderScriptTBAAMetadata(llvm::Module &Module) {
852bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::MDBuilder MDHelper(*Context);
853bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::MDNode *TBAARenderScript =
854bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      MDHelper.createTBAARoot("RenderScript TBAA");
85518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
856bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::MDNode *TBAARoot     = MDHelper.createTBAARoot("Simple C/C++ TBAA");
857bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    llvm::MDNode *TBAAMergedRS = MDHelper.createTBAANode("RenderScript",
858bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes                                                         TBAARoot);
85918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
86018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    TBAARenderScript->replaceAllUsesWith(TBAAMergedRS);
86118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser  }
86218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
863bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes  virtual bool runOnModule(llvm::Module &Module) {
864bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    bool Changed  = false;
865bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    this->Module  = &Module;
866bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    this->Context = &Module.getContext();
867bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
868bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    this->buildTypes();
869bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes
870bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    bcinfo::MetadataExtractor me(&Module);
87125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines    if (!me.extract()) {
87225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines      ALOGE("Could not extract metadata from module!");
87325eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines      return false;
87425eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines    }
87525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines    mExportForEachCount = me.getExportForEachSignatureCount();
87625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines    mExportForEachNameList = me.getExportForEachNameList();
87725eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines    mExportForEachSignatureList = me.getExportForEachSignatureList();
878db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
879bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes    bool AllocsExposed = allocPointersExposed(Module);
88018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
88125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines    for (size_t i = 0; i < mExportForEachCount; ++i) {
88225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines      const char *name = mExportForEachNameList[i];
88325eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines      uint32_t signature = mExportForEachSignatureList[i];
884bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      llvm::Function *kernel = Module.getFunction(name);
885cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser      if (kernel) {
886d88177580db4ddedf680854c51db333c97eabc59Stephen Hines        if (bcinfo::MetadataExtractor::hasForEachSignatureKernel(signature)) {
887cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser          Changed |= ExpandKernel(kernel, signature);
888acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser          kernel->setLinkage(llvm::GlobalValue::InternalLinkage);
889acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser        } else if (kernel->getReturnType()->isVoidTy()) {
890cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser          Changed |= ExpandFunction(kernel, signature);
891acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser          kernel->setLinkage(llvm::GlobalValue::InternalLinkage);
892acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser        } else {
893acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser          // There are some graphics root functions that are not
894acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser          // expanded, but that will be called directly. For those
895acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser          // functions, we can not set the linkage to internal.
896acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser        }
897cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines      }
898db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    }
899db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
9009c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines    if (gEnableRsTbaa && !AllocsExposed) {
901bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes      connectRenderScriptTBAAMetadata(Module);
90218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser    }
90318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser
904cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines    return Changed;
905db169187dea4602e4ad32058762d23d474753fd0Stephen Hines  }
906db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
907db169187dea4602e4ad32058762d23d474753fd0Stephen Hines  virtual const char *getPassName() const {
908db169187dea4602e4ad32058762d23d474753fd0Stephen Hines    return "ForEach-able Function Expansion";
909db169187dea4602e4ad32058762d23d474753fd0Stephen Hines  }
910db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
9117a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}; // end RSForEachExpandPass
912db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
9137a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end anonymous namespace
9147a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao
9157a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaochar RSForEachExpandPass::ID = 0;
91633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grossstatic llvm::RegisterPass<RSForEachExpandPass> X("foreachexp", "ForEach Expand Pass");
917db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
918db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace bcc {
919db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
9207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaollvm::ModulePass *
92125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen HinescreateRSForEachExpandPass(bool pEnableStepOpt){
92225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines  return new RSForEachExpandPass(pEnableStepOpt);
9237a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}
924db169187dea4602e4ad32058762d23d474753fd0Stephen Hines
9257a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end namespace bcc
926