RSKernelExpand.cpp revision 28c17995669687cecc96eaba232eecd3b65f5bbf
1db169187dea4602e4ad32058762d23d474753fd0Stephen Hines/* 2db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Copyright 2012, The Android Open Source Project 3db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 4db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Licensed under the Apache License, Version 2.0 (the "License"); 5db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * you may not use this file except in compliance with the License. 6db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * You may obtain a copy of the License at 7db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 8db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * http://www.apache.org/licenses/LICENSE-2.0 9db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 10db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Unless required by applicable law or agreed to in writing, software 11db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * distributed under the License is distributed on an "AS IS" BASIS, 12db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * See the License for the specific language governing permissions and 14db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * limitations under the License. 15db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 16db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 176e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines#include "bcc/Assert.h" 18e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSTransforms.h" 197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao#include <cstdlib> 2133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross#include <functional> 227a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 23b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DerivedTypes.h> 24b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Function.h> 25b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Instructions.h> 26b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/IRBuilder.h> 2718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser#include <llvm/IR/MDBuilder.h> 28b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Module.h> 29c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Pass.h> 307ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines#include <llvm/Support/raw_ostream.h> 31b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DataLayout.h> 32cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser#include <llvm/IR/Function.h> 33b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Type.h> 34806075b3a54af826fea78490fb213d8a0784138eTobias Grosser#include <llvm/Transforms/Utils/BasicBlockUtils.h> 35c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang 36c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include "bcc/Config/Config.h" 37ef73a242762bcd8113b9b65ceccbe7d909b5acbcZonr Chang#include "bcc/Support/Log.h" 38db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 39d88177580db4ddedf680854c51db333c97eabc59Stephen Hines#include "bcinfo/MetadataExtractor.h" 40d88177580db4ddedf680854c51db333c97eabc59Stephen Hines 415010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes#define NUM_EXPANDED_FUNCTION_PARAMS 4 42bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 437a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaousing namespace bcc; 447a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 45db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace { 467a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 47354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hinesstatic const bool gEnableRsTbaa = true; 489c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines 497a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao/* RSForEachExpandPass - This pass operates on functions that are able to be 507a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the 517a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * ForEach-able function to be invoked over the appropriate data cells of the 527a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * input/output allocations (adjusting other relevant parameters as we go). We 537a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * support doing this for any ForEach-able compute kernels. The new function 547a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * name is the original function name followed by ".expand". Note that we 557a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * still generate code for the original function. 567a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao */ 577a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoclass RSForEachExpandPass : public llvm::ModulePass { 5833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grosspublic: 59db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static char ID; 60db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grossprivate: 62e44a3525b9703739534c3b62d7d1af4c95649a38David Gross static const size_t RS_KERNEL_INPUT_LIMIT = 8; // see frameworks/base/libs/rs/cpu_ref/rsCpuCoreRuntime.h 63e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 64e44a3525b9703739534c3b62d7d1af4c95649a38David Gross enum RsLaunchDimensionsField { 65e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsFieldX, 66e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsFieldY, 67e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsFieldZ, 68e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsFieldLod, 69e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsFieldFace, 70e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsFieldArray, 71e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 72e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsFieldCount 73e44a3525b9703739534c3b62d7d1af4c95649a38David Gross }; 74e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 75e44a3525b9703739534c3b62d7d1af4c95649a38David Gross enum RsExpandKernelDriverInfoPfxField { 76e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldInPtr, 77e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldInStride, 78e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldInLen, 79e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldOutPtr, 80e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldOutStride, 81e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldOutLen, 82e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldDim, 83e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldCurrent, 84e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldUsr, 85e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldUsLenr, 86e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 87e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxFieldCount 88e44a3525b9703739534c3b62d7d1af4c95649a38David Gross }; 8933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 90bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Module *Module; 91bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::LLVMContext *Context; 92bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 93bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 94e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * Pointer to LLVM type information for the the function signature 95e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * for expanded kernels. This must be re-calculated for each 96bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * module the pass is run on. 97bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 98bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::FunctionType *ExpandedFunctionType; 99db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 10025eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t mExportForEachCount; 10125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char **mExportForEachNameList; 10225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const uint32_t *mExportForEachSignatureList; 103cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines 1042b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Turns on optimization of allocation stride values. 1052b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool mEnableStepOpt; 1062b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 107bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes uint32_t getRootSignature(llvm::Function *Function) { 108db169187dea4602e4ad32058762d23d474753fd0Stephen Hines const llvm::NamedMDNode *ExportForEachMetadata = 109bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Module->getNamedMetadata("#rs_export_foreach"); 110db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 111db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (!ExportForEachMetadata) { 112db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 8> RootArgTys; 113bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes for (llvm::Function::arg_iterator B = Function->arg_begin(), 114bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes E = Function->arg_end(); 115db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 116db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 117db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgTys.push_back(B->getType()); 118db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 119db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 120db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // For pre-ICS bitcode, we may not have signature information. In that 121db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // case, we use the size of the RootArgTys to select the number of 122db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // arguments. 123db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return (1 << RootArgTys.size()) - 1; 124db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 125db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1267ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ExportForEachMetadata->getNumOperands() == 0) { 1277ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return 0; 1287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 1297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 1306e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(ExportForEachMetadata->getNumOperands() > 0); 131db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 132cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // We only handle the case for legacy root() functions here, so this is 133cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // hard-coded to look at only the first such function. 134db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0); 135900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes if (SigNode != nullptr && SigNode->getNumOperands() == 1) { 1361bd9f627fa0affb457507e86b0b6684c695fe726Stephen Hines llvm::Metadata *SigMD = SigNode->getOperand(0); 1371bd9f627fa0affb457507e86b0b6684c695fe726Stephen Hines if (llvm::MDString *SigS = llvm::dyn_cast<llvm::MDString>(SigMD)) { 1381bd9f627fa0affb457507e86b0b6684c695fe726Stephen Hines llvm::StringRef SigString = SigS->getString(); 139db169187dea4602e4ad32058762d23d474753fd0Stephen Hines uint32_t Signature = 0; 140db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigString.getAsInteger(10, Signature)) { 141db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ALOGE("Non-integer signature value '%s'", SigString.str().c_str()); 142db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 143db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 144db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return Signature; 145db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 146db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 147db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 148db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 149db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 150db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 151429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray bool isStepOptSupported(llvm::Type *AllocType) { 152429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 153429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType); 154429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 155429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 156429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (mEnableStepOpt) { 157429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 158429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 159429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 160429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (AllocType == VoidPtrTy) { 161429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 162429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 163429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 164429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (!PT) { 165429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 166429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 167429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 168429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray // remaining conditions are 64-bit only 169429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (VoidPtrTy->getPrimitiveSizeInBits() == 32) { 170429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return true; 171429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 172429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 173429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray // coerce suggests an upconverted struct type, which we can't support 174429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (AllocType->getStructName().find("coerce") != llvm::StringRef::npos) { 175429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 176429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 177429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 178429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray // 2xi64 and i128 suggest an upconverted struct type, which are also unsupported 179429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::Type *V2xi64Ty = llvm::VectorType::get(llvm::Type::getInt64Ty(*Context), 2); 180429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::Type *Int128Ty = llvm::Type::getIntNTy(*Context, 128); 181429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (AllocType == V2xi64Ty || AllocType == Int128Ty) { 182429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 183429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 184429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 185429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return true; 186429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 187429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 1882b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Get the actual value we should use to step through an allocation. 1897b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 1907b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // Normally the value we use to step through an allocation is given to us by 1917b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // the driver. However, for certain primitive data types, we can derive an 1927b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // integer constant for the step value. We use this integer constant whenever 1937b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // possible to allow further compiler optimizations to take place. 1947b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 195b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines // DL - Target Data size/layout information. 1962b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // T - Type of allocation (should be a pointer). 1972b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // OrigStep - Original step increment (root.expand() input from driver). 198bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *AllocType, 1992b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OrigStep) { 200b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines bccAssert(DL); 201bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(AllocType); 2022b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(OrigStep); 203bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType); 204429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (isStepOptSupported(AllocType)) { 2052b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *ET = PT->getElementType(); 206b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines uint64_t ETSize = DL->getTypeAllocSize(ET); 207bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 2082b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return llvm::ConstantInt::get(Int32Ty, ETSize); 2092b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } else { 2102b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return OrigStep; 2112b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 2122b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 2132b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 214097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes /// Builds the types required by the pass for the given context. 215bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes void buildTypes(void) { 216e44a3525b9703739534c3b62d7d1af4c95649a38David Gross // Create the RsLaunchDimensionsTy and RsExpandKernelDriverInfoPfxTy structs. 217bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 218e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *Int8Ty = llvm::Type::getInt8Ty(*Context); 219e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *Int8PtrTy = Int8Ty->getPointerTo(); 220e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *Int8PtrArrayInputLimitTy = llvm::ArrayType::get(Int8PtrTy, RS_KERNEL_INPUT_LIMIT); 221e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 222e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *Int32ArrayInputLimitTy = llvm::ArrayType::get(Int32Ty, RS_KERNEL_INPUT_LIMIT); 223e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 224e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *Int32Array4Ty = llvm::ArrayType::get(Int32Ty, 4); 225097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes 226097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes /* Defined in frameworks/base/libs/rs/cpu_ref/rsCpuCore.h: 227db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 228e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * struct RsLaunchDimensions { 229e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t x; 230db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t y; 231db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t z; 232e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t lod; 233e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t face; 234e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t array[4]; 235e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * }; 236e44a3525b9703739534c3b62d7d1af4c95649a38David Gross */ 237e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::SmallVector<llvm::Type*, RsLaunchDimensionsFieldCount> RsLaunchDimensionsTypes; 238e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t x 239e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t y 240e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t z 241e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t lod 242e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsTypes.push_back(Int32Ty); // uint32_t face 243e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsLaunchDimensionsTypes.push_back(Int32Array4Ty); // uint32_t array[4] 244e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::StructType *RsLaunchDimensionsTy = 245e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::StructType::create(RsLaunchDimensionsTypes, "RsLaunchDimensions"); 246e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 2471d93a190e62ec1588b4724ca8759216b2d0b76d7David Gross /* Defined as the beginning of RsExpandKernelDriverInfo in frameworks/base/libs/rs/cpu_ref/rsCpuCoreRuntime.h: 248e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * 249e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * struct RsExpandKernelDriverInfoPfx { 250e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * const uint8_t *inPtr[RS_KERNEL_INPUT_LIMIT]; 251e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t inStride[RS_KERNEL_INPUT_LIMIT]; 252e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t inLen; 253e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * 254e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint8_t *outPtr[RS_KERNEL_INPUT_LIMIT]; 255e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t outStride[RS_KERNEL_INPUT_LIMIT]; 256e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t outLen; 257e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * 258e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * // Dimension of the launch 259e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * RsLaunchDimensions dim; 260e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * 261e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * // The walking iterator of the launch 262e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * RsLaunchDimensions current; 263e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * 264e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * const void *usr; 265e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * uint32_t usrLen; 266e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * 267e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * // Items below this line are not used by the compiler and can be change in the driver. 268e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * // So the compiler must assume there are an unknown number of fields of unknown type 269e44a3525b9703739534c3b62d7d1af4c95649a38David Gross * // beginning here. 270db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * }; 2711d93a190e62ec1588b4724ca8759216b2d0b76d7David Gross * 2721d93a190e62ec1588b4724ca8759216b2d0b76d7David Gross * The name "RsExpandKernelDriverInfoPfx" is known to RSInvariantPass (RSInvariant.cpp). 273db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 274e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::SmallVector<llvm::Type*, RsExpandKernelDriverInfoPfxFieldCount> RsExpandKernelDriverInfoPfxTypes; 275e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(Int8PtrArrayInputLimitTy); // const uint8_t *inPtr[RS_KERNEL_INPUT_LIMIT] 276e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(Int32ArrayInputLimitTy); // uint32_t inStride[RS_KERNEL_INPUT_LIMIT] 277e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(Int32Ty); // uint32_t inLen 278e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(Int8PtrArrayInputLimitTy); // uint8_t *outPtr[RS_KERNEL_INPUT_LIMIT] 279e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(Int32ArrayInputLimitTy); // uint32_t outStride[RS_KERNEL_INPUT_LIMIT] 280e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(Int32Ty); // uint32_t outLen 281e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(RsLaunchDimensionsTy); // RsLaunchDimensions dim 282e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(RsLaunchDimensionsTy); // RsLaunchDimensions current 283e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(VoidPtrTy); // const void *usr 284e44a3525b9703739534c3b62d7d1af4c95649a38David Gross RsExpandKernelDriverInfoPfxTypes.push_back(Int32Ty); // uint32_t usrLen 285e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::StructType *RsExpandKernelDriverInfoPfxTy = 286e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::StructType::create(RsExpandKernelDriverInfoPfxTypes, "RsExpandKernelDriverInfoPfx"); 287bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 288bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes // Create the function type for expanded kernels. 289bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 290e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Type *RsExpandKernelDriverInfoPfxPtrTy = RsExpandKernelDriverInfoPfxTy->getPointerTo(); 291bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 292bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::SmallVector<llvm::Type*, 8> ParamTypes; 293e44a3525b9703739534c3b62d7d1af4c95649a38David Gross ParamTypes.push_back(RsExpandKernelDriverInfoPfxPtrTy); // const RsExpandKernelDriverInfoPfx *p 294e44a3525b9703739534c3b62d7d1af4c95649a38David Gross ParamTypes.push_back(Int32Ty); // uint32_t x1 295e44a3525b9703739534c3b62d7d1af4c95649a38David Gross ParamTypes.push_back(Int32Ty); // uint32_t x2 296e44a3525b9703739534c3b62d7d1af4c95649a38David Gross ParamTypes.push_back(Int32Ty); // uint32_t outstep 297bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 298e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes ExpandedFunctionType = 299e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), ParamTypes, 300e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes false); 3018ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3028ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 303357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// @brief Create skeleton of the expanded function. 304357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 305357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// This creates a function with the following signature: 306357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 307357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 3085010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes /// uint32_t outstep) 309357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 310357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) { 311bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 312bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::Create(ExpandedFunctionType, 313bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::GlobalValue::ExternalLinkage, 314bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes OldName + ".expand", Module); 315bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 316bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 317bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 318bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator AI = ExpandedFunction->arg_begin(); 319bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 320bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("p"); 321bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("x1"); 322bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("x2"); 323bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("arg_outstep"); 324bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 325bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin", 326bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction); 327806075b3a54af826fea78490fb213d8a0784138eTobias Grosser llvm::IRBuilder<> Builder(Begin); 328806075b3a54af826fea78490fb213d8a0784138eTobias Grosser Builder.CreateRetVoid(); 329806075b3a54af826fea78490fb213d8a0784138eTobias Grosser 330bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes return ExpandedFunction; 331357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser } 332357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 333e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @brief Create an empty loop 334e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 335e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// Create a loop of the form: 336e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 337e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// for (i = LowerBound; i < UpperBound; i++) 338e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// ; 339e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 340e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// After the loop has been created, the builder is set such that 341e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// instructions can be added to the loop body. 342e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 343e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param Builder The builder to use to build this loop. The current 344e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// position of the builder is the position the loop 345e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// will be inserted. 346e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LowerBound The first value of the loop iterator 347e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param UpperBound The maximal value of the loop iterator 348e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LoopIV A reference that will be set to the loop iterator. 349e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @return The BasicBlock that will be executed after the loop. 350e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder, 351e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *LowerBound, 352e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *UpperBound, 353e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode **LoopIV) { 354c2ca742d7d0197c52e49467862844463fb42280fDavid Gross bccAssert(LowerBound->getType() == UpperBound->getType()); 355e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 356e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB; 357e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *Cond, *IVNext; 358e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 359e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 360e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB = Builder.GetInsertBlock(); 3611bd9f627fa0affb457507e86b0b6684c695fe726Stephen Hines AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), nullptr, nullptr); 362bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes HeaderBB = llvm::BasicBlock::Create(*Context, "Loop", CondBB->getParent()); 363e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 364e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (LowerBound < Upperbound) 365e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 366e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 367e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 368e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB->getTerminator()->eraseFromParent(); 369e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(CondBB); 370e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(LowerBound, UpperBound); 371e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 372e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 373e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ] 374e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv.next = iv + 1 375e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (iv.next < Upperbound) 376e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 377e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 378e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 379e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB); 380e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV = Builder.CreatePHI(LowerBound->getType(), 2, "X"); 381e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(LowerBound, CondBB); 382e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1)); 383e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(IVNext, HeaderBB); 384e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(IVNext, UpperBound); 385e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 386e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB->setName("Exit"); 387e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB->getFirstNonPHI()); 388e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser *LoopIV = IV; 389e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser return AfterBB; 390e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser } 391e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 39228c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // Finish building the outgoing argument list for calling a ForEach-able function. 39328c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // 39428c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // ArgVector - on input, the non-special arguments 39528c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // on output, the non-special arguments combined with the special arguments 39628c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // from SpecialArgVector 39728c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // SpecialArgVector - special arguments (from ExpandSpecialArguments()) 39828c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // SpecialArgContextIdx - return value of ExpandSpecialArguments() 39928c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // (position of context argument in SpecialArgVector) 40028c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // CalleeFunction - the ForEach-able function being called 40128c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // Builder - for inserting code into the caller function 40228c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross template<unsigned int ArgVectorLen, unsigned int SpecialArgVectorLen> 40328c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross void finishArgList( llvm::SmallVector<llvm::Value *, ArgVectorLen> &ArgVector, 40428c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross const llvm::SmallVector<llvm::Value *, SpecialArgVectorLen> &SpecialArgVector, 40528c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross const int SpecialArgContextIdx, 40628c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross const llvm::Function &CalleeFunction, 40728c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross llvm::IRBuilder<> &CallerBuilder) { 40828c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross /* The context argument (if any) is a pointer to an opaque user-visible type that differs from 40928c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross * the RsExpandKernelDriverInfoPfx type used in the function we are generating (although the 41028c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross * two types represent the same thing). Therefore, we must introduce a pointer cast when 41128c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross * generating a call to the kernel function. 41228c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross */ 41328c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross const int ArgContextIdx = 41428c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross SpecialArgContextIdx >= 0 ? (ArgVector.size() + SpecialArgContextIdx) : SpecialArgContextIdx; 41528c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross ArgVector.append(SpecialArgVector.begin(), SpecialArgVector.end()); 41628c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross if (ArgContextIdx >= 0) { 41728c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross llvm::Type *ContextArgType = nullptr; 41828c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross int ArgIdx = ArgContextIdx; 41928c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross for (const auto &Arg : CalleeFunction.getArgumentList()) { 42028c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross if (!ArgIdx--) { 42128c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross ContextArgType = Arg.getType(); 42228c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross break; 42328c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross } 42428c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross } 42528c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross bccAssert(ContextArgType); 42628c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross ArgVector[ArgContextIdx] = CallerBuilder.CreatePointerCast(ArgVector[ArgContextIdx], ContextArgType); 42728c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross } 42828c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross } 42928c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross 4308ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosserpublic: 43133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross RSForEachExpandPass(bool pEnableStepOpt = true) 432900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes : ModulePass(ID), Module(nullptr), Context(nullptr), 433bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes mEnableStepOpt(pEnableStepOpt) { 434bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 4358ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 4368ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 437c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override { 438c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines // This pass does not use any other analysis passes, but it does 439c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines // add/wrap the existing functions in the module (thus altering the CFG). 440c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines } 441c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines 44233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // Build contribution to outgoing argument list for calling a 44333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // ForEach-able function, based on the special parameters of that 44433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // function. 44533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // 44633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // Signature - metadata bits for the signature of the ForEach-able function 44733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // X, Arg_p - values derived directly from expanded function, 44833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // suitable for computing arguments for the ForEach-able function 44933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // CalleeArgs - contribution is accumulated here 45033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // Bump - invoked once for each contributed outgoing argument 45128c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // 45228c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // Return value is the (zero-based) position of the context (Arg_p) 45328c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // argument in the CalleeArgs vector, or a negative value if the 45428c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross // context argument is not placed in the CalleeArgs vector. 45528c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross int ExpandSpecialArguments(uint32_t Signature, 45628c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross llvm::Value *X, 45728c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross llvm::Value *Arg_p, 45828c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross llvm::IRBuilder<> &Builder, 45928c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross llvm::SmallVector<llvm::Value*, 8> &CalleeArgs, 46028c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross std::function<void ()> Bump) { 46128c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross 46228c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross bccAssert(CalleeArgs.empty()); 46328c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross 46428c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross int Return = -1; 46533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross if (bcinfo::MetadataExtractor::hasForEachSignatureCtxt(Signature)) { 46633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross CalleeArgs.push_back(Arg_p); 46733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Bump(); 46828c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross Return = CalleeArgs.size() - 1; 46933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 47033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 47133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 47233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross CalleeArgs.push_back(X); 47333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Bump(); 47433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 47533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 476e44a3525b9703739534c3b62d7d1af4c95649a38David Gross if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature) || 477e44a3525b9703739534c3b62d7d1af4c95649a38David Gross bcinfo::MetadataExtractor::hasForEachSignatureZ(Signature)) { 47833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 47998137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *Current = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldCurrent); 480e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 481e44a3525b9703739534c3b62d7d1af4c95649a38David Gross if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 482e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Value *Y = Builder.CreateLoad( 48398137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar Builder.CreateStructGEP(nullptr, Current, RsLaunchDimensionsFieldY), "Y"); 48498137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar 485e44a3525b9703739534c3b62d7d1af4c95649a38David Gross CalleeArgs.push_back(Y); 486e44a3525b9703739534c3b62d7d1af4c95649a38David Gross Bump(); 487e44a3525b9703739534c3b62d7d1af4c95649a38David Gross } 488e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 489e44a3525b9703739534c3b62d7d1af4c95649a38David Gross if (bcinfo::MetadataExtractor::hasForEachSignatureZ(Signature)) { 490e44a3525b9703739534c3b62d7d1af4c95649a38David Gross llvm::Value *Z = Builder.CreateLoad( 49198137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar Builder.CreateStructGEP(nullptr, Current, RsLaunchDimensionsFieldZ), "Z"); 492e44a3525b9703739534c3b62d7d1af4c95649a38David Gross CalleeArgs.push_back(Z); 493e44a3525b9703739534c3b62d7d1af4c95649a38David Gross Bump(); 494e44a3525b9703739534c3b62d7d1af4c95649a38David Gross } 49533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 49628c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross 49728c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross return Return; 49833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 49933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 5008ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /* Performs the actual optimization on a selected function. On success, the 5018ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * Module will contain a new function of the name "<NAME>.expand" that 5028ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * invokes <NAME>() in a loop with the appropriate parameters. 5038ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser */ 504bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool ExpandFunction(llvm::Function *Function, uint32_t Signature) { 505bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ALOGV("Expanding ForEach-able Function %s", 506bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Function->getName().str().c_str()); 5078ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 5088ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 509bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Signature = getRootSignature(Function); 5108ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 5118ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // We couldn't determine how to expand this function based on its 5128ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // function signature. 5138ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser return false; 5148ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 5158ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 5168ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 517bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::DataLayout DL(Module); 5188ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 519bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 520bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes createEmptyExpandedFunction(Function->getName()); 521db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 522bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 523bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Extract the expanded function's parameters. It is guaranteed by 524bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * createEmptyExpandedFunction that there will be five parameters. 525bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 52633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 52733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 52833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 529bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator ExpandedFunctionArgIter = 530bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction->arg_begin(); 531db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 532bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 533bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 534bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 5355010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter); 536bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 537900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *InStep = nullptr; 538900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutStep = nullptr; 539db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 540db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Construct the actual function body. 541bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 542db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 543cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Collect and construct the arguments for the kernel(). 544db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 545bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator FunctionArgIter = Function->arg_begin(); 546db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 547900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Type *InTy = nullptr; 548900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *InBasePtr = nullptr; 549d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) { 55098137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InsBasePtr = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInPtr, "inputs_base"); 551e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 55298137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InStepsBase = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInStride, "insteps_base"); 553e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 55498137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InStepAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InStepsBase, 0, 0); 555e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::LoadInst *InStepArg = Builder.CreateLoad(InStepAddr, 556e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes "instep_addr"); 557e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 558bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes InTy = (FunctionArgIter++)->getType(); 559e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes InStep = getStepValue(&DL, InTy, InStepArg); 560e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 5612b04086acbef6520ae2c54a868b1271abf053122Stephen Hines InStep->setName("instep"); 562e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 56398137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InputAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InsBasePtr, 0, 0); 564e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes InBasePtr = Builder.CreateLoad(InputAddr, "input_base"); 565db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 566db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 567900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Type *OutTy = nullptr; 568900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutBasePtr = nullptr; 569d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 570bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes OutTy = (FunctionArgIter++)->getType(); 571b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 5722b04086acbef6520ae2c54a868b1271abf053122Stephen Hines OutStep->setName("outstep"); 573097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes OutBasePtr = Builder.CreateLoad( 57498137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar Builder.CreateConstInBoundsGEP2_32(nullptr, 57598137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldOutPtr), 57698137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar 0, 0)); 577db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 578db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 579900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *UsrData = nullptr; 580d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)) { 581bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *UsrDataTy = (FunctionArgIter++)->getType(); 582db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData = Builder.CreatePointerCast(Builder.CreateLoad( 58398137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldUsr)), UsrDataTy); 584db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData->setName("UsrData"); 585db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 586db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 58733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::PHINode *IV; 58833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross createLoop(Builder, Arg_x1, Arg_x2, &IV); 589097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes 59033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::SmallVector<llvm::Value*, 8> CalleeArgs; 59128c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross const int CalleeArgsContextIdx = ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs, 59228c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross [&FunctionArgIter]() { FunctionArgIter++; }); 593db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 594bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(FunctionArgIter == Function->arg_end()); 595db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 596cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Populate the actual call to kernel(). 597db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 598db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 599900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *InPtr = nullptr; 600900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutPtr = nullptr; 601db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 602ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // Calculate the current input and output pointers 60302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 604ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // We always calculate the input/output pointers with a GEP operating on i8 60502f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // values and only cast at the very end to OutTy. This is because the step 60602f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // between two values is given in bytes. 60702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 60802f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // TODO: We could further optimize the output by using a GEP operation of 60902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // type 'OutTy' in cases where the element type of the allocation allows. 61002f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutBasePtr) { 61102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 61202f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutOffset = Builder.CreateMul(OutOffset, OutStep); 61302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 61402f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 61502f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser } 616bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 617ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InBasePtr) { 618ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1); 619ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InOffset = Builder.CreateMul(InOffset, InStep); 620ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreateGEP(InBasePtr, InOffset); 621ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreatePointerCast(InPtr, InTy); 622ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser } 62302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser 624ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InPtr) { 6257ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(InPtr); 626db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 627db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 62802f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutPtr) { 6297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(OutPtr); 630db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 631db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 632db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (UsrData) { 633db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(UsrData); 634db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 635db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 63628c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross finishArgList(RootArgs, CalleeArgs, CalleeArgsContextIdx, *Function, Builder); 637db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 638bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Builder.CreateCall(Function, RootArgs); 639db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6407ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return true; 6417ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 6427ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 6437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Expand a pass-by-value kernel. 6447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 645bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool ExpandKernel(llvm::Function *Function, uint32_t Signature) { 646d88177580db4ddedf680854c51db333c97eabc59Stephen Hines bccAssert(bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)); 647bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ALOGV("Expanding kernel Function %s", Function->getName().str().c_str()); 6487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 6497ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // TODO: Refactor this to share functionality with ExpandFunction. 650bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::DataLayout DL(Module); 6517ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 652bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 653bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes createEmptyExpandedFunction(Function->getName()); 6547ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 655bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 656bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Extract the expanded function's parameters. It is guaranteed by 657bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * createEmptyExpandedFunction that there will be five parameters. 658bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 659881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 660881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 661881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 662bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator ExpandedFunctionArgIter = 663bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction->arg_begin(); 664bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 665bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 666bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 667bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 6685010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter); 6697ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 6707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Construct the actual function body. 671bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 6727ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 67318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Create TBAA meta-data. 674354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines llvm::MDNode *TBAARenderScriptDistinct, *TBAARenderScript, 675354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines *TBAAAllocation, *TBAAPointer; 676bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDBuilder MDHelper(*Context); 67714588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien 678354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines TBAARenderScriptDistinct = 679354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines MDHelper.createTBAARoot("RenderScript Distinct TBAA"); 680354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines TBAARenderScript = MDHelper.createTBAANode("RenderScript TBAA", 681354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines TBAARenderScriptDistinct); 682e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAAllocation = MDHelper.createTBAAScalarTypeNode("allocation", 683e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAARenderScript); 684e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAAllocation = MDHelper.createTBAAStructTagNode(TBAAAllocation, 685e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAAllocation, 0); 686e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAPointer = MDHelper.createTBAAScalarTypeNode("pointer", 687e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAARenderScript); 68814588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien TBAAPointer = MDHelper.createTBAAStructTagNode(TBAAPointer, TBAAPointer, 0); 68918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 69050f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray llvm::MDNode *AliasingDomain, *AliasingScope; 69150f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray AliasingDomain = MDHelper.createAnonymousAliasScopeDomain("RS argument scope domain"); 69250f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray AliasingScope = MDHelper.createAnonymousAliasScope(AliasingDomain, "RS argument scope"); 69350f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 694881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes /* 695881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * Collect and construct the arguments for the kernel(). 696881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * 697881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * Note that we load any loop-invariant arguments before entering the Loop. 698881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes */ 699881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes size_t NumInputs = Function->arg_size(); 7007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 701881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // No usrData parameter on kernels. 702881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes bccAssert( 703881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes !bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)); 704881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 705881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Function::arg_iterator ArgIter = Function->arg_begin(); 706881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 707881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Check the return type 708bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Type *OutTy = nullptr; 709bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *OutStep = nullptr; 710bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::LoadInst *OutBasePtr = nullptr; 711bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *CastedOutBasePtr = nullptr; 712881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 713e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes bool PassOutByPointer = false; 714881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 715d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 716bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *OutBaseTy = Function->getReturnType(); 717881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 71874a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (OutBaseTy->isVoidTy()) { 719e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes PassOutByPointer = true; 720881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutTy = ArgIter->getType(); 721881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 722881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes ArgIter++; 723881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes --NumInputs; 72474a4b08235990916911b8fe758d656c1171faf26Stephen Hines } else { 72574a4b08235990916911b8fe758d656c1171faf26Stephen Hines // We don't increment Args, since we are using the actual return type. 726881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutTy = OutBaseTy->getPointerTo(); 72774a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 728881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 729b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 73074a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutStep->setName("outstep"); 731097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes OutBasePtr = Builder.CreateLoad( 73298137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar Builder.CreateConstInBoundsGEP2_32(nullptr, 73398137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldOutPtr), 73498137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar 0, 0)); 735097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes 7369c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 7379c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines OutBasePtr->setMetadata("tbaa", TBAAPointer); 7389c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 73950f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 74050f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray OutBasePtr->setMetadata("alias.scope", AliasingScope); 74150f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 742bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray CastedOutBasePtr = Builder.CreatePointerCast(OutBasePtr, OutTy, "casted_out"); 74374a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 74474a4b08235990916911b8fe758d656c1171faf26Stephen Hines 74533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::PHINode *IV; 74633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross createLoop(Builder, Arg_x1, Arg_x2, &IV); 74733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 74833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::SmallVector<llvm::Value*, 8> CalleeArgs; 74928c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross const int CalleeArgsContextIdx = ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs, 75028c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross [&NumInputs]() { --NumInputs; }); 75133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 752bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<llvm::Type*, 8> InTypes; 753bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<llvm::Value*, 8> InSteps; 754bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<llvm::Value*, 8> InBasePtrs; 755bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<bool, 8> InIsStructPointer; 756881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 757e44a3525b9703739534c3b62d7d1af4c95649a38David Gross bccAssert(NumInputs <= RS_KERNEL_INPUT_LIMIT); 758e44a3525b9703739534c3b62d7d1af4c95649a38David Gross 759e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes if (NumInputs > 0) { 76098137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InsBasePtr = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInPtr, "inputs_base"); 7617ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 76298137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InStepsBase = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldInStride, "insteps_base"); 763881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 764881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes for (size_t InputIndex = 0; InputIndex < NumInputs; 765881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes ++InputIndex, ArgIter++) { 766881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 76798137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InStepAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InStepsBase, 0, InputIndex); 768881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InStepArg = Builder.CreateLoad(InStepAddr, 769881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "instep_addr"); 770881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 771326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes llvm::Type *InType = ArgIter->getType(); 772326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 773326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes /* 774326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * AArch64 calling dictate that structs of sufficient size get passed by 775326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * pointer instead of passed by value. This, combined with the fact 776326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * that we don't allow kernels to operate on pointer data means that if 777326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * we see a kernel with a pointer parameter we know that it is struct 778326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * input that has been promoted. As such we don't need to convert its 779326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * type to a pointer. Later we will need to know to avoid a load, so we 780326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * save this information in InIsStructPointer. 781326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes */ 782326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes if (!InType->isPointerTy()) { 783326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InType = InType->getPointerTo(); 784326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InIsStructPointer.push_back(false); 785326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } else { 786326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InIsStructPointer.push_back(true); 787326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } 788326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 789881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InStep = getStepValue(&DL, InType, InStepArg); 790881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 791881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InStep->setName("instep"); 792881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 79398137cca7eebca946b869b010fef2821c9bf4971Pirama Arumuga Nainar llvm::Value *InputAddr = Builder.CreateConstInBoundsGEP2_32(nullptr, InsBasePtr, 0, InputIndex); 794881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InBasePtr = Builder.CreateLoad(InputAddr, 795881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "input_base"); 796bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *CastInBasePtr = Builder.CreatePointerCast(InBasePtr, 797bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray InType, "casted_in"); 798881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (gEnableRsTbaa) { 799881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InBasePtr->setMetadata("tbaa", TBAAPointer); 800881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 801881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 80250f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray InBasePtr->setMetadata("alias.scope", AliasingScope); 80350f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 804881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InTypes.push_back(InType); 805881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InSteps.push_back(InStep); 806bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray InBasePtrs.push_back(CastInBasePtr); 807881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 808881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 8097ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 8107ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Populate the actual call to kernel(). 8117ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 8127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 8134102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser // Calculate the current input and output pointers 8147b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 8157b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 816881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // We always calculate the input/output pointers with a GEP operating on i8 817881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // values combined with a multiplication and only cast at the very end to 818881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // OutTy. This is to account for dynamic stepping sizes when the value 819881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // isn't apparent at compile time. In the (very common) case when we know 820881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // the step size at compile time, due to haveing complete type information 821881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // this multiplication will optmized out and produces code equivalent to a 822881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // a GEP on a pointer of the correct type. 823881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 824881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Output 825881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 826900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutPtr = nullptr; 827bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray if (CastedOutBasePtr) { 8287b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 829881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 830bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray OutPtr = Builder.CreateGEP(CastedOutBasePtr, OutOffset); 831bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 832e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes if (PassOutByPointer) { 833881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes RootArgs.push_back(OutPtr); 834881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 8354102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser } 8367b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser 837881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Inputs 83874a4b08235990916911b8fe758d656c1171faf26Stephen Hines 839881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (NumInputs > 0) { 840881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *Offset = Builder.CreateSub(IV, Arg_x1); 841881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 842881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes for (size_t Index = 0; Index < NumInputs; ++Index) { 843bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *InPtr = Builder.CreateGEP(InBasePtrs[Index], Offset); 844326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes llvm::Value *Input; 845326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 846326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes if (InIsStructPointer[Index]) { 847326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes Input = InPtr; 848326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 849326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } else { 850326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes llvm::LoadInst *InputLoad = Builder.CreateLoad(InPtr, "input"); 851326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 852326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes if (gEnableRsTbaa) { 853326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InputLoad->setMetadata("tbaa", TBAAAllocation); 854326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } 855881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 85650f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray InputLoad->setMetadata("alias.scope", AliasingScope); 85750f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 858326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes Input = InputLoad; 859881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 860881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 861881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes RootArgs.push_back(Input); 8629c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 8637ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 8647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 86528c17995669687cecc96eaba232eecd3b65f5bbfDavid Gross finishArgList(RootArgs, CalleeArgs, CalleeArgsContextIdx, *Function, Builder); 8667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 867bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *RetVal = Builder.CreateCall(Function, RootArgs); 8687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 869e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes if (OutPtr && !PassOutByPointer) { 87018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr); 8719c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 8729c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines Store->setMetadata("tbaa", TBAAAllocation); 8739c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 87450f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray Store->setMetadata("alias.scope", AliasingScope); 8757ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 8767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 877db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return true; 878db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 879db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 88018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Checks if pointers to allocation internals are exposed 88118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 88218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// This function verifies if through the parameters passed to the kernel 88318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// or through calls to the runtime library the script gains access to 88418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers pointing to data within a RenderScript Allocation. 88518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// If we know we control all loads from and stores to data within 88618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// RenderScript allocations and if we know the run-time internal accesses 88718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// are all annotated with RenderScript TBAA metadata, only then we 88818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// can safely use TBAA to distinguish between generic and from-allocation 88918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers. 890bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool allocPointersExposed(llvm::Module &Module) { 89118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Old style kernel function can expose pointers to elements within 89218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // allocations. 89318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // TODO: Extend analysis to allow simple cases of old-style kernels. 89425eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines for (size_t i = 0; i < mExportForEachCount; ++i) { 89525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char *Name = mExportForEachNameList[i]; 89625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t Signature = mExportForEachSignatureList[i]; 897bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (Module.getFunction(Name) && 898d88177580db4ddedf680854c51db333c97eabc59Stephen Hines !bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)) { 89918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 90018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 90118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 90218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 90318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Check for library functions that expose a pointer to an Allocation or 90418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // that are not yet annotated with RenderScript-specific tbaa information. 905e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala static const std::vector<const char *> Funcs{ 906e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala // rsGetElementAt(...) 907e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z14rsGetElementAt13rs_allocationj", 908e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z14rsGetElementAt13rs_allocationjj", 909e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z14rsGetElementAt13rs_allocationjjj", 910e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala 911e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala // rsSetElementAt() 912e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z14rsSetElementAt13rs_allocationPvj", 913e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z14rsSetElementAt13rs_allocationPvjj", 914e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z14rsSetElementAt13rs_allocationPvjjj", 915e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala 916e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala // rsGetElementAtYuv_uchar_Y() 917e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z25rsGetElementAtYuv_uchar_Y13rs_allocationjj", 918e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala 919e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala // rsGetElementAtYuv_uchar_U() 920e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z25rsGetElementAtYuv_uchar_U13rs_allocationjj", 921e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala 922e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala // rsGetElementAtYuv_uchar_V() 923e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala "_Z25rsGetElementAtYuv_uchar_V13rs_allocationjj", 924e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala }; 925e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala 926e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala for (auto FI : Funcs) { 927e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala llvm::Function *Function = Module.getFunction(FI); 92818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 929bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (!Function) { 930e2423780bf482d2eeefa9a958b4793f865cba845Matt Wala ALOGE("Missing run-time function '%s'", FI); 93118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 93218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 93318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 934bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (Function->getNumUses() > 0) { 93518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 93618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 93718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 93818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 93918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return false; 94018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 94118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 94218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Connect RenderScript TBAA metadata to C/C++ metadata 94318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 94418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// The TBAA metadata used to annotate loads/stores from RenderScript 945e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// Allocations is generated in a separate TBAA tree with a 946354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines /// "RenderScript Distinct TBAA" root node. LLVM does assume may-alias for 947354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines /// all nodes in unrelated alias analysis trees. This function makes the 948354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines /// "RenderScript TBAA" node (which is parented by the Distinct TBAA root), 949e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// a subtree of the normal C/C++ TBAA tree aside of normal C/C++ types. With 950e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// the connected trees every access to an Allocation is resolved to 951e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// must-alias if compared to a normal C/C++ access. 952bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes void connectRenderScriptTBAAMetadata(llvm::Module &Module) { 953bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDBuilder MDHelper(*Context); 954354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines llvm::MDNode *TBAARenderScriptDistinct = 955354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines MDHelper.createTBAARoot("RenderScript Distinct TBAA"); 956354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines llvm::MDNode *TBAARenderScript = MDHelper.createTBAANode( 957354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines "RenderScript TBAA", TBAARenderScriptDistinct); 958bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDNode *TBAARoot = MDHelper.createTBAARoot("Simple C/C++ TBAA"); 959354d1c132ad7e1ff6fdb0da95443245848a0601fStephen Hines TBAARenderScript->replaceOperandWith(1, TBAARoot); 96018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 96118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 962bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes virtual bool runOnModule(llvm::Module &Module) { 963bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool Changed = false; 964bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->Module = &Module; 965bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->Context = &Module.getContext(); 966bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 967bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->buildTypes(); 968bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 969bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bcinfo::MetadataExtractor me(&Module); 97025eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines if (!me.extract()) { 97125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines ALOGE("Could not extract metadata from module!"); 97225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines return false; 97325eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines } 97425eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachCount = me.getExportForEachSignatureCount(); 97525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachNameList = me.getExportForEachNameList(); 97625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachSignatureList = me.getExportForEachSignatureList(); 977db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 978bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool AllocsExposed = allocPointersExposed(Module); 97918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 98025eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines for (size_t i = 0; i < mExportForEachCount; ++i) { 98125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char *name = mExportForEachNameList[i]; 98225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t signature = mExportForEachSignatureList[i]; 983bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *kernel = Module.getFunction(name); 984cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser if (kernel) { 985d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureKernel(signature)) { 986cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandKernel(kernel, signature); 987acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 988acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else if (kernel->getReturnType()->isVoidTy()) { 989cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandFunction(kernel, signature); 990acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 991acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else { 992acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // There are some graphics root functions that are not 993acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // expanded, but that will be called directly. For those 994acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // functions, we can not set the linkage to internal. 995acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } 996cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines } 997db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 998db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 9999c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa && !AllocsExposed) { 1000bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes connectRenderScriptTBAAMetadata(Module); 100118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 100218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 1003cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines return Changed; 1004db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 1005db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1006db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual const char *getPassName() const { 1007db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return "ForEach-able Function Expansion"; 1008db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 1009db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 10107a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}; // end RSForEachExpandPass 1011db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 10127a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end anonymous namespace 10137a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 10147a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaochar RSForEachExpandPass::ID = 0; 101533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grossstatic llvm::RegisterPass<RSForEachExpandPass> X("foreachexp", "ForEach Expand Pass"); 1016db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1017db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace bcc { 1018db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 10197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaollvm::ModulePass * 102025eb586bb055ae07c7e82a2b1bdbd6936641580cStephen HinescreateRSForEachExpandPass(bool pEnableStepOpt){ 102125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines return new RSForEachExpandPass(pEnableStepOpt); 10227a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} 1023db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 10247a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end namespace bcc 1025