RSKernelExpand.cpp revision 33cda5cf335afc6aa2dbe02062bc9e6649e1f87c
1db169187dea4602e4ad32058762d23d474753fd0Stephen Hines/* 2db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Copyright 2012, The Android Open Source Project 3db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 4db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Licensed under the Apache License, Version 2.0 (the "License"); 5db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * you may not use this file except in compliance with the License. 6db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * You may obtain a copy of the License at 7db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 8db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * http://www.apache.org/licenses/LICENSE-2.0 9db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 10db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Unless required by applicable law or agreed to in writing, software 11db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * distributed under the License is distributed on an "AS IS" BASIS, 12db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * See the License for the specific language governing permissions and 14db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * limitations under the License. 15db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 16db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 176e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines#include "bcc/Assert.h" 18e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSTransforms.h" 197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao#include <cstdlib> 2133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross#include <functional> 227a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 23b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DerivedTypes.h> 24b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Function.h> 25b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Instructions.h> 26b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/IRBuilder.h> 2718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser#include <llvm/IR/MDBuilder.h> 28b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Module.h> 29c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Pass.h> 307ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines#include <llvm/Support/raw_ostream.h> 31b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DataLayout.h> 32cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser#include <llvm/IR/Function.h> 33b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Type.h> 34806075b3a54af826fea78490fb213d8a0784138eTobias Grosser#include <llvm/Transforms/Utils/BasicBlockUtils.h> 35c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang 36c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include "bcc/Config/Config.h" 37ef73a242762bcd8113b9b65ceccbe7d909b5acbcZonr Chang#include "bcc/Support/Log.h" 38db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 39d88177580db4ddedf680854c51db333c97eabc59Stephen Hines#include "bcinfo/MetadataExtractor.h" 40d88177580db4ddedf680854c51db333c97eabc59Stephen Hines 415010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes#define NUM_EXPANDED_FUNCTION_PARAMS 4 42bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 437a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaousing namespace bcc; 447a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 45db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace { 467a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 4714588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chienstatic const bool gEnableRsTbaa = true; 489c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines 497a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao/* RSForEachExpandPass - This pass operates on functions that are able to be 507a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the 517a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * ForEach-able function to be invoked over the appropriate data cells of the 527a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * input/output allocations (adjusting other relevant parameters as we go). We 537a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * support doing this for any ForEach-able compute kernels. The new function 547a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * name is the original function name followed by ".expand". Note that we 557a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * still generate code for the original function. 567a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao */ 577a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoclass RSForEachExpandPass : public llvm::ModulePass { 5833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grosspublic: 59db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static char ID; 60db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grossprivate: 6233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 63bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Module *Module; 64bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::LLVMContext *Context; 65bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 66bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 67bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Pointer to LLVM type information for the ForEachStubType and the function 68bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * signature for expanded kernels. These must be re-calculated for each 69bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * module the pass is run on. 70bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 71bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::StructType *ForEachStubType; 72bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::FunctionType *ExpandedFunctionType; 73db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7425eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t mExportForEachCount; 7525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char **mExportForEachNameList; 7625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const uint32_t *mExportForEachSignatureList; 77cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines 782b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Turns on optimization of allocation stride values. 792b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool mEnableStepOpt; 802b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 81bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes uint32_t getRootSignature(llvm::Function *Function) { 82db169187dea4602e4ad32058762d23d474753fd0Stephen Hines const llvm::NamedMDNode *ExportForEachMetadata = 83bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Module->getNamedMetadata("#rs_export_foreach"); 84db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 85db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (!ExportForEachMetadata) { 86db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 8> RootArgTys; 87bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes for (llvm::Function::arg_iterator B = Function->arg_begin(), 88bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes E = Function->arg_end(); 89db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 90db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 91db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgTys.push_back(B->getType()); 92db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 93db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 94db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // For pre-ICS bitcode, we may not have signature information. In that 95db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // case, we use the size of the RootArgTys to select the number of 96db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // arguments. 97db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return (1 << RootArgTys.size()) - 1; 98db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 99db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ExportForEachMetadata->getNumOperands() == 0) { 1017ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return 0; 1027ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 1037ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 1046e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(ExportForEachMetadata->getNumOperands() > 0); 105db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 106cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // We only handle the case for legacy root() functions here, so this is 107cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // hard-coded to look at only the first such function. 108db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0); 109900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes if (SigNode != nullptr && SigNode->getNumOperands() == 1) { 110db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *SigVal = SigNode->getOperand(0); 111db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigVal->getValueID() == llvm::Value::MDStringVal) { 112db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::StringRef SigString = 113db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static_cast<llvm::MDString*>(SigVal)->getString(); 114db169187dea4602e4ad32058762d23d474753fd0Stephen Hines uint32_t Signature = 0; 115db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigString.getAsInteger(10, Signature)) { 116db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ALOGE("Non-integer signature value '%s'", SigString.str().c_str()); 117db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 118db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 119db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return Signature; 120db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 121db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 122db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 123db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 124db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 125db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 126429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray bool isStepOptSupported(llvm::Type *AllocType) { 127429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 128429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType); 129429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 130429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 131429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (mEnableStepOpt) { 132429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 133429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 134429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 135429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (AllocType == VoidPtrTy) { 136429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 137429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 138429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 139429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (!PT) { 140429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 141429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 142429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 143429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray // remaining conditions are 64-bit only 144429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (VoidPtrTy->getPrimitiveSizeInBits() == 32) { 145429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return true; 146429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 147429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 148429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray // coerce suggests an upconverted struct type, which we can't support 149429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (AllocType->getStructName().find("coerce") != llvm::StringRef::npos) { 150429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 151429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 152429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 153429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray // 2xi64 and i128 suggest an upconverted struct type, which are also unsupported 154429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::Type *V2xi64Ty = llvm::VectorType::get(llvm::Type::getInt64Ty(*Context), 2); 155429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray llvm::Type *Int128Ty = llvm::Type::getIntNTy(*Context, 128); 156429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (AllocType == V2xi64Ty || AllocType == Int128Ty) { 157429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return false; 158429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 159429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 160429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray return true; 161429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray } 162429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray 1632b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Get the actual value we should use to step through an allocation. 1647b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 1657b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // Normally the value we use to step through an allocation is given to us by 1667b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // the driver. However, for certain primitive data types, we can derive an 1677b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // integer constant for the step value. We use this integer constant whenever 1687b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // possible to allow further compiler optimizations to take place. 1697b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 170b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines // DL - Target Data size/layout information. 1712b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // T - Type of allocation (should be a pointer). 1722b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // OrigStep - Original step increment (root.expand() input from driver). 173bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *AllocType, 1742b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OrigStep) { 175b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines bccAssert(DL); 176bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(AllocType); 1772b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(OrigStep); 178bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType); 179429d94acbc64480d4f75233b66b4824cdc8b19c0Tim Murray if (isStepOptSupported(AllocType)) { 1802b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *ET = PT->getElementType(); 181b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines uint64_t ETSize = DL->getTypeAllocSize(ET); 182bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 1832b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return llvm::ConstantInt::get(Int32Ty, ETSize); 1842b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } else { 1852b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return OrigStep; 1862b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1872b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1882b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 189e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_INS 0 190e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_INESTRIDES 1 191e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_OUT 2 192e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_Y 3 193e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_Z 4 194e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_LID 5 195e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_USR 6 196e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_DIMX 7 197e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_DIMY 8 198e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_DIMZ 9 199e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes#define PARAM_FIELD_SLOT 10 200097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes 201097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes /// Builds the types required by the pass for the given context. 202bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes void buildTypes(void) { 203bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes // Create the RsForEachStubParam struct. 204bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 205097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 206097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes llvm::Type *VoidPtrPtrTy = VoidPtrTy->getPointerTo(); 207097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 208097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes llvm::Type *Int32PtrTy = Int32Ty->getPointerTo(); 209097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes 210097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes /* Defined in frameworks/base/libs/rs/cpu_ref/rsCpuCore.h: 211db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 212097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes * struct RsForEachKernelStruct{ 21333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross * const void **ins; 21433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross * uint32_t *inEStrides; 215db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * void *out; 216db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t y; 217db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t z; 218097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes * uint32_t lid; 219097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes * const void *usr; 220097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes * uint32_t dimX; 221097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes * uint32_t dimY; 222097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes * uint32_t dimZ; 223097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes * uint32_t slot; 224db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * }; 225db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 226097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes llvm::SmallVector<llvm::Type*, 12> StructTypes; 227e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes StructTypes.push_back(VoidPtrPtrTy); // const void **ins 228e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes StructTypes.push_back(Int32PtrTy); // uint32_t *inEStrides 229097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(VoidPtrTy); // void *out 230097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(Int32Ty); // uint32_t y 231097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(Int32Ty); // uint32_t z 232097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(Int32Ty); // uint32_t lid 233097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(VoidPtrTy); // const void *usr 234097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(Int32Ty); // uint32_t dimX 235097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(Int32Ty); // uint32_t dimY 236097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(Int32Ty); // uint32_t dimZ 237097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes StructTypes.push_back(Int32Ty); // uint32_t slot 238881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 239bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ForEachStubType = 240bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::StructType::create(StructTypes, "RsForEachStubParamStruct"); 241bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 242bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes // Create the function type for expanded kernels. 243bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 244bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *ForEachStubPtrTy = ForEachStubType->getPointerTo(); 245bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 246bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::SmallVector<llvm::Type*, 8> ParamTypes; 247bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p 248bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(Int32Ty); // uint32_t x1 249bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(Int32Ty); // uint32_t x2 250bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(Int32Ty); // uint32_t outstep 251bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 252e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes ExpandedFunctionType = 253e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), ParamTypes, 254e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes false); 2558ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 2568ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 257357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// @brief Create skeleton of the expanded function. 258357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 259357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// This creates a function with the following signature: 260357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 261357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 2625010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes /// uint32_t outstep) 263357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 264357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) { 265bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 266bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::Create(ExpandedFunctionType, 267bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::GlobalValue::ExternalLinkage, 268bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes OldName + ".expand", Module); 269bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 270bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 271bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 272bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator AI = ExpandedFunction->arg_begin(); 273bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 274bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("p"); 275bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("x1"); 276bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("x2"); 277bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("arg_outstep"); 278bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 279bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin", 280bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction); 281806075b3a54af826fea78490fb213d8a0784138eTobias Grosser llvm::IRBuilder<> Builder(Begin); 282806075b3a54af826fea78490fb213d8a0784138eTobias Grosser Builder.CreateRetVoid(); 283806075b3a54af826fea78490fb213d8a0784138eTobias Grosser 284bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes return ExpandedFunction; 285357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser } 286357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 287e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @brief Create an empty loop 288e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 289e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// Create a loop of the form: 290e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 291e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// for (i = LowerBound; i < UpperBound; i++) 292e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// ; 293e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 294e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// After the loop has been created, the builder is set such that 295e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// instructions can be added to the loop body. 296e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 297e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param Builder The builder to use to build this loop. The current 298e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// position of the builder is the position the loop 299e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// will be inserted. 300e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LowerBound The first value of the loop iterator 301e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param UpperBound The maximal value of the loop iterator 302e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LoopIV A reference that will be set to the loop iterator. 303e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @return The BasicBlock that will be executed after the loop. 304e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder, 305e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *LowerBound, 306e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *UpperBound, 307e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode **LoopIV) { 308e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser assert(LowerBound->getType() == UpperBound->getType()); 309e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 310e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB; 311e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *Cond, *IVNext; 312e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 313e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 314e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB = Builder.GetInsertBlock(); 315e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), this); 316bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes HeaderBB = llvm::BasicBlock::Create(*Context, "Loop", CondBB->getParent()); 317e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 318e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (LowerBound < Upperbound) 319e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 320e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 321e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 322e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB->getTerminator()->eraseFromParent(); 323e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(CondBB); 324e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(LowerBound, UpperBound); 325e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 326e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 327e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ] 328e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv.next = iv + 1 329e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (iv.next < Upperbound) 330e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 331e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 332e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 333e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB); 334e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV = Builder.CreatePHI(LowerBound->getType(), 2, "X"); 335e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(LowerBound, CondBB); 336e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1)); 337e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(IVNext, HeaderBB); 338e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(IVNext, UpperBound); 339e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 340e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB->setName("Exit"); 341e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB->getFirstNonPHI()); 342e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser *LoopIV = IV; 343e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser return AfterBB; 344e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser } 345e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 3468ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosserpublic: 34733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross RSForEachExpandPass(bool pEnableStepOpt = true) 348900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes : ModulePass(ID), Module(nullptr), Context(nullptr), 349bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes mEnableStepOpt(pEnableStepOpt) { 350bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 3518ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3528ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 353c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override { 354c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines // This pass does not use any other analysis passes, but it does 355c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines // add/wrap the existing functions in the module (thus altering the CFG). 356c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines } 357c754d49ee856be620e041348a9f2b3d5610a5a26Stephen Hines 35833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // Build contribution to outgoing argument list for calling a 35933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // ForEach-able function, based on the special parameters of that 36033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // function. 36133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // 36233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // Signature - metadata bits for the signature of the ForEach-able function 36333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // X, Arg_p - values derived directly from expanded function, 36433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // suitable for computing arguments for the ForEach-able function 36533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // CalleeArgs - contribution is accumulated here 36633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross // Bump - invoked once for each contributed outgoing argument 36733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross void ExpandSpecialArguments(uint32_t Signature, 36833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::Value *X, 36933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::Value *Arg_p, 37033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::IRBuilder<> &Builder, 37133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::SmallVector<llvm::Value*, 8> &CalleeArgs, 37233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross std::function<void ()> Bump) { 37333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 37433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross if (bcinfo::MetadataExtractor::hasForEachSignatureCtxt(Signature)) { 37533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross CalleeArgs.push_back(Arg_p); 37633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Bump(); 37733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 37833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 37933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 38033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross CalleeArgs.push_back(X); 38133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Bump(); 38233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 38333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 38433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 38533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::Value *Y = Builder.CreateLoad( 38633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Builder.CreateStructGEP(Arg_p, PARAM_FIELD_Y), "Y"); 38733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross CalleeArgs.push_back(Y); 38833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Bump(); 38933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 39033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 39133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross if (bcinfo::MetadataExtractor::hasForEachSignatureZ(Signature)) { 39233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::Value *Z = Builder.CreateLoad( 39333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Builder.CreateStructGEP(Arg_p, PARAM_FIELD_Z), "Z"); 39433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross CalleeArgs.push_back(Z); 39533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross Bump(); 39633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 39733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross } 39833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 3998ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /* Performs the actual optimization on a selected function. On success, the 4008ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * Module will contain a new function of the name "<NAME>.expand" that 4018ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * invokes <NAME>() in a loop with the appropriate parameters. 4028ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser */ 403bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool ExpandFunction(llvm::Function *Function, uint32_t Signature) { 404bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ALOGV("Expanding ForEach-able Function %s", 405bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Function->getName().str().c_str()); 4068ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 4078ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 408bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Signature = getRootSignature(Function); 4098ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 4108ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // We couldn't determine how to expand this function based on its 4118ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // function signature. 4128ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser return false; 4138ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 4148ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 4158ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 416bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::DataLayout DL(Module); 4178ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 418bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 419bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes createEmptyExpandedFunction(Function->getName()); 420db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 421bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 422bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Extract the expanded function's parameters. It is guaranteed by 423bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * createEmptyExpandedFunction that there will be five parameters. 424bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 42533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 42633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 42733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 428bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator ExpandedFunctionArgIter = 429bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction->arg_begin(); 430db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 431bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 432bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 433bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 4345010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter); 435bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 436900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *InStep = nullptr; 437900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutStep = nullptr; 438db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 439db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Construct the actual function body. 440bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 441db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 442cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Collect and construct the arguments for the kernel(). 443db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 444bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator FunctionArgIter = Function->arg_begin(); 445db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 446900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Type *InTy = nullptr; 447900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *InBasePtr = nullptr; 448d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) { 449e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::Value *InsMember = Builder.CreateStructGEP(Arg_p, 450e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes PARAM_FIELD_INS); 451e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::LoadInst *InsBasePtr = Builder.CreateLoad(InsMember, "inputs_base"); 452e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 453e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::Value *InStepsMember = 454e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes Builder.CreateStructGEP(Arg_p, PARAM_FIELD_INESTRIDES); 455e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::LoadInst *InStepsBase = Builder.CreateLoad(InStepsMember, 456e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes "insteps_base"); 457e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 458e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::Value *IndexVal = Builder.getInt32(0); 459e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 460e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::Value *InStepAddr = Builder.CreateGEP(InStepsBase, IndexVal); 461e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::LoadInst *InStepArg = Builder.CreateLoad(InStepAddr, 462e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes "instep_addr"); 463e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 464bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes InTy = (FunctionArgIter++)->getType(); 465e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes InStep = getStepValue(&DL, InTy, InStepArg); 466e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 4672b04086acbef6520ae2c54a868b1271abf053122Stephen Hines InStep->setName("instep"); 468e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes 469e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::Value *InputAddr = Builder.CreateGEP(InsBasePtr, IndexVal); 470e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes InBasePtr = Builder.CreateLoad(InputAddr, "input_base"); 471db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 472db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 473900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Type *OutTy = nullptr; 474900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutBasePtr = nullptr; 475d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 476bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes OutTy = (FunctionArgIter++)->getType(); 477b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 4782b04086acbef6520ae2c54a868b1271abf053122Stephen Hines OutStep->setName("outstep"); 479097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes OutBasePtr = Builder.CreateLoad( 480097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes Builder.CreateStructGEP(Arg_p, PARAM_FIELD_OUT)); 481db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 482db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 483900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *UsrData = nullptr; 484d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)) { 485bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *UsrDataTy = (FunctionArgIter++)->getType(); 486db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData = Builder.CreatePointerCast(Builder.CreateLoad( 487097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes Builder.CreateStructGEP(Arg_p, PARAM_FIELD_USR)), UsrDataTy); 488db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData->setName("UsrData"); 489db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 490db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 49133cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::PHINode *IV; 49233cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross createLoop(Builder, Arg_x1, Arg_x2, &IV); 493097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes 49433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::SmallVector<llvm::Value*, 8> CalleeArgs; 49533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs, 49633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross [&FunctionArgIter]() { FunctionArgIter++; }); 497db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 498bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(FunctionArgIter == Function->arg_end()); 499db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 500cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Populate the actual call to kernel(). 501db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 502db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 503900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *InPtr = nullptr; 504900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutPtr = nullptr; 505db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 506ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // Calculate the current input and output pointers 50702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 508ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // We always calculate the input/output pointers with a GEP operating on i8 50902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // values and only cast at the very end to OutTy. This is because the step 51002f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // between two values is given in bytes. 51102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 51202f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // TODO: We could further optimize the output by using a GEP operation of 51302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // type 'OutTy' in cases where the element type of the allocation allows. 51402f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutBasePtr) { 51502f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 51602f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutOffset = Builder.CreateMul(OutOffset, OutStep); 51702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 51802f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 51902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser } 520bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 521ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InBasePtr) { 522ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1); 523ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InOffset = Builder.CreateMul(InOffset, InStep); 524ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreateGEP(InBasePtr, InOffset); 525ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreatePointerCast(InPtr, InTy); 526ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser } 52702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser 528ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InPtr) { 5297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(InPtr); 530db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 531db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 53202f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutPtr) { 5337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(OutPtr); 534db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 535db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 536db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (UsrData) { 537db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(UsrData); 538db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 539db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 54033cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross RootArgs.append(CalleeArgs.begin(), CalleeArgs.end()); 541db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 542bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Builder.CreateCall(Function, RootArgs); 543db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 5447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return true; 5457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Expand a pass-by-value kernel. 5487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 549bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool ExpandKernel(llvm::Function *Function, uint32_t Signature) { 550d88177580db4ddedf680854c51db333c97eabc59Stephen Hines bccAssert(bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)); 551bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ALOGV("Expanding kernel Function %s", Function->getName().str().c_str()); 5527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5537ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // TODO: Refactor this to share functionality with ExpandFunction. 554bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::DataLayout DL(Module); 5557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 556bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 557bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes createEmptyExpandedFunction(Function->getName()); 5587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 559bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 560bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Extract the expanded function's parameters. It is guaranteed by 561bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * createEmptyExpandedFunction that there will be five parameters. 562bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 563881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 564881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 565881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 566bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator ExpandedFunctionArgIter = 567bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction->arg_begin(); 568bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 569bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 570bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 571bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 5725010f641d1df6bc3447646ca7ef837410fb9b3dcChris Wailes llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter); 5737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Construct the actual function body. 575bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 5767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 57718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Create TBAA meta-data. 57818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDNode *TBAARenderScript, *TBAAAllocation, *TBAAPointer; 579bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDBuilder MDHelper(*Context); 58014588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien 58118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAARenderScript = MDHelper.createTBAARoot("RenderScript TBAA"); 582e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAAllocation = MDHelper.createTBAAScalarTypeNode("allocation", 583e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAARenderScript); 584e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAAllocation = MDHelper.createTBAAStructTagNode(TBAAAllocation, 585e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAAllocation, 0); 586e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAAPointer = MDHelper.createTBAAScalarTypeNode("pointer", 587e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes TBAARenderScript); 58814588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien TBAAPointer = MDHelper.createTBAAStructTagNode(TBAAPointer, TBAAPointer, 0); 58918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 59050f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray llvm::MDNode *AliasingDomain, *AliasingScope; 59150f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray AliasingDomain = MDHelper.createAnonymousAliasScopeDomain("RS argument scope domain"); 59250f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray AliasingScope = MDHelper.createAnonymousAliasScope(AliasingDomain, "RS argument scope"); 59350f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 594881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes /* 595881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * Collect and construct the arguments for the kernel(). 596881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * 597881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * Note that we load any loop-invariant arguments before entering the Loop. 598881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes */ 599881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes size_t NumInputs = Function->arg_size(); 6007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 601881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // No usrData parameter on kernels. 602881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes bccAssert( 603881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes !bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)); 604881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 605881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Function::arg_iterator ArgIter = Function->arg_begin(); 606881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 607881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Check the return type 608bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Type *OutTy = nullptr; 609bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *OutStep = nullptr; 610bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::LoadInst *OutBasePtr = nullptr; 611bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *CastedOutBasePtr = nullptr; 612881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 613e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes bool PassOutByPointer = false; 614881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 615d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 616bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *OutBaseTy = Function->getReturnType(); 617881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 61874a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (OutBaseTy->isVoidTy()) { 619e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes PassOutByPointer = true; 620881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutTy = ArgIter->getType(); 621881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 622881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes ArgIter++; 623881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes --NumInputs; 62474a4b08235990916911b8fe758d656c1171faf26Stephen Hines } else { 62574a4b08235990916911b8fe758d656c1171faf26Stephen Hines // We don't increment Args, since we are using the actual return type. 626881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutTy = OutBaseTy->getPointerTo(); 62774a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 628881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 629b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 63074a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutStep->setName("outstep"); 631097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes OutBasePtr = Builder.CreateLoad( 632097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes Builder.CreateStructGEP(Arg_p, PARAM_FIELD_OUT)); 633097ca14d8b6908d0e5b7f8f38011cad4fb94bc26Chris Wailes 6349c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 6359c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines OutBasePtr->setMetadata("tbaa", TBAAPointer); 6369c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 63750f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 63850f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray OutBasePtr->setMetadata("alias.scope", AliasingScope); 63950f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 640bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray CastedOutBasePtr = Builder.CreatePointerCast(OutBasePtr, OutTy, "casted_out"); 64174a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 64274a4b08235990916911b8fe758d656c1171faf26Stephen Hines 64333cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::PHINode *IV; 64433cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross createLoop(Builder, Arg_x1, Arg_x2, &IV); 64533cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 64633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross llvm::SmallVector<llvm::Value*, 8> CalleeArgs; 64733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross ExpandSpecialArguments(Signature, IV, Arg_p, Builder, CalleeArgs, 64833cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross [&NumInputs]() { --NumInputs; }); 64933cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross 650bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<llvm::Type*, 8> InTypes; 651bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<llvm::Value*, 8> InSteps; 652bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<llvm::Value*, 8> InBasePtrs; 653bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::SmallVector<bool, 8> InIsStructPointer; 654881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 655e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes if (NumInputs > 0) { 656e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::Value *InsMember = Builder.CreateStructGEP(Arg_p, PARAM_FIELD_INS); 657e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::LoadInst *InsBasePtr = Builder.CreateLoad(InsMember, "inputs_base"); 6587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 659e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::Value *InStepsMember = 660e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes Builder.CreateStructGEP(Arg_p, PARAM_FIELD_INESTRIDES); 661e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes llvm::LoadInst *InStepsBase = Builder.CreateLoad(InStepsMember, 662881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "insteps_base"); 663881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 664881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes for (size_t InputIndex = 0; InputIndex < NumInputs; 665881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes ++InputIndex, ArgIter++) { 666881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 667881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *IndexVal = Builder.getInt32(InputIndex); 668881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 669881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InStepAddr = Builder.CreateGEP(InStepsBase, IndexVal); 670881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InStepArg = Builder.CreateLoad(InStepAddr, 671881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "instep_addr"); 672881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 673326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes llvm::Type *InType = ArgIter->getType(); 674326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 675326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes /* 676326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * AArch64 calling dictate that structs of sufficient size get passed by 677326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * pointer instead of passed by value. This, combined with the fact 678326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * that we don't allow kernels to operate on pointer data means that if 679326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * we see a kernel with a pointer parameter we know that it is struct 680326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * input that has been promoted. As such we don't need to convert its 681326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * type to a pointer. Later we will need to know to avoid a load, so we 682326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes * save this information in InIsStructPointer. 683326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes */ 684326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes if (!InType->isPointerTy()) { 685326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InType = InType->getPointerTo(); 686326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InIsStructPointer.push_back(false); 687326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } else { 688326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InIsStructPointer.push_back(true); 689326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } 690326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 691881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InStep = getStepValue(&DL, InType, InStepArg); 692881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 693881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InStep->setName("instep"); 694881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 695881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InputAddr = Builder.CreateGEP(InsBasePtr, IndexVal); 696881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InBasePtr = Builder.CreateLoad(InputAddr, 697881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "input_base"); 698bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *CastInBasePtr = Builder.CreatePointerCast(InBasePtr, 699bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray InType, "casted_in"); 700881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (gEnableRsTbaa) { 701881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InBasePtr->setMetadata("tbaa", TBAAPointer); 702881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 703881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 70450f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray InBasePtr->setMetadata("alias.scope", AliasingScope); 70550f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 706881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InTypes.push_back(InType); 707881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InSteps.push_back(InStep); 708bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray InBasePtrs.push_back(CastInBasePtr); 709881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 710881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 7117ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 7127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Populate the actual call to kernel(). 7137ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 7147ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 7154102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser // Calculate the current input and output pointers 7167b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 7177b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 718881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // We always calculate the input/output pointers with a GEP operating on i8 719881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // values combined with a multiplication and only cast at the very end to 720881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // OutTy. This is to account for dynamic stepping sizes when the value 721881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // isn't apparent at compile time. In the (very common) case when we know 722881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // the step size at compile time, due to haveing complete type information 723881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // this multiplication will optmized out and produces code equivalent to a 724881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // a GEP on a pointer of the correct type. 725881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 726881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Output 727881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 728900c6c1f08f7c572125d7d39abe0f0f9eafbfa14Chris Wailes llvm::Value *OutPtr = nullptr; 729bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray if (CastedOutBasePtr) { 7307b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 731881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 732bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray OutPtr = Builder.CreateGEP(CastedOutBasePtr, OutOffset); 733bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 734e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes if (PassOutByPointer) { 735881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes RootArgs.push_back(OutPtr); 736881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 7374102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser } 7387b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser 739881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Inputs 74074a4b08235990916911b8fe758d656c1171faf26Stephen Hines 741881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (NumInputs > 0) { 742881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *Offset = Builder.CreateSub(IV, Arg_x1); 743881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 744881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes for (size_t Index = 0; Index < NumInputs; ++Index) { 745bb73b74a9f6ad26c2ab30557bfe6916a44ed75f6Tim Murray llvm::Value *InPtr = Builder.CreateGEP(InBasePtrs[Index], Offset); 746326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes llvm::Value *Input; 747326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 748326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes if (InIsStructPointer[Index]) { 749326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes Input = InPtr; 750326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 751326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } else { 752326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes llvm::LoadInst *InputLoad = Builder.CreateLoad(InPtr, "input"); 753326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes 754326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes if (gEnableRsTbaa) { 755326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes InputLoad->setMetadata("tbaa", TBAAAllocation); 756326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes } 757881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 75850f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray InputLoad->setMetadata("alias.scope", AliasingScope); 75950f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray 760326d02a9f3cfe30caa21e5c2aecbd4c85112b363Chris Wailes Input = InputLoad; 761881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 762881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 763881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes RootArgs.push_back(Input); 7649c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 7657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 7667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 76733cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Gross RootArgs.append(CalleeArgs.begin(), CalleeArgs.end()); 7687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 769bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *RetVal = Builder.CreateCall(Function, RootArgs); 7707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 771e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes if (OutPtr && !PassOutByPointer) { 77218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr); 7739c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 7749c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines Store->setMetadata("tbaa", TBAAAllocation); 7759c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 77650f5eb4b27ce6dd2a02cf389ac2acfa95c6939f1Tim Murray Store->setMetadata("alias.scope", AliasingScope); 7777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 7787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 779db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return true; 780db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 781db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 78218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Checks if pointers to allocation internals are exposed 78318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 78418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// This function verifies if through the parameters passed to the kernel 78518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// or through calls to the runtime library the script gains access to 78618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers pointing to data within a RenderScript Allocation. 78718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// If we know we control all loads from and stores to data within 78818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// RenderScript allocations and if we know the run-time internal accesses 78918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// are all annotated with RenderScript TBAA metadata, only then we 79018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// can safely use TBAA to distinguish between generic and from-allocation 79118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers. 792bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool allocPointersExposed(llvm::Module &Module) { 79318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Old style kernel function can expose pointers to elements within 79418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // allocations. 79518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // TODO: Extend analysis to allow simple cases of old-style kernels. 79625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines for (size_t i = 0; i < mExportForEachCount; ++i) { 79725eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char *Name = mExportForEachNameList[i]; 79825eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t Signature = mExportForEachSignatureList[i]; 799bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (Module.getFunction(Name) && 800d88177580db4ddedf680854c51db333c97eabc59Stephen Hines !bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)) { 80118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 80218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 80318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 80418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 80518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Check for library functions that expose a pointer to an Allocation or 80618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // that are not yet annotated with RenderScript-specific tbaa information. 80718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser static std::vector<std::string> Funcs; 80818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 80918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAt(...) 81018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationj"); 81118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationjj"); 81218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationjjj"); 81318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsSetElementAt() 81418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvj"); 81518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjj"); 81618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjjj"); 81718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_Y() 81818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_Y13rs_allocationjj"); 81918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_U() 82018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_U13rs_allocationjj"); 82118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_V() 82218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_V13rs_allocationjj"); 82318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 82418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser for (std::vector<std::string>::iterator FI = Funcs.begin(), 82518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser FE = Funcs.end(); 82618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser FI != FE; ++FI) { 827bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *Function = Module.getFunction(*FI); 82818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 829bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (!Function) { 83018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser ALOGE("Missing run-time function '%s'", FI->c_str()); 83118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 83218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 83318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 834bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (Function->getNumUses() > 0) { 83518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 83618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 83718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 83818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 83918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return false; 84018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 84118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 84218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Connect RenderScript TBAA metadata to C/C++ metadata 84318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 84418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// The TBAA metadata used to annotate loads/stores from RenderScript 845e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// Allocations is generated in a separate TBAA tree with a 846e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// "RenderScript TBAA" root node. LLVM does assume may-alias for all nodes in 847e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// unrelated alias analysis trees. This function makes the RenderScript TBAA 848e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// a subtree of the normal C/C++ TBAA tree aside of normal C/C++ types. With 849e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// the connected trees every access to an Allocation is resolved to 850e10b8641813c701159c6615faaa58dd1a9b31ce5Chris Wailes /// must-alias if compared to a normal C/C++ access. 851bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes void connectRenderScriptTBAAMetadata(llvm::Module &Module) { 852bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDBuilder MDHelper(*Context); 853bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDNode *TBAARenderScript = 854bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes MDHelper.createTBAARoot("RenderScript TBAA"); 85518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 856bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDNode *TBAARoot = MDHelper.createTBAARoot("Simple C/C++ TBAA"); 857bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDNode *TBAAMergedRS = MDHelper.createTBAANode("RenderScript", 858bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes TBAARoot); 85918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 86018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAARenderScript->replaceAllUsesWith(TBAAMergedRS); 86118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 86218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 863bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes virtual bool runOnModule(llvm::Module &Module) { 864bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool Changed = false; 865bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->Module = &Module; 866bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->Context = &Module.getContext(); 867bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 868bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->buildTypes(); 869bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 870bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bcinfo::MetadataExtractor me(&Module); 87125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines if (!me.extract()) { 87225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines ALOGE("Could not extract metadata from module!"); 87325eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines return false; 87425eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines } 87525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachCount = me.getExportForEachSignatureCount(); 87625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachNameList = me.getExportForEachNameList(); 87725eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachSignatureList = me.getExportForEachSignatureList(); 878db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 879bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool AllocsExposed = allocPointersExposed(Module); 88018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 88125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines for (size_t i = 0; i < mExportForEachCount; ++i) { 88225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char *name = mExportForEachNameList[i]; 88325eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t signature = mExportForEachSignatureList[i]; 884bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *kernel = Module.getFunction(name); 885cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser if (kernel) { 886d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureKernel(signature)) { 887cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandKernel(kernel, signature); 888acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 889acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else if (kernel->getReturnType()->isVoidTy()) { 890cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandFunction(kernel, signature); 891acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 892acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else { 893acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // There are some graphics root functions that are not 894acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // expanded, but that will be called directly. For those 895acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // functions, we can not set the linkage to internal. 896acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } 897cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines } 898db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 899db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 9009c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa && !AllocsExposed) { 901bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes connectRenderScriptTBAAMetadata(Module); 90218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 90318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 904cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines return Changed; 905db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 906db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 907db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual const char *getPassName() const { 908db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return "ForEach-able Function Expansion"; 909db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 910db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 9117a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}; // end RSForEachExpandPass 912db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 9137a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end anonymous namespace 9147a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 9157a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaochar RSForEachExpandPass::ID = 0; 91633cda5cf335afc6aa2dbe02062bc9e6649e1f87cDavid Grossstatic llvm::RegisterPass<RSForEachExpandPass> X("foreachexp", "ForEach Expand Pass"); 917db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 918db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace bcc { 919db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 9207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaollvm::ModulePass * 92125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen HinescreateRSForEachExpandPass(bool pEnableStepOpt){ 92225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines return new RSForEachExpandPass(pEnableStepOpt); 9237a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} 924db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 9257a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end namespace bcc 926