RSKernelExpand.cpp revision 881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fd
1db169187dea4602e4ad32058762d23d474753fd0Stephen Hines/* 2db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Copyright 2012, The Android Open Source Project 3db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 4db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Licensed under the Apache License, Version 2.0 (the "License"); 5db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * you may not use this file except in compliance with the License. 6db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * You may obtain a copy of the License at 7db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 8db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * http://www.apache.org/licenses/LICENSE-2.0 9db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 10db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Unless required by applicable law or agreed to in writing, software 11db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * distributed under the License is distributed on an "AS IS" BASIS, 12db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * See the License for the specific language governing permissions and 14db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * limitations under the License. 15db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 16db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 176e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines#include "bcc/Assert.h" 18e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSTransforms.h" 197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao#include <cstdlib> 217a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 22b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DerivedTypes.h> 23b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Function.h> 24b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Instructions.h> 25b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/IRBuilder.h> 2618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser#include <llvm/IR/MDBuilder.h> 27b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Module.h> 28c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Pass.h> 297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines#include <llvm/Support/raw_ostream.h> 30b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DataLayout.h> 31cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser#include <llvm/IR/Function.h> 32b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Type.h> 33806075b3a54af826fea78490fb213d8a0784138eTobias Grosser#include <llvm/Transforms/Utils/BasicBlockUtils.h> 34c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang 35c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include "bcc/Config/Config.h" 36ef73a242762bcd8113b9b65ceccbe7d909b5acbcZonr Chang#include "bcc/Support/Log.h" 37db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 38d88177580db4ddedf680854c51db333c97eabc59Stephen Hines#include "bcinfo/MetadataExtractor.h" 39d88177580db4ddedf680854c51db333c97eabc59Stephen Hines 40bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes#define NUM_EXPANDED_FUNCTION_PARAMS 5 41bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 427a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaousing namespace bcc; 437a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 44db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace { 457a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 4614588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chienstatic const bool gEnableRsTbaa = true; 479c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines 487a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao/* RSForEachExpandPass - This pass operates on functions that are able to be 497a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the 507a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * ForEach-able function to be invoked over the appropriate data cells of the 517a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * input/output allocations (adjusting other relevant parameters as we go). We 527a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * support doing this for any ForEach-able compute kernels. The new function 537a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * name is the original function name followed by ".expand". Note that we 547a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * still generate code for the original function. 557a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao */ 567a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoclass RSForEachExpandPass : public llvm::ModulePass { 577a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoprivate: 58db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static char ID; 59db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 60bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Module *Module; 61bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::LLVMContext *Context; 62bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 63bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 64bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Pointer to LLVM type information for the ForEachStubType and the function 65bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * signature for expanded kernels. These must be re-calculated for each 66bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * module the pass is run on. 67bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 68bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::StructType *ForEachStubType; 69bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::FunctionType *ExpandedFunctionType; 70db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t mExportForEachCount; 7225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char **mExportForEachNameList; 7325eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const uint32_t *mExportForEachSignatureList; 74cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines 752b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Turns on optimization of allocation stride values. 762b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool mEnableStepOpt; 772b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 78bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes uint32_t getRootSignature(llvm::Function *Function) { 79db169187dea4602e4ad32058762d23d474753fd0Stephen Hines const llvm::NamedMDNode *ExportForEachMetadata = 80bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Module->getNamedMetadata("#rs_export_foreach"); 81db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 82db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (!ExportForEachMetadata) { 83db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 8> RootArgTys; 84bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes for (llvm::Function::arg_iterator B = Function->arg_begin(), 85bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes E = Function->arg_end(); 86db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 87db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 88db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgTys.push_back(B->getType()); 89db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 90db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 91db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // For pre-ICS bitcode, we may not have signature information. In that 92db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // case, we use the size of the RootArgTys to select the number of 93db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // arguments. 94db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return (1 << RootArgTys.size()) - 1; 95db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 96db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 977ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ExportForEachMetadata->getNumOperands() == 0) { 987ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return 0; 997ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 1007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 1016e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(ExportForEachMetadata->getNumOperands() > 0); 102db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 103cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // We only handle the case for legacy root() functions here, so this is 104cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // hard-coded to look at only the first such function. 105db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0); 106db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigNode != NULL && SigNode->getNumOperands() == 1) { 107db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *SigVal = SigNode->getOperand(0); 108db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigVal->getValueID() == llvm::Value::MDStringVal) { 109db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::StringRef SigString = 110db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static_cast<llvm::MDString*>(SigVal)->getString(); 111db169187dea4602e4ad32058762d23d474753fd0Stephen Hines uint32_t Signature = 0; 112db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigString.getAsInteger(10, Signature)) { 113db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ALOGE("Non-integer signature value '%s'", SigString.str().c_str()); 114db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 115db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 116db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return Signature; 117db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 118db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 119db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 120db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 121db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 122db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1232b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Get the actual value we should use to step through an allocation. 1247b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 1257b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // Normally the value we use to step through an allocation is given to us by 1267b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // the driver. However, for certain primitive data types, we can derive an 1277b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // integer constant for the step value. We use this integer constant whenever 1287b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // possible to allow further compiler optimizations to take place. 1297b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 130b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines // DL - Target Data size/layout information. 1312b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // T - Type of allocation (should be a pointer). 1322b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // OrigStep - Original step increment (root.expand() input from driver). 133bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *AllocType, 1342b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OrigStep) { 135b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines bccAssert(DL); 136bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(AllocType); 1372b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(OrigStep); 138bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType); 139bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 140bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (mEnableStepOpt && AllocType != VoidPtrTy && PT) { 1412b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *ET = PT->getElementType(); 142b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines uint64_t ETSize = DL->getTypeAllocSize(ET); 143bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 1442b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return llvm::ConstantInt::get(Int32Ty, ETSize); 1452b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } else { 1462b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return OrigStep; 1472b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1482b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1492b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 150bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /// @brief Builds the types required by the pass for the given context. 151bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes void buildTypes(void) { 152bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes // Create the RsForEachStubParam struct. 153bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 154bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 155bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 156db169187dea4602e4ad32058762d23d474753fd0Stephen Hines /* Defined in frameworks/base/libs/rs/rs_hal.h: 157db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 158db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * struct RsForEachStubParamStruct { 159db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *in; 160db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * void *out; 161db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *usr; 162bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * uint32_t usr_len; 163db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t x; 164db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t y; 165db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t z; 166db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t lod; 167db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * enum RsAllocationCubemapFace face; 168db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t ar[16]; 169881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * const void **ins; 170881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * uint32_t *eStrideIns; 171db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * }; 172db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 173bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::SmallVector<llvm::Type*, 16> StructTypes; 174bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(VoidPtrTy); // const void *in 175bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(VoidPtrTy); // void *out 176bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(VoidPtrTy); // const void *usr 177bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(Int32Ty); // uint32_t usr_len 178bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(Int32Ty); // uint32_t x 179bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(Int32Ty); // uint32_t y 180bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(Int32Ty); // uint32_t z 181bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(Int32Ty); // uint32_t lod 182bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(Int32Ty); // enum RsAllocationCubemapFace 183bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes StructTypes.push_back(llvm::ArrayType::get(Int32Ty, 16)); // uint32_t ar[16] 184bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 185881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes StructTypes.push_back(llvm::PointerType::getUnqual(VoidPtrTy)); // const void **ins 186881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes StructTypes.push_back(Int32Ty->getPointerTo()); // uint32_t *eStrideIns 187881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 188bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ForEachStubType = 189bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::StructType::create(StructTypes, "RsForEachStubParamStruct"); 190bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 191bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes // Create the function type for expanded kernels. 192bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 193bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *ForEachStubPtrTy = ForEachStubType->getPointerTo(); 194bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 195bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::SmallVector<llvm::Type*, 8> ParamTypes; 196bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p 197bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(Int32Ty); // uint32_t x1 198bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(Int32Ty); // uint32_t x2 199bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(Int32Ty); // uint32_t instep 200bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes.push_back(Int32Ty); // uint32_t outstep 201bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 202bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunctionType = llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), 203bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ParamTypes, 204bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes false); 2058ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 2068ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 207357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// @brief Create skeleton of the expanded function. 208357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 209357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// This creates a function with the following signature: 210357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 211357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 212357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// uint32_t instep, uint32_t outstep) 213357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 214357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) { 215bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 216bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::Create(ExpandedFunctionType, 217bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::GlobalValue::ExternalLinkage, 218bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes OldName + ".expand", Module); 219bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 220bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 221bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 222bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator AI = ExpandedFunction->arg_begin(); 223bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 224bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("p"); 225bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("x1"); 226bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("x2"); 227bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("arg_instep"); 228bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes (AI++)->setName("arg_outstep"); 229bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 230bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin", 231bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction); 232806075b3a54af826fea78490fb213d8a0784138eTobias Grosser llvm::IRBuilder<> Builder(Begin); 233806075b3a54af826fea78490fb213d8a0784138eTobias Grosser Builder.CreateRetVoid(); 234806075b3a54af826fea78490fb213d8a0784138eTobias Grosser 235bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes return ExpandedFunction; 236357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser } 237357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 238e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @brief Create an empty loop 239e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 240e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// Create a loop of the form: 241e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 242e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// for (i = LowerBound; i < UpperBound; i++) 243e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// ; 244e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 245e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// After the loop has been created, the builder is set such that 246e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// instructions can be added to the loop body. 247e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 248e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param Builder The builder to use to build this loop. The current 249e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// position of the builder is the position the loop 250e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// will be inserted. 251e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LowerBound The first value of the loop iterator 252e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param UpperBound The maximal value of the loop iterator 253e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LoopIV A reference that will be set to the loop iterator. 254e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @return The BasicBlock that will be executed after the loop. 255e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder, 256e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *LowerBound, 257e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *UpperBound, 258e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode **LoopIV) { 259e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser assert(LowerBound->getType() == UpperBound->getType()); 260e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 261e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB; 262e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *Cond, *IVNext; 263e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 264e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 265e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB = Builder.GetInsertBlock(); 266e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), this); 267bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes HeaderBB = llvm::BasicBlock::Create(*Context, "Loop", CondBB->getParent()); 268e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 269e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (LowerBound < Upperbound) 270e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 271e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 272e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 273e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB->getTerminator()->eraseFromParent(); 274e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(CondBB); 275e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(LowerBound, UpperBound); 276e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 277e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 278e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ] 279e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv.next = iv + 1 280e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (iv.next < Upperbound) 281e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 282e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 283e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 284e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB); 285e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV = Builder.CreatePHI(LowerBound->getType(), 2, "X"); 286e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(LowerBound, CondBB); 287e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1)); 288e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(IVNext, HeaderBB); 289e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(IVNext, UpperBound); 290e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 291e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB->setName("Exit"); 292e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB->getFirstNonPHI()); 293e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser *LoopIV = IV; 294e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser return AfterBB; 295e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser } 296e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 2978ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosserpublic: 29825eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines RSForEachExpandPass(bool pEnableStepOpt) 299bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes : ModulePass(ID), Module(NULL), Context(NULL), 300bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes mEnableStepOpt(pEnableStepOpt) { 301bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 3028ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3038ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 3048ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /* Performs the actual optimization on a selected function. On success, the 3058ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * Module will contain a new function of the name "<NAME>.expand" that 3068ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * invokes <NAME>() in a loop with the appropriate parameters. 3078ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser */ 308bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool ExpandFunction(llvm::Function *Function, uint32_t Signature) { 309bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ALOGV("Expanding ForEach-able Function %s", 310bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Function->getName().str().c_str()); 3118ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 3128ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 313bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Signature = getRootSignature(Function); 3148ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 3158ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // We couldn't determine how to expand this function based on its 3168ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // function signature. 3178ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser return false; 3188ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3198ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3208ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 321bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::DataLayout DL(Module); 3228ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 323bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 324bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes createEmptyExpandedFunction(Function->getName()); 325db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 326bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 327db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 328bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 329bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Extract the expanded function's parameters. It is guaranteed by 330bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * createEmptyExpandedFunction that there will be five parameters. 331bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 332bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator ExpandedFunctionArgIter = 333bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction->arg_begin(); 334db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 335bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 336bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 337bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 338bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_instep = &*(ExpandedFunctionArgIter++); 339bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_outstep = &*ExpandedFunctionArgIter; 340bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 341bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *InStep = NULL; 3422b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OutStep = NULL; 343db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 344db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Construct the actual function body. 345bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 346db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 347cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Collect and construct the arguments for the kernel(). 348db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 349bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator FunctionArgIter = Function->arg_begin(); 350db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 351db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *InTy = NULL; 352ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser llvm::Value *InBasePtr = NULL; 353d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) { 354bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes InTy = (FunctionArgIter++)->getType(); 355b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines InStep = getStepValue(&DL, InTy, Arg_instep); 3562b04086acbef6520ae2c54a868b1271abf053122Stephen Hines InStep->setName("instep"); 357ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 0)); 358db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 359db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 360db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *OutTy = NULL; 36102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser llvm::Value *OutBasePtr = NULL; 362d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 363bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes OutTy = (FunctionArgIter++)->getType(); 364b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 3652b04086acbef6520ae2c54a868b1271abf053122Stephen Hines OutStep->setName("outstep"); 36602f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 1)); 367db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 368db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 369db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *UsrData = NULL; 370d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)) { 371bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *UsrDataTy = (FunctionArgIter++)->getType(); 372db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData = Builder.CreatePointerCast(Builder.CreateLoad( 373db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 2)), UsrDataTy); 374db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData->setName("UsrData"); 375db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 376db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 377d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 378bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes FunctionArgIter++; 379db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 380db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 381db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Y = NULL; 382d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 383db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 384bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes FunctionArgIter++; 385db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 386db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 387bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bccAssert(FunctionArgIter == Function->arg_end()); 388db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 389e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 390e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser createLoop(Builder, Arg_x1, Arg_x2, &IV); 391db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 392cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Populate the actual call to kernel(). 393db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 394db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 395bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *InPtr = NULL; 3967ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutPtr = NULL; 397db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 398ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // Calculate the current input and output pointers 39902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 400ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // We always calculate the input/output pointers with a GEP operating on i8 40102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // values and only cast at the very end to OutTy. This is because the step 40202f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // between two values is given in bytes. 40302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 40402f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // TODO: We could further optimize the output by using a GEP operation of 40502f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // type 'OutTy' in cases where the element type of the allocation allows. 40602f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutBasePtr) { 40702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 40802f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutOffset = Builder.CreateMul(OutOffset, OutStep); 40902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 41002f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 41102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser } 412bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 413ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InBasePtr) { 414ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1); 415ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InOffset = Builder.CreateMul(InOffset, InStep); 416ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreateGEP(InBasePtr, InOffset); 417ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreatePointerCast(InPtr, InTy); 418ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser } 41902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser 420ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InPtr) { 4217ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(InPtr); 422db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 423db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 42402f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutPtr) { 4257ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(OutPtr); 426db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 427db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 428db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (UsrData) { 429db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(UsrData); 430db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 431db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 432e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *X = IV; 433d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 434db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(X); 435db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 436db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 437db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (Y) { 438db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(Y); 439db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 440db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 441bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes Builder.CreateCall(Function, RootArgs); 442db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 4437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return true; 4447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Expand a pass-by-value kernel. 4477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 448bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool ExpandKernel(llvm::Function *Function, uint32_t Signature) { 449d88177580db4ddedf680854c51db333c97eabc59Stephen Hines bccAssert(bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)); 450bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ALOGV("Expanding kernel Function %s", Function->getName().str().c_str()); 4517ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // TODO: Refactor this to share functionality with ExpandFunction. 453bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::DataLayout DL(Module); 4547ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 455bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *ExpandedFunction = 456bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes createEmptyExpandedFunction(Function->getName()); 4577ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 458bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes /* 459bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * Extract the expanded function's parameters. It is guaranteed by 460bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes * createEmptyExpandedFunction that there will be five parameters. 461bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes */ 462881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 463881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 464881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 465bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function::arg_iterator ExpandedFunctionArgIter = 466bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes ExpandedFunction->arg_begin(); 467bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 468bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 469bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 470bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 471bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_instep = &*(ExpandedFunctionArgIter++); 472bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *Arg_outstep = &*ExpandedFunctionArgIter; 4737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Construct the actual function body. 475bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 4767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 47718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Create TBAA meta-data. 47818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDNode *TBAARenderScript, *TBAAAllocation, *TBAAPointer; 479bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDBuilder MDHelper(*Context); 48014588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien 48118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAARenderScript = MDHelper.createTBAARoot("RenderScript TBAA"); 48214588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien TBAAAllocation = MDHelper.createTBAAScalarTypeNode("allocation", TBAARenderScript); 48314588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien TBAAAllocation = MDHelper.createTBAAStructTagNode(TBAAAllocation, TBAAAllocation, 0); 48414588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien TBAAPointer = MDHelper.createTBAAScalarTypeNode("pointer", TBAARenderScript); 48514588cf0babf4596f1bcf4ea05ddd2ceb458a916Logan Chien TBAAPointer = MDHelper.createTBAAStructTagNode(TBAAPointer, TBAAPointer, 0); 48618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 487881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes /* 488881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * Collect and construct the arguments for the kernel(). 489881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * 490881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes * Note that we load any loop-invariant arguments before entering the Loop. 491881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes */ 492881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes size_t NumInputs = Function->arg_size(); 4937ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 494881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *Y = NULL; 495881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 496881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 497881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes --NumInputs; 498881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 499881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 500881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 501881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes --NumInputs; 502881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 503881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 504881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // No usrData parameter on kernels. 505881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes bccAssert( 506881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes !bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)); 507881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 508881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Function::arg_iterator ArgIter = Function->arg_begin(); 509881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 510881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Check the return type 511881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Type *OutTy = NULL; 512881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *OutStep = NULL; 51318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::LoadInst *OutBasePtr = NULL; 514881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 515881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes bool PassOutByReference = false; 516881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 517d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 518bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Type *OutBaseTy = Function->getReturnType(); 519881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 52074a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (OutBaseTy->isVoidTy()) { 52174a4b08235990916911b8fe758d656c1171faf26Stephen Hines PassOutByReference = true; 522881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutTy = ArgIter->getType(); 523881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 524881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes ArgIter++; 525881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes --NumInputs; 52674a4b08235990916911b8fe758d656c1171faf26Stephen Hines } else { 52774a4b08235990916911b8fe758d656c1171faf26Stephen Hines // We don't increment Args, since we are using the actual return type. 528881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutTy = OutBaseTy->getPointerTo(); 52974a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 530881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 531b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 53274a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutStep->setName("outstep"); 5337b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser OutBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 1)); 5349c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 5359c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines OutBasePtr->setMetadata("tbaa", TBAAPointer); 5369c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 53774a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 53874a4b08235990916911b8fe758d656c1171faf26Stephen Hines 539881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::SmallVector<llvm::Type*, 8> InTypes; 540881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::SmallVector<llvm::Value*, 8> InSteps; 541881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::SmallVector<llvm::LoadInst*, 8> InBasePtrs; 542881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 543881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (NumInputs == 1) { 544881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Type *InType = ArgIter->getType()->getPointerTo(); 545881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InStep = getStepValue(&DL, InType, Arg_instep); 546881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 5477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InStep->setName("instep"); 548881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 549881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *Input = Builder.CreateStructGEP(Arg_p, 0); 550881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InBasePtr = Builder.CreateLoad(Input, "input_base"); 551881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 5529c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 5539c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines InBasePtr->setMetadata("tbaa", TBAAPointer); 5549c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 5557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 556881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InTypes.push_back(InType); 557881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InSteps.push_back(InStep); 558881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InBasePtrs.push_back(InBasePtr); 5597ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 560881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } else if (NumInputs > 1) { 561881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InsMember = Builder.CreateStructGEP(Arg_p, 10); 562881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InsBasePtr = Builder.CreateLoad(InsMember, 563881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "inputs_base"); 5647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 565881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InStepsMember = Builder.CreateStructGEP(Arg_p, 11); 566881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InStepsBase = Builder.CreateLoad(InStepsMember, 567881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "insteps_base"); 568881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 569881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes for (size_t InputIndex = 0; InputIndex < NumInputs; 570881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes ++InputIndex, ArgIter++) { 571881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 572881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *IndexVal = Builder.getInt32(InputIndex); 573881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 574881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InStepAddr = Builder.CreateGEP(InStepsBase, IndexVal); 575881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InStepArg = Builder.CreateLoad(InStepAddr, 576881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "instep_addr"); 577881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 578881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Type *InType = ArgIter->getType()->getPointerTo(); 579881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InStep = getStepValue(&DL, InType, InStepArg); 580881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 581881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InStep->setName("instep"); 582881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 583881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InputAddr = Builder.CreateGEP(InsBasePtr, IndexVal); 584881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *InBasePtr = Builder.CreateLoad(InputAddr, 585881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes "input_base"); 5867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 587881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (gEnableRsTbaa) { 588881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InBasePtr->setMetadata("tbaa", TBAAPointer); 589881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 590881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 591881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InTypes.push_back(InType); 592881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InSteps.push_back(InStep); 593881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InBasePtrs.push_back(InBasePtr); 594881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 595881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 5967ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 597e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 598e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser createLoop(Builder, Arg_x1, Arg_x2, &IV); 5997ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 6007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Populate the actual call to kernel(). 6017ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 6027ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 6034102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser // Calculate the current input and output pointers 6047b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 6057b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 606881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // We always calculate the input/output pointers with a GEP operating on i8 607881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // values combined with a multiplication and only cast at the very end to 608881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // OutTy. This is to account for dynamic stepping sizes when the value 609881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // isn't apparent at compile time. In the (very common) case when we know 610881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // the step size at compile time, due to haveing complete type information 611881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // this multiplication will optmized out and produces code equivalent to a 612881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // a GEP on a pointer of the correct type. 613881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 614881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Output 615881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 616881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *OutPtr = NULL; 6177b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser if (OutBasePtr) { 6187b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 619881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 6207b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser OutOffset = Builder.CreateMul(OutOffset, OutStep); 621881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 622881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 623bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 624881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (PassOutByReference) { 625881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes RootArgs.push_back(OutPtr); 626881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 6274102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser } 6287b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser 629881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes // Inputs 63074a4b08235990916911b8fe758d656c1171faf26Stephen Hines 631881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (NumInputs > 0) { 632881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *Offset = Builder.CreateSub(IV, Arg_x1); 633881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 634881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes for (size_t Index = 0; Index < NumInputs; ++Index) { 635881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InOffset = Builder.CreateMul(Offset, InSteps[Index]); 636881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::Value *InPtr = Builder.CreateGEP(InBasePtrs[Index], InOffset); 637881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 638881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes InPtr = Builder.CreatePointerCast(InPtr, InTypes[Index]); 639881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 640881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes llvm::LoadInst *Input = Builder.CreateLoad(InPtr, "input"); 641881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 642881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes if (gEnableRsTbaa) { 643881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes Input->setMetadata("tbaa", TBAAAllocation); 644881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes } 645881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes 646881cda4f5b793a1e7f7d33cf4a31efc3fdbba4fdChris Wailes RootArgs.push_back(Input); 6479c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 6487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 6497ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 650e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *X = IV; 651d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 6527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(X); 6537ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 6547ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 6557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (Y) { 6567ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(Y); 6577ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 6587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 659bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Value *RetVal = Builder.CreateCall(Function, RootArgs); 6607ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 6617b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser if (OutPtr && !PassOutByReference) { 66218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr); 6639c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 6649c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines Store->setMetadata("tbaa", TBAAAllocation); 6659c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 6667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 6677ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 668db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return true; 669db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 670db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 67118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Checks if pointers to allocation internals are exposed 67218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 67318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// This function verifies if through the parameters passed to the kernel 67418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// or through calls to the runtime library the script gains access to 67518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers pointing to data within a RenderScript Allocation. 67618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// If we know we control all loads from and stores to data within 67718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// RenderScript allocations and if we know the run-time internal accesses 67818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// are all annotated with RenderScript TBAA metadata, only then we 67918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// can safely use TBAA to distinguish between generic and from-allocation 68018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers. 681bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool allocPointersExposed(llvm::Module &Module) { 68218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Old style kernel function can expose pointers to elements within 68318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // allocations. 68418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // TODO: Extend analysis to allow simple cases of old-style kernels. 68525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines for (size_t i = 0; i < mExportForEachCount; ++i) { 68625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char *Name = mExportForEachNameList[i]; 68725eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t Signature = mExportForEachSignatureList[i]; 688bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (Module.getFunction(Name) && 689d88177580db4ddedf680854c51db333c97eabc59Stephen Hines !bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)) { 69018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 69118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 69218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 69318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 69418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Check for library functions that expose a pointer to an Allocation or 69518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // that are not yet annotated with RenderScript-specific tbaa information. 69618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser static std::vector<std::string> Funcs; 69718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 69818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAt(...) 69918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationj"); 70018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationjj"); 70118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationjjj"); 70218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsSetElementAt() 70318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvj"); 70418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjj"); 70518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjjj"); 70618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_Y() 70718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_Y13rs_allocationjj"); 70818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_U() 70918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_U13rs_allocationjj"); 71018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_V() 71118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_V13rs_allocationjj"); 71218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 71318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser for (std::vector<std::string>::iterator FI = Funcs.begin(), 71418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser FE = Funcs.end(); 71518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser FI != FE; ++FI) { 716bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *Function = Module.getFunction(*FI); 71718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 718bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (!Function) { 71918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser ALOGE("Missing run-time function '%s'", FI->c_str()); 72018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 72118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 72218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 723bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes if (Function->getNumUses() > 0) { 72418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 72518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 72618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 72718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 72818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return false; 72918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 73018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 73118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Connect RenderScript TBAA metadata to C/C++ metadata 73218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 73318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// The TBAA metadata used to annotate loads/stores from RenderScript 73418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// Allocations is generated in a separate TBAA tree with a "RenderScript TBAA" 73518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// root node. LLVM does assume may-alias for all nodes in unrelated alias 73618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// analysis trees. This function makes the RenderScript TBAA a subtree of the 73718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// normal C/C++ TBAA tree aside of normal C/C++ types. With the connected trees 73818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// every access to an Allocation is resolved to must-alias if compared to 73918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// a normal C/C++ access. 740bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes void connectRenderScriptTBAAMetadata(llvm::Module &Module) { 741bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDBuilder MDHelper(*Context); 742bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDNode *TBAARenderScript = 743bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes MDHelper.createTBAARoot("RenderScript TBAA"); 74418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 745bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDNode *TBAARoot = MDHelper.createTBAARoot("Simple C/C++ TBAA"); 746bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::MDNode *TBAAMergedRS = MDHelper.createTBAANode("RenderScript", 747bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes TBAARoot); 74818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 74918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAARenderScript->replaceAllUsesWith(TBAAMergedRS); 75018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 75118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 752bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes virtual bool runOnModule(llvm::Module &Module) { 753bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool Changed = false; 754bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->Module = &Module; 755bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->Context = &Module.getContext(); 756bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 757bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes this->buildTypes(); 758bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes 759bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bcinfo::MetadataExtractor me(&Module); 76025eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines if (!me.extract()) { 76125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines ALOGE("Could not extract metadata from module!"); 76225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines return false; 76325eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines } 76425eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachCount = me.getExportForEachSignatureCount(); 76525eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachNameList = me.getExportForEachNameList(); 76625eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines mExportForEachSignatureList = me.getExportForEachSignatureList(); 767db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 768bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes bool AllocsExposed = allocPointersExposed(Module); 76918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 77025eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines for (size_t i = 0; i < mExportForEachCount; ++i) { 77125eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines const char *name = mExportForEachNameList[i]; 77225eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines uint32_t signature = mExportForEachSignatureList[i]; 773bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes llvm::Function *kernel = Module.getFunction(name); 774cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser if (kernel) { 775d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureKernel(signature)) { 776cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandKernel(kernel, signature); 777acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 778acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else if (kernel->getReturnType()->isVoidTy()) { 779cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandFunction(kernel, signature); 780acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 781acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else { 782acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // There are some graphics root functions that are not 783acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // expanded, but that will be called directly. For those 784acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // functions, we can not set the linkage to internal. 785acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } 786cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines } 787db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 788db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7899c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa && !AllocsExposed) { 790bdbff6e600b0d834e4770f65c7d2df93d7ef305cChris Wailes connectRenderScriptTBAAMetadata(Module); 79118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 79218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 793cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines return Changed; 794db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 795db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 796db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual const char *getPassName() const { 797db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return "ForEach-able Function Expansion"; 798db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 799db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 8007a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}; // end RSForEachExpandPass 801db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 8027a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end anonymous namespace 8037a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 8047a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaochar RSForEachExpandPass::ID = 0; 805db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 806db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace bcc { 807db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 8087a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaollvm::ModulePass * 80925eb586bb055ae07c7e82a2b1bdbd6936641580cStephen HinescreateRSForEachExpandPass(bool pEnableStepOpt){ 81025eb586bb055ae07c7e82a2b1bdbd6936641580cStephen Hines return new RSForEachExpandPass(pEnableStepOpt); 8117a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} 812db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 8137a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end namespace bcc 814