RSKernelExpand.cpp revision 9c5263e00d89dfdd25d3a1706eb319e8529ec604
1db169187dea4602e4ad32058762d23d474753fd0Stephen Hines/* 2db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Copyright 2012, The Android Open Source Project 3db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 4db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Licensed under the Apache License, Version 2.0 (the "License"); 5db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * you may not use this file except in compliance with the License. 6db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * You may obtain a copy of the License at 7db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 8db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * http://www.apache.org/licenses/LICENSE-2.0 9db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 10db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Unless required by applicable law or agreed to in writing, software 11db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * distributed under the License is distributed on an "AS IS" BASIS, 12db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * See the License for the specific language governing permissions and 14db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * limitations under the License. 15db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 16db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 176e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines#include "bcc/Assert.h" 18e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSTransforms.h" 197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao#include <cstdlib> 217a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 22b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DerivedTypes.h> 23b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Function.h> 24b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Instructions.h> 25b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/IRBuilder.h> 2618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser#include <llvm/IR/MDBuilder.h> 27b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Module.h> 28c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Pass.h> 297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines#include <llvm/Support/raw_ostream.h> 30b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DataLayout.h> 31cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser#include <llvm/IR/Function.h> 32b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Type.h> 33806075b3a54af826fea78490fb213d8a0784138eTobias Grosser#include <llvm/Transforms/Utils/BasicBlockUtils.h> 34c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang 35c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include "bcc/Config/Config.h" 36e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSInfo.h" 37ef73a242762bcd8113b9b65ceccbe7d909b5acbcZonr Chang#include "bcc/Support/Log.h" 38db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 39d88177580db4ddedf680854c51db333c97eabc59Stephen Hines#include "bcinfo/MetadataExtractor.h" 40d88177580db4ddedf680854c51db333c97eabc59Stephen Hines 417a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaousing namespace bcc; 427a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 43db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace { 447a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 459c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hinesstatic const bool gEnableRsTbaa = false; 469c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines 477a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao/* RSForEachExpandPass - This pass operates on functions that are able to be 487a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the 497a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * ForEach-able function to be invoked over the appropriate data cells of the 507a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * input/output allocations (adjusting other relevant parameters as we go). We 517a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * support doing this for any ForEach-able compute kernels. The new function 527a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * name is the original function name followed by ".expand". Note that we 537a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * still generate code for the original function. 547a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao */ 557a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoclass RSForEachExpandPass : public llvm::ModulePass { 567a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoprivate: 57db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static char ID; 58db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 59db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Module *M; 60db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::LLVMContext *C; 61db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 627a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao const RSInfo::ExportForeachFuncListTy &mFuncs; 63cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines 642b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Turns on optimization of allocation stride values. 652b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool mEnableStepOpt; 662b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 67cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines uint32_t getRootSignature(llvm::Function *F) { 68db169187dea4602e4ad32058762d23d474753fd0Stephen Hines const llvm::NamedMDNode *ExportForEachMetadata = 69db169187dea4602e4ad32058762d23d474753fd0Stephen Hines M->getNamedMetadata("#rs_export_foreach"); 70db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 71db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (!ExportForEachMetadata) { 72db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 8> RootArgTys; 73db169187dea4602e4ad32058762d23d474753fd0Stephen Hines for (llvm::Function::arg_iterator B = F->arg_begin(), 74db169187dea4602e4ad32058762d23d474753fd0Stephen Hines E = F->arg_end(); 75db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 76db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 77db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgTys.push_back(B->getType()); 78db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 79db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 80db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // For pre-ICS bitcode, we may not have signature information. In that 81db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // case, we use the size of the RootArgTys to select the number of 82db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // arguments. 83db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return (1 << RootArgTys.size()) - 1; 84db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 85db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ExportForEachMetadata->getNumOperands() == 0) { 877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return 0; 887ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 897ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 906e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(ExportForEachMetadata->getNumOperands() > 0); 91db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 92cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // We only handle the case for legacy root() functions here, so this is 93cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // hard-coded to look at only the first such function. 94db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0); 95db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigNode != NULL && SigNode->getNumOperands() == 1) { 96db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *SigVal = SigNode->getOperand(0); 97db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigVal->getValueID() == llvm::Value::MDStringVal) { 98db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::StringRef SigString = 99db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static_cast<llvm::MDString*>(SigVal)->getString(); 100db169187dea4602e4ad32058762d23d474753fd0Stephen Hines uint32_t Signature = 0; 101db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigString.getAsInteger(10, Signature)) { 102db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ALOGE("Non-integer signature value '%s'", SigString.str().c_str()); 103db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 104db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 105db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return Signature; 106db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 107db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 108db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 109db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 110db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 111db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1122b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Get the actual value we should use to step through an allocation. 1137b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 1147b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // Normally the value we use to step through an allocation is given to us by 1157b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // the driver. However, for certain primitive data types, we can derive an 1167b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // integer constant for the step value. We use this integer constant whenever 1177b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // possible to allow further compiler optimizations to take place. 1187b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 119b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines // DL - Target Data size/layout information. 1202b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // T - Type of allocation (should be a pointer). 1212b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // OrigStep - Original step increment (root.expand() input from driver). 122b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *T, 1232b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OrigStep) { 124b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines bccAssert(DL); 1252b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(T); 1262b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(OrigStep); 1272b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(T); 1282b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C); 1292b04086acbef6520ae2c54a868b1271abf053122Stephen Hines if (mEnableStepOpt && T != VoidPtrTy && PT) { 1302b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *ET = PT->getElementType(); 131b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines uint64_t ETSize = DL->getTypeAllocSize(ET); 1322b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 1332b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return llvm::ConstantInt::get(Int32Ty, ETSize); 1342b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } else { 1352b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return OrigStep; 1362b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1372b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1382b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 1398ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// @brief Returns the type of the ForEach stub parameter structure. 1408ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// 1418ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// Renderscript uses a single structure in which all parameters are passed 1428ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// to keep the signature of the expanded function independent of the 1438ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// parameters passed to it. 1448ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser llvm::Type *getForeachStubTy() { 145db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C); 146db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 147db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *SizeTy = Int32Ty; 148db169187dea4602e4ad32058762d23d474753fd0Stephen Hines /* Defined in frameworks/base/libs/rs/rs_hal.h: 149db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 150db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * struct RsForEachStubParamStruct { 151db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *in; 152db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * void *out; 153db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *usr; 154db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * size_t usr_len; 155db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t x; 156db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t y; 157db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t z; 158db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t lod; 159db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * enum RsAllocationCubemapFace face; 160db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t ar[16]; 161db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * }; 162db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 163db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 9> StructTys; 164db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // const void *in 165db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // void *out 166db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // const void *usr 167db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(SizeTy); // size_t usr_len 168db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t x 169db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t y 170db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t z 171db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t lod 172db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // enum RsAllocationCubemapFace 173db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(llvm::ArrayType::get(Int32Ty, 16)); // uint32_t ar[16] 174db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1758ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser return llvm::StructType::create(StructTys, "RsForEachStubParamStruct"); 1768ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 1778ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 178357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// @brief Create skeleton of the expanded function. 179357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 180357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// This creates a function with the following signature: 181357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 182357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 183357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// uint32_t instep, uint32_t outstep) 184357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 185357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) { 186357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Type *ForEachStubPtrTy = getForeachStubTy()->getPointerTo(); 187357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 188357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 189357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::SmallVector<llvm::Type*, 8> ParamTys; 190357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p 191357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t x1 192357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t x2 193357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t instep 194357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t outstep 195357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 196357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::FunctionType *FT = 197357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::FunctionType::get(llvm::Type::getVoidTy(*C), ParamTys, false); 198802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser llvm::Function *F = 199802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage, 200802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser OldName + ".expand", M); 201802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 202802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser llvm::Function::arg_iterator AI = F->arg_begin(); 203802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 204802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("p"); 205802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 206802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("x1"); 207802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 208802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("x2"); 209802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 210802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("arg_instep"); 211802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 212802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("arg_outstep"); 213802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 214802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 215802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser assert(AI == F->arg_end()); 216802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 217806075b3a54af826fea78490fb213d8a0784138eTobias Grosser llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*C, "Begin", F); 218806075b3a54af826fea78490fb213d8a0784138eTobias Grosser llvm::IRBuilder<> Builder(Begin); 219806075b3a54af826fea78490fb213d8a0784138eTobias Grosser Builder.CreateRetVoid(); 220806075b3a54af826fea78490fb213d8a0784138eTobias Grosser 221802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser return F; 222357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser } 223357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 224e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @brief Create an empty loop 225e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 226e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// Create a loop of the form: 227e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 228e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// for (i = LowerBound; i < UpperBound; i++) 229e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// ; 230e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 231e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// After the loop has been created, the builder is set such that 232e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// instructions can be added to the loop body. 233e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 234e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param Builder The builder to use to build this loop. The current 235e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// position of the builder is the position the loop 236e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// will be inserted. 237e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LowerBound The first value of the loop iterator 238e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param UpperBound The maximal value of the loop iterator 239e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LoopIV A reference that will be set to the loop iterator. 240e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @return The BasicBlock that will be executed after the loop. 241e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder, 242e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *LowerBound, 243e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *UpperBound, 244e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode **LoopIV) { 245e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser assert(LowerBound->getType() == UpperBound->getType()); 246e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 247e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB; 248e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *Cond, *IVNext; 249e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 250e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 251e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB = Builder.GetInsertBlock(); 252e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), this); 253e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser HeaderBB = llvm::BasicBlock::Create(*C, "Loop", CondBB->getParent()); 254e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 255e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (LowerBound < Upperbound) 256e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 257e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 258e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 259e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB->getTerminator()->eraseFromParent(); 260e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(CondBB); 261e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(LowerBound, UpperBound); 262e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 263e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 264e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ] 265e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv.next = iv + 1 266e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (iv.next < Upperbound) 267e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 268e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 269e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 270e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB); 271e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV = Builder.CreatePHI(LowerBound->getType(), 2, "X"); 272e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(LowerBound, CondBB); 273e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1)); 274e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(IVNext, HeaderBB); 275e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(IVNext, UpperBound); 276e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 277e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB->setName("Exit"); 278e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB->getFirstNonPHI()); 279e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser *LoopIV = IV; 280e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser return AfterBB; 281e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser } 282e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 2838ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosserpublic: 2848ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser RSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs, 2858ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser bool pEnableStepOpt) 2868ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser : ModulePass(ID), M(NULL), C(NULL), mFuncs(pForeachFuncs), 2878ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser mEnableStepOpt(pEnableStepOpt) { 2888ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 2898ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 2908ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /* Performs the actual optimization on a selected function. On success, the 2918ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * Module will contain a new function of the name "<NAME>.expand" that 2928ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * invokes <NAME>() in a loop with the appropriate parameters. 2938ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser */ 2948ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser bool ExpandFunction(llvm::Function *F, uint32_t Signature) { 2958ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser ALOGV("Expanding ForEach-able Function %s", F->getName().str().c_str()); 2968ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 2978ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 2988ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser Signature = getRootSignature(F); 2998ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 3008ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // We couldn't determine how to expand this function based on its 3018ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // function signature. 3028ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser return false; 3038ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3048ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3058ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 3068ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser llvm::DataLayout DL(M); 3078ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 308357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *ExpandedFunc = createEmptyExpandedFunction(F->getName()); 309db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 310db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Create and name the actual arguments to this expanded function. 311db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Argument*, 8> ArgVec; 312db169187dea4602e4ad32058762d23d474753fd0Stephen Hines for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(), 313db169187dea4602e4ad32058762d23d474753fd0Stephen Hines E = ExpandedFunc->arg_end(); 314db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 315db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 316db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ArgVec.push_back(B); 317db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 318db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 319db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (ArgVec.size() != 5) { 32089e8490c80505468f2b816ca9d12fefa53f05959Shih-wei Liao ALOGE("Incorrect number of arguments to function: %zu", 32189e8490c80505468f2b816ca9d12fefa53f05959Shih-wei Liao ArgVec.size()); 322db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return false; 323db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 324db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_p = ArgVec[0]; 325db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_x1 = ArgVec[1]; 326db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_x2 = ArgVec[2]; 327db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_instep = ArgVec[3]; 328db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_outstep = ArgVec[4]; 329db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3302b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *InStep = NULL; 3312b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OutStep = NULL; 332db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 333db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Construct the actual function body. 334e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::IRBuilder<> Builder(ExpandedFunc->getEntryBlock().begin()); 335db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 336cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Collect and construct the arguments for the kernel(). 337db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 338db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Function::arg_iterator Args = F->arg_begin(); 339db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 340db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *InTy = NULL; 341ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser llvm::Value *InBasePtr = NULL; 342d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) { 343db169187dea4602e4ad32058762d23d474753fd0Stephen Hines InTy = Args->getType(); 344b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines InStep = getStepValue(&DL, InTy, Arg_instep); 3452b04086acbef6520ae2c54a868b1271abf053122Stephen Hines InStep->setName("instep"); 346ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 0)); 347db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 348db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 349db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 350db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *OutTy = NULL; 35102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser llvm::Value *OutBasePtr = NULL; 352d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 353db169187dea4602e4ad32058762d23d474753fd0Stephen Hines OutTy = Args->getType(); 354b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 3552b04086acbef6520ae2c54a868b1271abf053122Stephen Hines OutStep->setName("outstep"); 35602f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 1)); 357db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 358db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 359db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 360db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *UsrData = NULL; 361d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)) { 362db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *UsrDataTy = Args->getType(); 363db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData = Builder.CreatePointerCast(Builder.CreateLoad( 364db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 2)), UsrDataTy); 365db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData->setName("UsrData"); 366db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 367db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 368db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 369d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 370db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 371db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 372db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 373db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Y = NULL; 374d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 375db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 376db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 377db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 378db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3796e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(Args == F->arg_end()); 380db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 381e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 382e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser createLoop(Builder, Arg_x1, Arg_x2, &IV); 383db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 384cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Populate the actual call to kernel(). 385db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 386db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InPtr = NULL; 3887ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutPtr = NULL; 389db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 390ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // Calculate the current input and output pointers 39102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 392ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser // We always calculate the input/output pointers with a GEP operating on i8 39302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // values and only cast at the very end to OutTy. This is because the step 39402f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // between two values is given in bytes. 39502f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // 39602f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // TODO: We could further optimize the output by using a GEP operation of 39702f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser // type 'OutTy' in cases where the element type of the allocation allows. 39802f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutBasePtr) { 39902f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 40002f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutOffset = Builder.CreateMul(OutOffset, OutStep); 40102f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 40202f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 40302f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser } 404ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InBasePtr) { 405ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1); 406ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InOffset = Builder.CreateMul(InOffset, InStep); 407ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreateGEP(InBasePtr, InOffset); 408ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser InPtr = Builder.CreatePointerCast(InPtr, InTy); 409ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser } 41002f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser 411ae937ec9658c219a38d9d631d2569f19e21b50c0Tobias Grosser if (InPtr) { 4127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(InPtr); 413db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 414db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 41502f3cd69a4103f91a51a2f988f2179c885d734b5Tobias Grosser if (OutPtr) { 4167ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(OutPtr); 417db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 418db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 419db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (UsrData) { 420db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(UsrData); 421db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 422db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 423e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *X = IV; 424d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 425db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(X); 426db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 427db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 428db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (Y) { 429db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(Y); 430db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 431db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 432db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateCall(F, RootArgs); 433db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 4347ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return true; 4357ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4367ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4377ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Expand a pass-by-value kernel. 4387ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 4397ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bool ExpandKernel(llvm::Function *F, uint32_t Signature) { 440d88177580db4ddedf680854c51db333c97eabc59Stephen Hines bccAssert(bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)); 4417ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ALOGV("Expanding kernel Function %s", F->getName().str().c_str()); 4427ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // TODO: Refactor this to share functionality with ExpandFunction. 444b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines llvm::DataLayout DL(M); 4457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 446357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *ExpandedFunc = createEmptyExpandedFunction(F->getName()); 4477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Create and name the actual arguments to this expanded function. 4497ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Argument*, 8> ArgVec; 4507ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(), 4517ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines E = ExpandedFunc->arg_end(); 4527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines B != E; 4537ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ++B) { 4547ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ArgVec.push_back(B); 4557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4567ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4577ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ArgVec.size() != 5) { 4587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ALOGE("Incorrect number of arguments to function: %zu", 4597ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ArgVec.size()); 4607ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return false; 4617ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4627ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_p = ArgVec[0]; 4637ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_x1 = ArgVec[1]; 4647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_x2 = ArgVec[2]; 4657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_instep = ArgVec[3]; 4667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_outstep = ArgVec[4]; 4677ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InStep = NULL; 4697ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutStep = NULL; 4707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4717ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Construct the actual function body. 472e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::IRBuilder<> Builder(ExpandedFunc->getEntryBlock().begin()); 4737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 47418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Create TBAA meta-data. 47518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDNode *TBAARenderScript, *TBAAAllocation, *TBAAPointer; 47618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 47718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDBuilder MDHelper(*C); 47818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAARenderScript = MDHelper.createTBAARoot("RenderScript TBAA"); 47918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAAAllocation = MDHelper.createTBAANode("allocation", TBAARenderScript); 48018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAAPointer = MDHelper.createTBAANode("pointer", TBAARenderScript); 48118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 4827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Collect and construct the arguments for the kernel(). 4837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 4847ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Function::arg_iterator Args = F->arg_begin(); 4857ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 48674a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::Type *OutTy = NULL; 48774a4b08235990916911b8fe758d656c1171faf26Stephen Hines bool PassOutByReference = false; 48818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::LoadInst *OutBasePtr = NULL; 489d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 49074a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::Type *OutBaseTy = F->getReturnType(); 49174a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (OutBaseTy->isVoidTy()) { 49274a4b08235990916911b8fe758d656c1171faf26Stephen Hines PassOutByReference = true; 49374a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutTy = Args->getType(); 49474a4b08235990916911b8fe758d656c1171faf26Stephen Hines Args++; 49574a4b08235990916911b8fe758d656c1171faf26Stephen Hines } else { 49674a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutTy = OutBaseTy->getPointerTo(); 49774a4b08235990916911b8fe758d656c1171faf26Stephen Hines // We don't increment Args, since we are using the actual return type. 49874a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 499b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 50074a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutStep->setName("outstep"); 5017b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser OutBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 1)); 5029c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 5039c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines OutBasePtr->setMetadata("tbaa", TBAAPointer); 5049c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 50574a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 50674a4b08235990916911b8fe758d656c1171faf26Stephen Hines 5077ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *InBaseTy = NULL; 5087ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *InTy = NULL; 50918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::LoadInst *InBasePtr = NULL; 510d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) { 5117ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InBaseTy = Args->getType(); 5127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InTy =InBaseTy->getPointerTo(); 513b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines InStep = getStepValue(&DL, InTy, Arg_instep); 5147ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InStep->setName("instep"); 5154102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser InBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 0)); 5169c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 5179c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines InBasePtr->setMetadata("tbaa", TBAAPointer); 5189c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 5197ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5207ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5217ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5227ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // No usrData parameter on kernels. 523d88177580db4ddedf680854c51db333c97eabc59Stephen Hines bccAssert( 524d88177580db4ddedf680854c51db333c97eabc59Stephen Hines !bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)); 5257ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 526d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 5277ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5307ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Y = NULL; 531d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 5327ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 5337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5347ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5357ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5367ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bccAssert(Args == F->arg_end()); 5377ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 538e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 539e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser createLoop(Builder, Arg_x1, Arg_x2, &IV); 5407ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5417ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Populate the actual call to kernel(). 5427ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 5437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InPtr = NULL; 5457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutPtr = NULL; 5467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5474102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser // Calculate the current input and output pointers 5487b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 5494102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser // We always calculate the input/output pointers with a GEP operating on i8 5507b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // values and only cast at the very end to OutTy. This is because the step 5517b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // between two values is given in bytes. 5527b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // 5537b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // TODO: We could further optimize the output by using a GEP operation of 5547b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser // type 'OutTy' in cases where the element type of the allocation allows. 5557b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser if (OutBasePtr) { 5567b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 5577b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser OutOffset = Builder.CreateMul(OutOffset, OutStep); 5587b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 5597b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 5607b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser } 5614102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser if (InBasePtr) { 5624102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1); 5634102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser InOffset = Builder.CreateMul(InOffset, InStep); 5644102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser InPtr = Builder.CreateGEP(InBasePtr, InOffset); 5654102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser InPtr = Builder.CreatePointerCast(InPtr, InTy); 5664102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser } 5677b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser 56874a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (PassOutByReference) { 56974a4b08235990916911b8fe758d656c1171faf26Stephen Hines RootArgs.push_back(OutPtr); 57074a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 57174a4b08235990916911b8fe758d656c1171faf26Stephen Hines 5724102bec56151fb5d9c962fb298412f34a6eacaa8Tobias Grosser if (InPtr) { 57318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::LoadInst *In = Builder.CreateLoad(InPtr, "In"); 5749c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 5759c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines In->setMetadata("tbaa", TBAAAllocation); 5769c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 5777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(In); 5787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5797ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 580e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *X = IV; 581d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 5827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(X); 5837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5847ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5857ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (Y) { 5867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(Y); 5877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5887ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5897ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *RetVal = Builder.CreateCall(F, RootArgs); 5907ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5917b6629024395bf085ca64fcd5c69d2ded95e8d18Tobias Grosser if (OutPtr && !PassOutByReference) { 59218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr); 5939c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa) { 5949c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines Store->setMetadata("tbaa", TBAAAllocation); 5959c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines } 5967ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5977ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 598db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return true; 599db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 600db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 60118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Checks if pointers to allocation internals are exposed 60218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 60318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// This function verifies if through the parameters passed to the kernel 60418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// or through calls to the runtime library the script gains access to 60518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers pointing to data within a RenderScript Allocation. 60618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// If we know we control all loads from and stores to data within 60718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// RenderScript allocations and if we know the run-time internal accesses 60818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// are all annotated with RenderScript TBAA metadata, only then we 60918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// can safely use TBAA to distinguish between generic and from-allocation 61018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// pointers. 61118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser bool allocPointersExposed(llvm::Module &M) { 61218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Old style kernel function can expose pointers to elements within 61318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // allocations. 61418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // TODO: Extend analysis to allow simple cases of old-style kernels. 61518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser for (RSInfo::ExportForeachFuncListTy::const_iterator 61618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser func_iter = mFuncs.begin(), func_end = mFuncs.end(); 61718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser func_iter != func_end; func_iter++) { 61818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser const char *Name = func_iter->first; 61918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser uint32_t Signature = func_iter->second; 620d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (M.getFunction(Name) && 621d88177580db4ddedf680854c51db333c97eabc59Stephen Hines !bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)) { 62218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 62318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 62418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 62518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 62618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // Check for library functions that expose a pointer to an Allocation or 62718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // that are not yet annotated with RenderScript-specific tbaa information. 62818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser static std::vector<std::string> Funcs; 62918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 63018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAt(...) 63118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationj"); 63218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationjj"); 63318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsGetElementAt13rs_allocationjjj"); 63418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsSetElementAt() 63518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvj"); 63618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjj"); 63718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjjj"); 63818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_Y() 63918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_Y13rs_allocationjj"); 64018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_U() 64118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_U13rs_allocationjj"); 64218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser // rsGetElementAtYuv_uchar_V() 64318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser Funcs.push_back("_Z25rsGetElementAtYuv_uchar_V13rs_allocationjj"); 64418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 64518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser for (std::vector<std::string>::iterator FI = Funcs.begin(), 64618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser FE = Funcs.end(); 64718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser FI != FE; ++FI) { 64818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::Function *F = M.getFunction(*FI); 64918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 65018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser if (!F) { 65118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser ALOGE("Missing run-time function '%s'", FI->c_str()); 65218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 65318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 65418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 65518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser if (F->getNumUses() > 0) { 65618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return true; 65718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 65818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 65918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 66018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser return false; 66118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 66218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 66318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// @brief Connect RenderScript TBAA metadata to C/C++ metadata 66418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// 66518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// The TBAA metadata used to annotate loads/stores from RenderScript 66618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// Allocations is generated in a separate TBAA tree with a "RenderScript TBAA" 66718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// root node. LLVM does assume may-alias for all nodes in unrelated alias 66818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// analysis trees. This function makes the RenderScript TBAA a subtree of the 66918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// normal C/C++ TBAA tree aside of normal C/C++ types. With the connected trees 67018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// every access to an Allocation is resolved to must-alias if compared to 67118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser /// a normal C/C++ access. 67218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser void connectRenderScriptTBAAMetadata(llvm::Module &M) { 67318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDBuilder MDHelper(*C); 67418a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDNode *TBAARenderScript = MDHelper.createTBAARoot("RenderScript TBAA"); 67518a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 67618a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDNode *TBAARoot = MDHelper.createTBAARoot("Simple C/C++ TBAA"); 67718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser llvm::MDNode *TBAAMergedRS = MDHelper.createTBAANode("RenderScript", TBAARoot); 67818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 67918a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser TBAARenderScript->replaceAllUsesWith(TBAAMergedRS); 68018a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 68118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 682db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual bool runOnModule(llvm::Module &M) { 683cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines bool Changed = false; 684db169187dea4602e4ad32058762d23d474753fd0Stephen Hines this->M = &M; 685db169187dea4602e4ad32058762d23d474753fd0Stephen Hines C = &M.getContext(); 686db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 68718a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser bool AllocsExposed = allocPointersExposed(M); 68818a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 6897a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao for (RSInfo::ExportForeachFuncListTy::const_iterator 6907a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao func_iter = mFuncs.begin(), func_end = mFuncs.end(); 6917a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao func_iter != func_end; func_iter++) { 6927a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao const char *name = func_iter->first; 6937a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao uint32_t signature = func_iter->second; 6947a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao llvm::Function *kernel = M.getFunction(name); 695cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser if (kernel) { 696d88177580db4ddedf680854c51db333c97eabc59Stephen Hines if (bcinfo::MetadataExtractor::hasForEachSignatureKernel(signature)) { 697cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandKernel(kernel, signature); 698acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 699acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else if (kernel->getReturnType()->isVoidTy()) { 700cd5b657c39a348bd7652a19c5be5035e9df44a42Tobias Grosser Changed |= ExpandFunction(kernel, signature); 701acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 702acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } else { 703acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // There are some graphics root functions that are not 704acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // expanded, but that will be called directly. For those 705acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser // functions, we can not set the linkage to internal. 706acde6013e9c448547e59eed04afd2adbd9681a3aTobias Grosser } 707cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines } 708db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 709db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7109c5263e00d89dfdd25d3a1706eb319e8529ec604Stephen Hines if (gEnableRsTbaa && !AllocsExposed) { 71118a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser connectRenderScriptTBAAMetadata(M); 71218a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser } 71318a38a3fc6fad8355891b771dd3c6537fa8699ecTobias Grosser 714cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines return Changed; 715db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 716db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 717db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual const char *getPassName() const { 718db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return "ForEach-able Function Expansion"; 719db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 720db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7217a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}; // end RSForEachExpandPass 722db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7237a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end anonymous namespace 7247a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 7257a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaochar RSForEachExpandPass::ID = 0; 726db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 727db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace bcc { 728db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7297a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaollvm::ModulePass * 7302b04086acbef6520ae2c54a868b1271abf053122Stephen HinescreateRSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs, 7312b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool pEnableStepOpt){ 7322b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return new RSForEachExpandPass(pForeachFuncs, pEnableStepOpt); 7337a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} 734db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 7357a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end namespace bcc 736