RSKernelExpand.cpp revision e87a0518647d1f9c5249d6990c67737e0fb579e9
1db169187dea4602e4ad32058762d23d474753fd0Stephen Hines/* 2db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Copyright 2012, The Android Open Source Project 3db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 4db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Licensed under the Apache License, Version 2.0 (the "License"); 5db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * you may not use this file except in compliance with the License. 6db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * You may obtain a copy of the License at 7db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 8db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * http://www.apache.org/licenses/LICENSE-2.0 9db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 10db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Unless required by applicable law or agreed to in writing, software 11db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * distributed under the License is distributed on an "AS IS" BASIS, 12db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * See the License for the specific language governing permissions and 14db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * limitations under the License. 15db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 16db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 176e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines#include "bcc/Assert.h" 18e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSTransforms.h" 197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao#include <cstdlib> 217a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 22b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DerivedTypes.h> 23b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Function.h> 24b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Instructions.h> 25b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/IRBuilder.h> 26b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Module.h> 27c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Pass.h> 287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines#include <llvm/Support/raw_ostream.h> 29b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/DataLayout.h> 30b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines#include <llvm/IR/Type.h> 31806075b3a54af826fea78490fb213d8a0784138eTobias Grosser#include <llvm/Transforms/Utils/BasicBlockUtils.h> 32c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang 33c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include "bcc/Config/Config.h" 34e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSInfo.h" 35ef73a242762bcd8113b9b65ceccbe7d909b5acbcZonr Chang#include "bcc/Support/Log.h" 36db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 377a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaousing namespace bcc; 387a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 39db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace { 407a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 417a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao/* RSForEachExpandPass - This pass operates on functions that are able to be 427a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the 437a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * ForEach-able function to be invoked over the appropriate data cells of the 447a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * input/output allocations (adjusting other relevant parameters as we go). We 457a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * support doing this for any ForEach-able compute kernels. The new function 467a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * name is the original function name followed by ".expand". Note that we 477a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * still generate code for the original function. 487a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao */ 497a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoclass RSForEachExpandPass : public llvm::ModulePass { 507a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoprivate: 51db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static char ID; 52db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 53db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Module *M; 54db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::LLVMContext *C; 55db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 567a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao const RSInfo::ExportForeachFuncListTy &mFuncs; 57cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines 582b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Turns on optimization of allocation stride values. 592b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool mEnableStepOpt; 602b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 61cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines uint32_t getRootSignature(llvm::Function *F) { 62db169187dea4602e4ad32058762d23d474753fd0Stephen Hines const llvm::NamedMDNode *ExportForEachMetadata = 63db169187dea4602e4ad32058762d23d474753fd0Stephen Hines M->getNamedMetadata("#rs_export_foreach"); 64db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 65db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (!ExportForEachMetadata) { 66db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 8> RootArgTys; 67db169187dea4602e4ad32058762d23d474753fd0Stephen Hines for (llvm::Function::arg_iterator B = F->arg_begin(), 68db169187dea4602e4ad32058762d23d474753fd0Stephen Hines E = F->arg_end(); 69db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 70db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 71db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgTys.push_back(B->getType()); 72db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 73db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 74db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // For pre-ICS bitcode, we may not have signature information. In that 75db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // case, we use the size of the RootArgTys to select the number of 76db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // arguments. 77db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return (1 << RootArgTys.size()) - 1; 78db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 79db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 807ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ExportForEachMetadata->getNumOperands() == 0) { 817ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return 0; 827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 846e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(ExportForEachMetadata->getNumOperands() > 0); 85db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 86cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // We only handle the case for legacy root() functions here, so this is 87cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // hard-coded to look at only the first such function. 88db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0); 89db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigNode != NULL && SigNode->getNumOperands() == 1) { 90db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *SigVal = SigNode->getOperand(0); 91db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigVal->getValueID() == llvm::Value::MDStringVal) { 92db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::StringRef SigString = 93db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static_cast<llvm::MDString*>(SigVal)->getString(); 94db169187dea4602e4ad32058762d23d474753fd0Stephen Hines uint32_t Signature = 0; 95db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigString.getAsInteger(10, Signature)) { 96db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ALOGE("Non-integer signature value '%s'", SigString.str().c_str()); 97db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 98db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 99db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return Signature; 100db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 101db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 102db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 103db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 104db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 105db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1062b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Get the actual value we should use to step through an allocation. 107b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines // DL - Target Data size/layout information. 1082b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // T - Type of allocation (should be a pointer). 1092b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // OrigStep - Original step increment (root.expand() input from driver). 110b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *T, 1112b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OrigStep) { 112b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines bccAssert(DL); 1132b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(T); 1142b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(OrigStep); 1152b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(T); 1162b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C); 1172b04086acbef6520ae2c54a868b1271abf053122Stephen Hines if (mEnableStepOpt && T != VoidPtrTy && PT) { 1182b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *ET = PT->getElementType(); 119b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines uint64_t ETSize = DL->getTypeAllocSize(ET); 1202b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 1212b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return llvm::ConstantInt::get(Int32Ty, ETSize); 1222b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } else { 1232b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return OrigStep; 1242b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1252b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1262b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 127db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasIn(uint32_t Signature) { 1287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x01; 129db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 130db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 131db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasOut(uint32_t Signature) { 1327ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x02; 133db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 134db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 135db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasUsrData(uint32_t Signature) { 1367ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x04; 137db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 138db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 139db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasX(uint32_t Signature) { 1407ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x08; 141db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 142db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 143db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasY(uint32_t Signature) { 1447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x10; 1457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 1467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 1477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines static bool isKernel(uint32_t Signature) { 1487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x20; 149db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 150db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1518ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// @brief Returns the type of the ForEach stub parameter structure. 1528ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// 1538ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// Renderscript uses a single structure in which all parameters are passed 1548ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// to keep the signature of the expanded function independent of the 1558ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /// parameters passed to it. 1568ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser llvm::Type *getForeachStubTy() { 157db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C); 158db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 159db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *SizeTy = Int32Ty; 160db169187dea4602e4ad32058762d23d474753fd0Stephen Hines /* Defined in frameworks/base/libs/rs/rs_hal.h: 161db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 162db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * struct RsForEachStubParamStruct { 163db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *in; 164db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * void *out; 165db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *usr; 166db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * size_t usr_len; 167db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t x; 168db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t y; 169db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t z; 170db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t lod; 171db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * enum RsAllocationCubemapFace face; 172db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t ar[16]; 173db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * }; 174db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 175db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 9> StructTys; 176db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // const void *in 177db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // void *out 178db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // const void *usr 179db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(SizeTy); // size_t usr_len 180db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t x 181db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t y 182db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t z 183db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t lod 184db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // enum RsAllocationCubemapFace 185db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(llvm::ArrayType::get(Int32Ty, 16)); // uint32_t ar[16] 186db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1878ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser return llvm::StructType::create(StructTys, "RsForEachStubParamStruct"); 1888ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 1898ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 190357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// @brief Create skeleton of the expanded function. 191357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 192357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// This creates a function with the following signature: 193357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 194357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 195357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// uint32_t instep, uint32_t outstep) 196357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser /// 197357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) { 198357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Type *ForEachStubPtrTy = getForeachStubTy()->getPointerTo(); 199357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 200357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 201357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::SmallVector<llvm::Type*, 8> ParamTys; 202357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p 203357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t x1 204357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t x2 205357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t instep 206357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser ParamTys.push_back(Int32Ty); // uint32_t outstep 207357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 208357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::FunctionType *FT = 209357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::FunctionType::get(llvm::Type::getVoidTy(*C), ParamTys, false); 210802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser llvm::Function *F = 211802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage, 212802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser OldName + ".expand", M); 213802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 214802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser llvm::Function::arg_iterator AI = F->arg_begin(); 215802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 216802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("p"); 217802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 218802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("x1"); 219802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 220802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("x2"); 221802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 222802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("arg_instep"); 223802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 224802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI->setName("arg_outstep"); 225802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser AI++; 226802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 227802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser assert(AI == F->arg_end()); 228802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser 229806075b3a54af826fea78490fb213d8a0784138eTobias Grosser llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*C, "Begin", F); 230806075b3a54af826fea78490fb213d8a0784138eTobias Grosser llvm::IRBuilder<> Builder(Begin); 231806075b3a54af826fea78490fb213d8a0784138eTobias Grosser Builder.CreateRetVoid(); 232806075b3a54af826fea78490fb213d8a0784138eTobias Grosser 233802f65931852d925bbe2e478bafe422b4002e7c4Tobias Grosser return F; 234357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser } 235357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser 236e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @brief Create an empty loop 237e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 238e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// Create a loop of the form: 239e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 240e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// for (i = LowerBound; i < UpperBound; i++) 241e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// ; 242e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 243e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// After the loop has been created, the builder is set such that 244e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// instructions can be added to the loop body. 245e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// 246e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param Builder The builder to use to build this loop. The current 247e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// position of the builder is the position the loop 248e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// will be inserted. 249e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LowerBound The first value of the loop iterator 250e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param UpperBound The maximal value of the loop iterator 251e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @param LoopIV A reference that will be set to the loop iterator. 252e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser /// @return The BasicBlock that will be executed after the loop. 253e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder, 254e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *LowerBound, 255e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *UpperBound, 256e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode **LoopIV) { 257e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser assert(LowerBound->getType() == UpperBound->getType()); 258e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 259e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB; 260e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *Cond, *IVNext; 261e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 262e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 263e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB = Builder.GetInsertBlock(); 264e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), this); 265e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser HeaderBB = llvm::BasicBlock::Create(*C, "Loop", CondBB->getParent()); 266e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 267e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (LowerBound < Upperbound) 268e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 269e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 270e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 271e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser CondBB->getTerminator()->eraseFromParent(); 272e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(CondBB); 273e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(LowerBound, UpperBound); 274e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 275e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 276e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ] 277e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // iv.next = iv + 1 278e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // if (iv.next < Upperbound) 279e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto LoopHeader 280e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // else 281e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser // goto AfterBB 282e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB); 283e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV = Builder.CreatePHI(LowerBound->getType(), 2, "X"); 284e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(LowerBound, CondBB); 285e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1)); 286e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser IV->addIncoming(IVNext, HeaderBB); 287e87a0518647d1f9c5249d6990c67737e0fb579e9Tobias Grosser Cond = Builder.CreateICmpULT(IVNext, UpperBound); 288e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 289e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser AfterBB->setName("Exit"); 290e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser Builder.SetInsertPoint(HeaderBB->getFirstNonPHI()); 291e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser *LoopIV = IV; 292e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser return AfterBB; 293e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser } 294e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser 2958ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosserpublic: 2968ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser RSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs, 2978ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser bool pEnableStepOpt) 2988ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser : ModulePass(ID), M(NULL), C(NULL), mFuncs(pForeachFuncs), 2998ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser mEnableStepOpt(pEnableStepOpt) { 3008ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3018ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 3028ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser /* Performs the actual optimization on a selected function. On success, the 3038ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * Module will contain a new function of the name "<NAME>.expand" that 3048ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser * invokes <NAME>() in a loop with the appropriate parameters. 3058ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser */ 3068ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser bool ExpandFunction(llvm::Function *F, uint32_t Signature) { 3078ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser ALOGV("Expanding ForEach-able Function %s", F->getName().str().c_str()); 3088ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 3098ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 3108ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser Signature = getRootSignature(F); 3118ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser if (!Signature) { 3128ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // We couldn't determine how to expand this function based on its 3138ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser // function signature. 3148ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser return false; 3158ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3168ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser } 3178ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 3188ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser llvm::DataLayout DL(M); 3198ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser 3208ae4607d2dc5bc655d0c9225565d36ce2ebfc798Tobias Grosser llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 321357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *ExpandedFunc = createEmptyExpandedFunction(F->getName()); 322db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 323db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Create and name the actual arguments to this expanded function. 324db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Argument*, 8> ArgVec; 325db169187dea4602e4ad32058762d23d474753fd0Stephen Hines for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(), 326db169187dea4602e4ad32058762d23d474753fd0Stephen Hines E = ExpandedFunc->arg_end(); 327db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 328db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 329db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ArgVec.push_back(B); 330db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 331db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 332db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (ArgVec.size() != 5) { 33389e8490c80505468f2b816ca9d12fefa53f05959Shih-wei Liao ALOGE("Incorrect number of arguments to function: %zu", 33489e8490c80505468f2b816ca9d12fefa53f05959Shih-wei Liao ArgVec.size()); 335db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return false; 336db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 337db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_p = ArgVec[0]; 338db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_x1 = ArgVec[1]; 339db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_x2 = ArgVec[2]; 340db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_instep = ArgVec[3]; 341db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_outstep = ArgVec[4]; 342db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3432b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *InStep = NULL; 3442b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OutStep = NULL; 345db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 346db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Construct the actual function body. 347e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::IRBuilder<> Builder(ExpandedFunc->getEntryBlock().begin()); 348db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 349cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Collect and construct the arguments for the kernel(). 350db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 351db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Function::arg_iterator Args = F->arg_begin(); 352db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 353db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *InTy = NULL; 354db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::AllocaInst *AIn = NULL; 355db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasIn(Signature)) { 356db169187dea4602e4ad32058762d23d474753fd0Stephen Hines InTy = Args->getType(); 357db169187dea4602e4ad32058762d23d474753fd0Stephen Hines AIn = Builder.CreateAlloca(InTy, 0, "AIn"); 358b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines InStep = getStepValue(&DL, InTy, Arg_instep); 3592b04086acbef6520ae2c54a868b1271abf053122Stephen Hines InStep->setName("instep"); 360db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 361db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 0)), InTy), AIn); 362db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 363db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 364db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 365db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *OutTy = NULL; 366db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::AllocaInst *AOut = NULL; 367db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasOut(Signature)) { 368db169187dea4602e4ad32058762d23d474753fd0Stephen Hines OutTy = Args->getType(); 369db169187dea4602e4ad32058762d23d474753fd0Stephen Hines AOut = Builder.CreateAlloca(OutTy, 0, "AOut"); 370b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 3712b04086acbef6520ae2c54a868b1271abf053122Stephen Hines OutStep->setName("outstep"); 372db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 373db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 1)), OutTy), AOut); 374db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 375db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 376db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 377db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *UsrData = NULL; 378db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasUsrData(Signature)) { 379db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *UsrDataTy = Args->getType(); 380db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData = Builder.CreatePointerCast(Builder.CreateLoad( 381db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 2)), UsrDataTy); 382db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData->setName("UsrData"); 383db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 384db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 385db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 386db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasX(Signature)) { 387db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 388db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 389db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 390db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Y = NULL; 391db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasY(Signature)) { 392db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 393db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 394db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 395db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3966e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(Args == F->arg_end()); 397db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 398e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 399e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser createLoop(Builder, Arg_x1, Arg_x2, &IV); 400db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 401cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Populate the actual call to kernel(). 402db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 403db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 4047ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InPtr = NULL; 4057ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutPtr = NULL; 406db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 407db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (AIn) { 4087ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InPtr = Builder.CreateLoad(AIn, "InPtr"); 4097ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(InPtr); 410db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 411db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 412db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (AOut) { 4137ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines OutPtr = Builder.CreateLoad(AOut, "OutPtr"); 4147ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(OutPtr); 415db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 416db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 417db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (UsrData) { 418db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(UsrData); 419db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 420db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 421e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *X = IV; 422db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasX(Signature)) { 423db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(X); 424db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 425db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 426db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (Y) { 427db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(Y); 428db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 429db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 430db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateCall(F, RootArgs); 431db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 4327ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (InPtr) { 4337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // InPtr += instep 434db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *NewIn = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 4357ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(InPtr, Int32Ty), InStep), InTy); 436db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(NewIn, AIn); 437db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 438db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 4397ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (OutPtr) { 4407ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // OutPtr += outstep 441db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *NewOut = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 4427ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(OutPtr, Int32Ty), OutStep), OutTy); 4437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(NewOut, AOut); 4447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return true; 4477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4497ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Expand a pass-by-value kernel. 4507ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 4517ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bool ExpandKernel(llvm::Function *F, uint32_t Signature) { 4527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bccAssert(isKernel(Signature)); 4537ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ALOGV("Expanding kernel Function %s", F->getName().str().c_str()); 4547ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // TODO: Refactor this to share functionality with ExpandFunction. 456b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines llvm::DataLayout DL(M); 4577ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 459357b58691936bef425bd315c13a2d8019d7e9c7eTobias Grosser llvm::Function *ExpandedFunc = createEmptyExpandedFunction(F->getName()); 4607ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4617ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Create and name the actual arguments to this expanded function. 4627ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Argument*, 8> ArgVec; 4637ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(), 4647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines E = ExpandedFunc->arg_end(); 4657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines B != E; 4667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ++B) { 4677ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ArgVec.push_back(B); 4687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4697ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ArgVec.size() != 5) { 4717ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ALOGE("Incorrect number of arguments to function: %zu", 4727ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ArgVec.size()); 4737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return false; 4747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4757ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_p = ArgVec[0]; 4767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_x1 = ArgVec[1]; 4777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_x2 = ArgVec[2]; 4787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_instep = ArgVec[3]; 4797ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_outstep = ArgVec[4]; 4807ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4817ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InStep = NULL; 4827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutStep = NULL; 4837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4847ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Construct the actual function body. 485e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::IRBuilder<> Builder(ExpandedFunc->getEntryBlock().begin()); 4867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Collect and construct the arguments for the kernel(). 4887ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 4897ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Function::arg_iterator Args = F->arg_begin(); 4907ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 49174a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::Type *OutTy = NULL; 49274a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::AllocaInst *AOut = NULL; 49374a4b08235990916911b8fe758d656c1171faf26Stephen Hines bool PassOutByReference = false; 49474a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (hasOut(Signature)) { 49574a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::Type *OutBaseTy = F->getReturnType(); 49674a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (OutBaseTy->isVoidTy()) { 49774a4b08235990916911b8fe758d656c1171faf26Stephen Hines PassOutByReference = true; 49874a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutTy = Args->getType(); 49974a4b08235990916911b8fe758d656c1171faf26Stephen Hines Args++; 50074a4b08235990916911b8fe758d656c1171faf26Stephen Hines } else { 50174a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutTy = OutBaseTy->getPointerTo(); 50274a4b08235990916911b8fe758d656c1171faf26Stephen Hines // We don't increment Args, since we are using the actual return type. 50374a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 50474a4b08235990916911b8fe758d656c1171faf26Stephen Hines AOut = Builder.CreateAlloca(OutTy, 0, "AOut"); 505b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines OutStep = getStepValue(&DL, OutTy, Arg_outstep); 50674a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutStep->setName("outstep"); 50774a4b08235990916911b8fe758d656c1171faf26Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 50874a4b08235990916911b8fe758d656c1171faf26Stephen Hines Builder.CreateStructGEP(Arg_p, 1)), OutTy), AOut); 50974a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 51074a4b08235990916911b8fe758d656c1171faf26Stephen Hines 5117ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *InBaseTy = NULL; 5127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *InTy = NULL; 5137ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::AllocaInst *AIn = NULL; 5147ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasIn(Signature)) { 5157ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InBaseTy = Args->getType(); 5167ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InTy =InBaseTy->getPointerTo(); 5177ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines AIn = Builder.CreateAlloca(InTy, 0, "AIn"); 518b730e239619a546d93e5926ea92d698ab77ec7f6Stephen Hines InStep = getStepValue(&DL, InTy, Arg_instep); 5197ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InStep->setName("instep"); 5207ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 5217ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStructGEP(Arg_p, 0)), InTy), AIn); 5227ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5237ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5247ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5257ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // No usrData parameter on kernels. 5267ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bccAssert(!hasUsrData(Signature)); 5277ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasX(Signature)) { 5297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5307ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5317ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5327ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Y = NULL; 5337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasY(Signature)) { 5347ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 5357ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5367ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5377ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5387ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bccAssert(Args == F->arg_end()); 5397ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 540e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::PHINode *IV; 541e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser createLoop(Builder, Arg_x1, Arg_x2, &IV); 5427ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Populate the actual call to kernel(). 5447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 5457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InPtr = NULL; 5477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *In = NULL; 5487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutPtr = NULL; 5497ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 55074a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (PassOutByReference) { 55174a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutPtr = Builder.CreateLoad(AOut, "OutPtr"); 55274a4b08235990916911b8fe758d656c1171faf26Stephen Hines RootArgs.push_back(OutPtr); 55374a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 55474a4b08235990916911b8fe758d656c1171faf26Stephen Hines 5557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (AIn) { 5567ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InPtr = Builder.CreateLoad(AIn, "InPtr"); 5577ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines In = Builder.CreateLoad(InPtr, "In"); 5587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(In); 5597ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5607ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 561e4a73f68e1b338881adf682c458e0b4b92ecd91eTobias Grosser llvm::Value *X = IV; 5627ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasX(Signature)) { 5637ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(X); 5647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (Y) { 5677ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(Y); 5687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5697ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *RetVal = Builder.CreateCall(F, RootArgs); 5717ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 57274a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (AOut && !PassOutByReference) { 5737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines OutPtr = Builder.CreateLoad(AOut, "OutPtr"); 5747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(RetVal, OutPtr); 5757ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (InPtr) { 5787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // InPtr += instep 5797ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *NewIn = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 5807ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(InPtr, Int32Ty), InStep), InTy); 5817ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(NewIn, AIn); 5827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5847ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (OutPtr) { 5857ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // OutPtr += outstep 5867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *NewOut = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 5877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(OutPtr, Int32Ty), OutStep), OutTy); 588db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(NewOut, AOut); 589db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 590db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 591db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return true; 592db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 593db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 594db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual bool runOnModule(llvm::Module &M) { 595cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines bool Changed = false; 596db169187dea4602e4ad32058762d23d474753fd0Stephen Hines this->M = &M; 597db169187dea4602e4ad32058762d23d474753fd0Stephen Hines C = &M.getContext(); 598db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 5997a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao for (RSInfo::ExportForeachFuncListTy::const_iterator 6007a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao func_iter = mFuncs.begin(), func_end = mFuncs.end(); 6017a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao func_iter != func_end; func_iter++) { 6027a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao const char *name = func_iter->first; 6037a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao uint32_t signature = func_iter->second; 6047a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao llvm::Function *kernel = M.getFunction(name); 6057ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (kernel && isKernel(signature)) { 6067ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Changed |= ExpandKernel(kernel, signature); 6077ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 6087ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines else if (kernel && kernel->getReturnType()->isVoidTy()) { 6097a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao Changed |= ExpandFunction(kernel, signature); 610cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines } 611db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 612db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 613cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines return Changed; 614db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 615db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 616db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual const char *getPassName() const { 617db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return "ForEach-able Function Expansion"; 618db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 619db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}; // end RSForEachExpandPass 621db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6227a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end anonymous namespace 6237a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 6247a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaochar RSForEachExpandPass::ID = 0; 625db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 626db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace bcc { 627db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6287a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaollvm::ModulePass * 6292b04086acbef6520ae2c54a868b1271abf053122Stephen HinescreateRSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs, 6302b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool pEnableStepOpt){ 6312b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return new RSForEachExpandPass(pForeachFuncs, pEnableStepOpt); 6327a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} 633db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6347a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end namespace bcc 635