1db169187dea4602e4ad32058762d23d474753fd0Stephen Hines/* 2db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Copyright 2012, The Android Open Source Project 3db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 4db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Licensed under the Apache License, Version 2.0 (the "License"); 5db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * you may not use this file except in compliance with the License. 6db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * You may obtain a copy of the License at 7db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 8db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * http://www.apache.org/licenses/LICENSE-2.0 9db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 10db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Unless required by applicable law or agreed to in writing, software 11db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * distributed under the License is distributed on an "AS IS" BASIS, 12db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * See the License for the specific language governing permissions and 14db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * limitations under the License. 15db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 16db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 176e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines#include "bcc/Assert.h" 18e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSTransforms.h" 197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao#include <cstdlib> 217a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 22c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/DerivedTypes.h> 23c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Function.h> 24c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Instructions.h> 258b5be8604307d0dd342235803dd45775830b84c6Shih-wei Liao#include <llvm/IRBuilder.h> 26c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Module.h> 27c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Pass.h> 287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines#include <llvm/Support/raw_ostream.h> 292b04086acbef6520ae2c54a868b1271abf053122Stephen Hines#include <llvm/Target/TargetData.h> 30c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include <llvm/Type.h> 31c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang 32c72c4ddfcd79c74f70713da91a69569451b5c19eZonr Chang#include "bcc/Config/Config.h" 33e198abec6c5e3eab380ccf6897b0a0b9c2dd92ddStephen Hines#include "bcc/Renderscript/RSInfo.h" 34ef73a242762bcd8113b9b65ceccbe7d909b5acbcZonr Chang#include "bcc/Support/Log.h" 35db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 367a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaousing namespace bcc; 377a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 38db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace { 397a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 407a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao/* RSForEachExpandPass - This pass operates on functions that are able to be 417a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the 427a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * ForEach-able function to be invoked over the appropriate data cells of the 437a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * input/output allocations (adjusting other relevant parameters as we go). We 447a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * support doing this for any ForEach-able compute kernels. The new function 457a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * name is the original function name followed by ".expand". Note that we 467a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao * still generate code for the original function. 477a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao */ 487a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoclass RSForEachExpandPass : public llvm::ModulePass { 497a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaoprivate: 50db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static char ID; 51db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 52db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Module *M; 53db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::LLVMContext *C; 54db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 557a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao const RSInfo::ExportForeachFuncListTy &mFuncs; 56cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines 572b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Turns on optimization of allocation stride values. 582b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool mEnableStepOpt; 592b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 60cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines uint32_t getRootSignature(llvm::Function *F) { 61db169187dea4602e4ad32058762d23d474753fd0Stephen Hines const llvm::NamedMDNode *ExportForEachMetadata = 62db169187dea4602e4ad32058762d23d474753fd0Stephen Hines M->getNamedMetadata("#rs_export_foreach"); 63db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 64db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (!ExportForEachMetadata) { 65db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 8> RootArgTys; 66db169187dea4602e4ad32058762d23d474753fd0Stephen Hines for (llvm::Function::arg_iterator B = F->arg_begin(), 67db169187dea4602e4ad32058762d23d474753fd0Stephen Hines E = F->arg_end(); 68db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 69db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 70db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgTys.push_back(B->getType()); 71db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 72db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 73db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // For pre-ICS bitcode, we may not have signature information. In that 74db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // case, we use the size of the RootArgTys to select the number of 75db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // arguments. 76db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return (1 << RootArgTys.size()) - 1; 77db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 78db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 797ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ExportForEachMetadata->getNumOperands() == 0) { 807ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return 0; 817ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 836e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(ExportForEachMetadata->getNumOperands() > 0); 84db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 85cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // We only handle the case for legacy root() functions here, so this is 86cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // hard-coded to look at only the first such function. 87db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0); 88db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigNode != NULL && SigNode->getNumOperands() == 1) { 89db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *SigVal = SigNode->getOperand(0); 90db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigVal->getValueID() == llvm::Value::MDStringVal) { 91db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::StringRef SigString = 92db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static_cast<llvm::MDString*>(SigVal)->getString(); 93db169187dea4602e4ad32058762d23d474753fd0Stephen Hines uint32_t Signature = 0; 94db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (SigString.getAsInteger(10, Signature)) { 95db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ALOGE("Non-integer signature value '%s'", SigString.str().c_str()); 96db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 97db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 98db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return Signature; 99db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 100db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 101db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 102db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return 0; 103db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 104db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1052b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // Get the actual value we should use to step through an allocation. 1062b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // TD - Target Data size/layout information. 1072b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // T - Type of allocation (should be a pointer). 1082b04086acbef6520ae2c54a868b1271abf053122Stephen Hines // OrigStep - Original step increment (root.expand() input from driver). 1092b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *getStepValue(llvm::TargetData *TD, llvm::Type *T, 1102b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OrigStep) { 1112b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(TD); 1122b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(T); 1132b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bccAssert(OrigStep); 1142b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(T); 1152b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C); 1162b04086acbef6520ae2c54a868b1271abf053122Stephen Hines if (mEnableStepOpt && T != VoidPtrTy && PT) { 1172b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *ET = PT->getElementType(); 118ceb4c0f5fc5f63c6e494fc2fda25e4461d27abb5Stephen Hines uint64_t ETSize = TD->getTypeAllocSize(ET); 1192b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 1202b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return llvm::ConstantInt::get(Int32Ty, ETSize); 1212b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } else { 1222b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return OrigStep; 1232b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1242b04086acbef6520ae2c54a868b1271abf053122Stephen Hines } 1252b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 126db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasIn(uint32_t Signature) { 1277ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x01; 128db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 129db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 130db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasOut(uint32_t Signature) { 1317ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x02; 132db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 133db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 134db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasUsrData(uint32_t Signature) { 1357ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x04; 136db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 137db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 138db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasX(uint32_t Signature) { 1397ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x08; 140db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 141db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 142db169187dea4602e4ad32058762d23d474753fd0Stephen Hines static bool hasY(uint32_t Signature) { 1437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x10; 1447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 1457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 1467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines static bool isKernel(uint32_t Signature) { 1477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return Signature & 0x20; 148db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 149db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1507ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 1517a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaopublic: 1522b04086acbef6520ae2c54a868b1271abf053122Stephen Hines RSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs, 1532b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool pEnableStepOpt) 1542b04086acbef6520ae2c54a868b1271abf053122Stephen Hines : ModulePass(ID), M(NULL), C(NULL), mFuncs(pForeachFuncs), 1552b04086acbef6520ae2c54a868b1271abf053122Stephen Hines mEnableStepOpt(pEnableStepOpt) { 156db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 157db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 158db169187dea4602e4ad32058762d23d474753fd0Stephen Hines /* Performs the actual optimization on a selected function. On success, the 159db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * Module will contain a new function of the name "<NAME>.expand" that 160db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * invokes <NAME>() in a loop with the appropriate parameters. 161db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 162cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines bool ExpandFunction(llvm::Function *F, uint32_t Signature) { 163cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines ALOGV("Expanding ForEach-able Function %s", F->getName().str().c_str()); 164db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 165db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (!Signature) { 166cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines Signature = getRootSignature(F); 167cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines if (!Signature) { 168cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // We couldn't determine how to expand this function based on its 169cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // function signature. 170cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines return false; 171cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines } 172db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 173db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 1742b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::TargetData TD(M); 1752b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 176db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C); 177db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 178db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *SizeTy = Int32Ty; 179db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 180db169187dea4602e4ad32058762d23d474753fd0Stephen Hines /* Defined in frameworks/base/libs/rs/rs_hal.h: 181db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * 182db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * struct RsForEachStubParamStruct { 183db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *in; 184db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * void *out; 185db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * const void *usr; 186db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * size_t usr_len; 187db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t x; 188db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t y; 189db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t z; 190db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t lod; 191db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * enum RsAllocationCubemapFace face; 192db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t ar[16]; 193db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * }; 194db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 195db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 9> StructTys; 196db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // const void *in 197db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // void *out 198db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(VoidPtrTy); // const void *usr 199db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(SizeTy); // size_t usr_len 200db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t x 201db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t y 202db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t z 203db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // uint32_t lod 204db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(Int32Ty); // enum RsAllocationCubemapFace 205db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys.push_back(llvm::ArrayType::get(Int32Ty, 16)); // uint32_t ar[16] 206db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 207db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *ForEachStubPtrTy = llvm::StructType::create( 208db169187dea4602e4ad32058762d23d474753fd0Stephen Hines StructTys, "RsForEachStubParamStruct")->getPointerTo(); 209db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 210db169187dea4602e4ad32058762d23d474753fd0Stephen Hines /* Create the function signature for our expanded function. 211db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 212db169187dea4602e4ad32058762d23d474753fd0Stephen Hines * uint32_t instep, uint32_t outstep) 213db169187dea4602e4ad32058762d23d474753fd0Stephen Hines */ 214db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Type*, 8> ParamTys; 215db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ParamTys.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p 216db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t x1 217db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t x2 218db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t instep 219db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t outstep 220db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 221db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::FunctionType *FT = 222db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::FunctionType::get(llvm::Type::getVoidTy(*C), ParamTys, false); 223db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Function *ExpandedFunc = 224db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Function::Create(FT, 225db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::GlobalValue::ExternalLinkage, 226db169187dea4602e4ad32058762d23d474753fd0Stephen Hines F->getName() + ".expand", M); 227db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 228db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Create and name the actual arguments to this expanded function. 229db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Argument*, 8> ArgVec; 230db169187dea4602e4ad32058762d23d474753fd0Stephen Hines for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(), 231db169187dea4602e4ad32058762d23d474753fd0Stephen Hines E = ExpandedFunc->arg_end(); 232db169187dea4602e4ad32058762d23d474753fd0Stephen Hines B != E; 233db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ++B) { 234db169187dea4602e4ad32058762d23d474753fd0Stephen Hines ArgVec.push_back(B); 235db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 236db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 237db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (ArgVec.size() != 5) { 23889e8490c80505468f2b816ca9d12fefa53f05959Shih-wei Liao ALOGE("Incorrect number of arguments to function: %zu", 23989e8490c80505468f2b816ca9d12fefa53f05959Shih-wei Liao ArgVec.size()); 240db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return false; 241db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 242db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_p = ArgVec[0]; 243db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_x1 = ArgVec[1]; 244db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_x2 = ArgVec[2]; 245db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_instep = ArgVec[3]; 246db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Arg_outstep = ArgVec[4]; 247db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 248db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Arg_p->setName("p"); 249db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Arg_x1->setName("x1"); 250db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Arg_x2->setName("x2"); 2512b04086acbef6520ae2c54a868b1271abf053122Stephen Hines Arg_instep->setName("arg_instep"); 2522b04086acbef6520ae2c54a868b1271abf053122Stephen Hines Arg_outstep->setName("arg_outstep"); 2532b04086acbef6520ae2c54a868b1271abf053122Stephen Hines 2542b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *InStep = NULL; 2552b04086acbef6520ae2c54a868b1271abf053122Stephen Hines llvm::Value *OutStep = NULL; 256db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 257db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Construct the actual function body. 258db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::BasicBlock *Begin = 259db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::BasicBlock::Create(*C, "Begin", ExpandedFunc); 260db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::IRBuilder<> Builder(Begin); 261db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 262db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // uint32_t X = x1; 263db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::AllocaInst *AX = Builder.CreateAlloca(Int32Ty, 0, "AX"); 264db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(Arg_x1, AX); 265db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 266cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Collect and construct the arguments for the kernel(). 267db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 268db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Function::arg_iterator Args = F->arg_begin(); 269db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 270db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *InTy = NULL; 271db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::AllocaInst *AIn = NULL; 272db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasIn(Signature)) { 273db169187dea4602e4ad32058762d23d474753fd0Stephen Hines InTy = Args->getType(); 274db169187dea4602e4ad32058762d23d474753fd0Stephen Hines AIn = Builder.CreateAlloca(InTy, 0, "AIn"); 2752b04086acbef6520ae2c54a868b1271abf053122Stephen Hines InStep = getStepValue(&TD, InTy, Arg_instep); 2762b04086acbef6520ae2c54a868b1271abf053122Stephen Hines InStep->setName("instep"); 277db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 278db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 0)), InTy), AIn); 279db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 280db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 281db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 282db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *OutTy = NULL; 283db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::AllocaInst *AOut = NULL; 284db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasOut(Signature)) { 285db169187dea4602e4ad32058762d23d474753fd0Stephen Hines OutTy = Args->getType(); 286db169187dea4602e4ad32058762d23d474753fd0Stephen Hines AOut = Builder.CreateAlloca(OutTy, 0, "AOut"); 2872b04086acbef6520ae2c54a868b1271abf053122Stephen Hines OutStep = getStepValue(&TD, OutTy, Arg_outstep); 2882b04086acbef6520ae2c54a868b1271abf053122Stephen Hines OutStep->setName("outstep"); 289db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 290db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 1)), OutTy), AOut); 291db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 292db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 293db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 294db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *UsrData = NULL; 295db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasUsrData(Signature)) { 296db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Type *UsrDataTy = Args->getType(); 297db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData = Builder.CreatePointerCast(Builder.CreateLoad( 298db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStructGEP(Arg_p, 2)), UsrDataTy); 299db169187dea4602e4ad32058762d23d474753fd0Stephen Hines UsrData->setName("UsrData"); 300db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 301db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 302db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 303db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasX(Signature)) { 304db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 305db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 306db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 307db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Y = NULL; 308db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasY(Signature)) { 309db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 310db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Args++; 311db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 312db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3136e9e89d1ed049ec931bb1000948c12698e6c5484Stephen Hines bccAssert(Args == F->arg_end()); 314db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 315db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::BasicBlock *Loop = llvm::BasicBlock::Create(*C, "Loop", ExpandedFunc); 316db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::BasicBlock *Exit = llvm::BasicBlock::Create(*C, "Exit", ExpandedFunc); 317db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 318db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // if (x1 < x2) goto Loop; else goto Exit; 319db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *Cond = Builder.CreateICmpSLT(Arg_x1, Arg_x2); 320db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateCondBr(Cond, Loop, Exit); 321db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 322db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Loop: 323db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.SetInsertPoint(Loop); 324db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 325cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines // Populate the actual call to kernel(). 326db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 327db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InPtr = NULL; 3297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutPtr = NULL; 330db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 331db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (AIn) { 3327ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InPtr = Builder.CreateLoad(AIn, "InPtr"); 3337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(InPtr); 334db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 335db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 336db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (AOut) { 3377ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines OutPtr = Builder.CreateLoad(AOut, "OutPtr"); 3387ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(OutPtr); 339db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 340db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 341db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (UsrData) { 342db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(UsrData); 343db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 344db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 345db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // We always have to load X, since it is used to iterate through the loop. 346db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *X = Builder.CreateLoad(AX, "X"); 347db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (hasX(Signature)) { 348db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(X); 349db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 350db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 351db169187dea4602e4ad32058762d23d474753fd0Stephen Hines if (Y) { 352db169187dea4602e4ad32058762d23d474753fd0Stephen Hines RootArgs.push_back(Y); 353db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 354db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 355db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateCall(F, RootArgs); 356db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3577ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (InPtr) { 3587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // InPtr += instep 359db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *NewIn = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 3607ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(InPtr, Int32Ty), InStep), InTy); 361db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(NewIn, AIn); 362db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 363db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 3647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (OutPtr) { 3657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // OutPtr += outstep 366db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *NewOut = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 3677ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(OutPtr, Int32Ty), OutStep), OutTy); 3687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(NewOut, AOut); 3697ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 3707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 3717ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // X++; 3727ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *XPlusOne = 3737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateNUWAdd(X, llvm::ConstantInt::get(Int32Ty, 1)); 3747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(XPlusOne, AX); 3757ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 3767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // If (X < x2) goto Loop; else goto Exit; 3777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Cond = Builder.CreateICmpSLT(XPlusOne, Arg_x2); 3787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateCondBr(Cond, Loop, Exit); 3797ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 3807ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Exit: 3817ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.SetInsertPoint(Exit); 3827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateRetVoid(); 3837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 3847ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return true; 3857ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 3867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 3877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Expand a pass-by-value kernel. 3887ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 3897ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bool ExpandKernel(llvm::Function *F, uint32_t Signature) { 3907ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bccAssert(isKernel(Signature)); 3917ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ALOGV("Expanding kernel Function %s", F->getName().str().c_str()); 3927ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 3937ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // TODO: Refactor this to share functionality with ExpandFunction. 3947ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::TargetData TD(M); 3957ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 3967ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*C); 3977ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*C); 3987ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *SizeTy = Int32Ty; 3997ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4007ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Defined in frameworks/base/libs/rs/rs_hal.h: 4017ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * 4027ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * struct RsForEachStubParamStruct { 4037ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * const void *in; 4047ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * void *out; 4057ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * const void *usr; 4067ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * size_t usr_len; 4077ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * uint32_t x; 4087ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * uint32_t y; 4097ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * uint32_t z; 4107ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * uint32_t lod; 4117ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * enum RsAllocationCubemapFace face; 4127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * uint32_t ar[16]; 4137ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * }; 4147ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 4157ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Type*, 9> StructTys; 4167ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(VoidPtrTy); // const void *in 4177ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(VoidPtrTy); // void *out 4187ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(VoidPtrTy); // const void *usr 4197ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(SizeTy); // size_t usr_len 4207ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(Int32Ty); // uint32_t x 4217ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(Int32Ty); // uint32_t y 4227ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(Int32Ty); // uint32_t z 4237ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(Int32Ty); // uint32_t lod 4247ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(Int32Ty); // enum RsAllocationCubemapFace 4257ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys.push_back(llvm::ArrayType::get(Int32Ty, 16)); // uint32_t ar[16] 4267ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4277ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *ForEachStubPtrTy = llvm::StructType::create( 4287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines StructTys, "RsForEachStubParamStruct")->getPointerTo(); 4297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4307ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines /* Create the function signature for our expanded function. 4317ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 4327ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines * uint32_t instep, uint32_t outstep) 4337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines */ 4347ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Type*, 8> ParamTys; 4357ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ParamTys.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p 4367ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t x1 4377ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t x2 4387ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t instep 4397ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ParamTys.push_back(Int32Ty); // uint32_t outstep 4407ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4417ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::FunctionType *FT = 4427ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::FunctionType::get(llvm::Type::getVoidTy(*C), ParamTys, false); 4437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Function *ExpandedFunc = 4447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Function::Create(FT, 4457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::GlobalValue::ExternalLinkage, 4467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines F->getName() + ".expand", M); 4477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Create and name the actual arguments to this expanded function. 4497ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Argument*, 8> ArgVec; 4507ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines for (llvm::Function::arg_iterator B = ExpandedFunc->arg_begin(), 4517ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines E = ExpandedFunc->arg_end(); 4527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines B != E; 4537ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ++B) { 4547ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ArgVec.push_back(B); 4557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4567ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4577ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (ArgVec.size() != 5) { 4587ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ALOGE("Incorrect number of arguments to function: %zu", 4597ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines ArgVec.size()); 4607ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines return false; 4617ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 4627ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_p = ArgVec[0]; 4637ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_x1 = ArgVec[1]; 4647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_x2 = ArgVec[2]; 4657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_instep = ArgVec[3]; 4667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Arg_outstep = ArgVec[4]; 4677ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Arg_p->setName("p"); 4697ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Arg_x1->setName("x1"); 4707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Arg_x2->setName("x2"); 4717ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Arg_instep->setName("arg_instep"); 4727ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Arg_outstep->setName("arg_outstep"); 4737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InStep = NULL; 4757ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutStep = NULL; 4767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Construct the actual function body. 4787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::BasicBlock *Begin = 4797ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::BasicBlock::Create(*C, "Begin", ExpandedFunc); 4807ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::IRBuilder<> Builder(Begin); 4817ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // uint32_t X = x1; 4837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::AllocaInst *AX = Builder.CreateAlloca(Int32Ty, 0, "AX"); 4847ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(Arg_x1, AX); 4857ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 4867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Collect and construct the arguments for the kernel(). 4877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Note that we load any loop-invariant arguments before entering the Loop. 4887ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Function::arg_iterator Args = F->arg_begin(); 4897ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 49074a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::Type *OutTy = NULL; 49174a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::AllocaInst *AOut = NULL; 49274a4b08235990916911b8fe758d656c1171faf26Stephen Hines bool PassOutByReference = false; 49374a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (hasOut(Signature)) { 49474a4b08235990916911b8fe758d656c1171faf26Stephen Hines llvm::Type *OutBaseTy = F->getReturnType(); 49574a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (OutBaseTy->isVoidTy()) { 49674a4b08235990916911b8fe758d656c1171faf26Stephen Hines PassOutByReference = true; 49774a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutTy = Args->getType(); 49874a4b08235990916911b8fe758d656c1171faf26Stephen Hines Args++; 49974a4b08235990916911b8fe758d656c1171faf26Stephen Hines } else { 50074a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutTy = OutBaseTy->getPointerTo(); 50174a4b08235990916911b8fe758d656c1171faf26Stephen Hines // We don't increment Args, since we are using the actual return type. 50274a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 50374a4b08235990916911b8fe758d656c1171faf26Stephen Hines AOut = Builder.CreateAlloca(OutTy, 0, "AOut"); 50474a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutStep = getStepValue(&TD, OutTy, Arg_outstep); 50574a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutStep->setName("outstep"); 50674a4b08235990916911b8fe758d656c1171faf26Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 50774a4b08235990916911b8fe758d656c1171faf26Stephen Hines Builder.CreateStructGEP(Arg_p, 1)), OutTy), AOut); 50874a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 50974a4b08235990916911b8fe758d656c1171faf26Stephen Hines 5107ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *InBaseTy = NULL; 5117ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Type *InTy = NULL; 5127ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::AllocaInst *AIn = NULL; 5137ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasIn(Signature)) { 5147ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InBaseTy = Args->getType(); 5157ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InTy =InBaseTy->getPointerTo(); 5167ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines AIn = Builder.CreateAlloca(InTy, 0, "AIn"); 5177ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InStep = getStepValue(&TD, InTy, Arg_instep); 5187ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InStep->setName("instep"); 5197ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(Builder.CreatePointerCast(Builder.CreateLoad( 5207ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStructGEP(Arg_p, 0)), InTy), AIn); 5217ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5227ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5237ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5247ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // No usrData parameter on kernels. 5257ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bccAssert(!hasUsrData(Signature)); 5267ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5277ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasX(Signature)) { 5287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5297ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5307ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5317ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Y = NULL; 5327ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasY(Signature)) { 5337ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 5347ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Args++; 5357ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5367ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5377ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines bccAssert(Args == F->arg_end()); 5387ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5397ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::BasicBlock *Loop = llvm::BasicBlock::Create(*C, "Loop", ExpandedFunc); 5407ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::BasicBlock *Exit = llvm::BasicBlock::Create(*C, "Exit", ExpandedFunc); 5417ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5427ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // if (x1 < x2) goto Loop; else goto Exit; 5437ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *Cond = Builder.CreateICmpSLT(Arg_x1, Arg_x2); 5447ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateCondBr(Cond, Loop, Exit); 5457ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5467ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Loop: 5477ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.SetInsertPoint(Loop); 5487ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5497ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // Populate the actual call to kernel(). 5507ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::SmallVector<llvm::Value*, 8> RootArgs; 5517ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5527ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *InPtr = NULL; 5537ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *In = NULL; 5547ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *OutPtr = NULL; 5557ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 55674a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (PassOutByReference) { 55774a4b08235990916911b8fe758d656c1171faf26Stephen Hines OutPtr = Builder.CreateLoad(AOut, "OutPtr"); 55874a4b08235990916911b8fe758d656c1171faf26Stephen Hines RootArgs.push_back(OutPtr); 55974a4b08235990916911b8fe758d656c1171faf26Stephen Hines } 56074a4b08235990916911b8fe758d656c1171faf26Stephen Hines 5617ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (AIn) { 5627ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines InPtr = Builder.CreateLoad(AIn, "InPtr"); 5637ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines In = Builder.CreateLoad(InPtr, "In"); 5647ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(In); 5657ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5667ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5677ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // We always have to load X, since it is used to iterate through the loop. 5687ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *X = Builder.CreateLoad(AX, "X"); 5697ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (hasX(Signature)) { 5707ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(X); 5717ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5727ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5737ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (Y) { 5747ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines RootArgs.push_back(Y); 5757ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5767ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5777ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *RetVal = Builder.CreateCall(F, RootArgs); 5787ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 57974a4b08235990916911b8fe758d656c1171faf26Stephen Hines if (AOut && !PassOutByReference) { 5807ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines OutPtr = Builder.CreateLoad(AOut, "OutPtr"); 5817ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(RetVal, OutPtr); 5827ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5837ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5847ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (InPtr) { 5857ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // InPtr += instep 5867ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *NewIn = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 5877ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(InPtr, Int32Ty), InStep), InTy); 5887ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreateStore(NewIn, AIn); 5897ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 5907ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines 5917ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (OutPtr) { 5927ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines // OutPtr += outstep 5937ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines llvm::Value *NewOut = Builder.CreateIntToPtr(Builder.CreateNUWAdd( 5947ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Builder.CreatePtrToInt(OutPtr, Int32Ty), OutStep), OutTy); 595db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(NewOut, AOut); 596db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 597db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 598db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // X++; 599db169187dea4602e4ad32058762d23d474753fd0Stephen Hines llvm::Value *XPlusOne = 600db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateNUWAdd(X, llvm::ConstantInt::get(Int32Ty, 1)); 601db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateStore(XPlusOne, AX); 602db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 603db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // If (X < x2) goto Loop; else goto Exit; 604db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Cond = Builder.CreateICmpSLT(XPlusOne, Arg_x2); 605db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateCondBr(Cond, Loop, Exit); 606db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 607db169187dea4602e4ad32058762d23d474753fd0Stephen Hines // Exit: 608db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.SetInsertPoint(Exit); 609db169187dea4602e4ad32058762d23d474753fd0Stephen Hines Builder.CreateRetVoid(); 610db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 611db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return true; 612db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 613db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 614db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual bool runOnModule(llvm::Module &M) { 615cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines bool Changed = false; 616db169187dea4602e4ad32058762d23d474753fd0Stephen Hines this->M = &M; 617db169187dea4602e4ad32058762d23d474753fd0Stephen Hines C = &M.getContext(); 618db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6197a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao for (RSInfo::ExportForeachFuncListTy::const_iterator 6207a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao func_iter = mFuncs.begin(), func_end = mFuncs.end(); 6217a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao func_iter != func_end; func_iter++) { 6227a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao const char *name = func_iter->first; 6237a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao uint32_t signature = func_iter->second; 6247a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao llvm::Function *kernel = M.getFunction(name); 6257ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines if (kernel && isKernel(signature)) { 6267ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines Changed |= ExpandKernel(kernel, signature); 6277ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines } 6287ae3a828fade4e30f7940415949f33ac1f15dc77Stephen Hines else if (kernel && kernel->getReturnType()->isVoidTy()) { 6297a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao Changed |= ExpandFunction(kernel, signature); 630cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines } 631db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 632db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 633cc366e573e31f43a6101fd6e04b90c6afdc3b7a7Stephen Hines return Changed; 634db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 635db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 636db169187dea4602e4ad32058762d23d474753fd0Stephen Hines virtual const char *getPassName() const { 637db169187dea4602e4ad32058762d23d474753fd0Stephen Hines return "ForEach-able Function Expansion"; 638db169187dea4602e4ad32058762d23d474753fd0Stephen Hines } 639db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6407a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao}; // end RSForEachExpandPass 641db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6427a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end anonymous namespace 6437a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao 6447a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaochar RSForEachExpandPass::ID = 0; 645db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 646db169187dea4602e4ad32058762d23d474753fd0Stephen Hinesnamespace bcc { 647db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6487a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liaollvm::ModulePass * 6492b04086acbef6520ae2c54a868b1271abf053122Stephen HinescreateRSForEachExpandPass(const RSInfo::ExportForeachFuncListTy &pForeachFuncs, 6502b04086acbef6520ae2c54a868b1271abf053122Stephen Hines bool pEnableStepOpt){ 6512b04086acbef6520ae2c54a868b1271abf053122Stephen Hines return new RSForEachExpandPass(pForeachFuncs, pEnableStepOpt); 6527a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} 653db169187dea4602e4ad32058762d23d474753fd0Stephen Hines 6547a66e6cbb1ae32cd56b19822c4e66560deb857dbShih-wei Liao} // end namespace bcc 655