RSForEachExpand.cpp revision bdbff6e600b0d834e4770f65c7d2df93d7ef305c
1/* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "bcc/Assert.h" 18#include "bcc/Renderscript/RSTransforms.h" 19 20#include <cstdlib> 21 22#include <llvm/IR/DerivedTypes.h> 23#include <llvm/IR/Function.h> 24#include <llvm/IR/Instructions.h> 25#include <llvm/IR/IRBuilder.h> 26#include <llvm/IR/MDBuilder.h> 27#include <llvm/IR/Module.h> 28#include <llvm/Pass.h> 29#include <llvm/Support/raw_ostream.h> 30#include <llvm/IR/DataLayout.h> 31#include <llvm/IR/Function.h> 32#include <llvm/IR/Type.h> 33#include <llvm/Transforms/Utils/BasicBlockUtils.h> 34 35#include "bcc/Config/Config.h" 36#include "bcc/Support/Log.h" 37 38#include "bcinfo/MetadataExtractor.h" 39 40#define NUM_EXPANDED_FUNCTION_PARAMS 5 41 42using namespace bcc; 43 44namespace { 45 46static const bool gEnableRsTbaa = true; 47 48/* RSForEachExpandPass - This pass operates on functions that are able to be 49 * called via rsForEach() or "foreach_<NAME>". We create an inner loop for the 50 * ForEach-able function to be invoked over the appropriate data cells of the 51 * input/output allocations (adjusting other relevant parameters as we go). We 52 * support doing this for any ForEach-able compute kernels. The new function 53 * name is the original function name followed by ".expand". Note that we 54 * still generate code for the original function. 55 */ 56class RSForEachExpandPass : public llvm::ModulePass { 57private: 58 static char ID; 59 60 llvm::Module *Module; 61 llvm::LLVMContext *Context; 62 63 /* 64 * Pointer to LLVM type information for the ForEachStubType and the function 65 * signature for expanded kernels. These must be re-calculated for each 66 * module the pass is run on. 67 */ 68 llvm::StructType *ForEachStubType; 69 llvm::FunctionType *ExpandedFunctionType; 70 71 uint32_t mExportForEachCount; 72 const char **mExportForEachNameList; 73 const uint32_t *mExportForEachSignatureList; 74 75 // Turns on optimization of allocation stride values. 76 bool mEnableStepOpt; 77 78 uint32_t getRootSignature(llvm::Function *Function) { 79 const llvm::NamedMDNode *ExportForEachMetadata = 80 Module->getNamedMetadata("#rs_export_foreach"); 81 82 if (!ExportForEachMetadata) { 83 llvm::SmallVector<llvm::Type*, 8> RootArgTys; 84 for (llvm::Function::arg_iterator B = Function->arg_begin(), 85 E = Function->arg_end(); 86 B != E; 87 ++B) { 88 RootArgTys.push_back(B->getType()); 89 } 90 91 // For pre-ICS bitcode, we may not have signature information. In that 92 // case, we use the size of the RootArgTys to select the number of 93 // arguments. 94 return (1 << RootArgTys.size()) - 1; 95 } 96 97 if (ExportForEachMetadata->getNumOperands() == 0) { 98 return 0; 99 } 100 101 bccAssert(ExportForEachMetadata->getNumOperands() > 0); 102 103 // We only handle the case for legacy root() functions here, so this is 104 // hard-coded to look at only the first such function. 105 llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0); 106 if (SigNode != NULL && SigNode->getNumOperands() == 1) { 107 llvm::Value *SigVal = SigNode->getOperand(0); 108 if (SigVal->getValueID() == llvm::Value::MDStringVal) { 109 llvm::StringRef SigString = 110 static_cast<llvm::MDString*>(SigVal)->getString(); 111 uint32_t Signature = 0; 112 if (SigString.getAsInteger(10, Signature)) { 113 ALOGE("Non-integer signature value '%s'", SigString.str().c_str()); 114 return 0; 115 } 116 return Signature; 117 } 118 } 119 120 return 0; 121 } 122 123 // Get the actual value we should use to step through an allocation. 124 // 125 // Normally the value we use to step through an allocation is given to us by 126 // the driver. However, for certain primitive data types, we can derive an 127 // integer constant for the step value. We use this integer constant whenever 128 // possible to allow further compiler optimizations to take place. 129 // 130 // DL - Target Data size/layout information. 131 // T - Type of allocation (should be a pointer). 132 // OrigStep - Original step increment (root.expand() input from driver). 133 llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *AllocType, 134 llvm::Value *OrigStep) { 135 bccAssert(DL); 136 bccAssert(AllocType); 137 bccAssert(OrigStep); 138 llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType); 139 llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 140 if (mEnableStepOpt && AllocType != VoidPtrTy && PT) { 141 llvm::Type *ET = PT->getElementType(); 142 uint64_t ETSize = DL->getTypeAllocSize(ET); 143 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 144 return llvm::ConstantInt::get(Int32Ty, ETSize); 145 } else { 146 return OrigStep; 147 } 148 } 149 150 /// @brief Builds the types required by the pass for the given context. 151 void buildTypes(void) { 152 // Create the RsForEachStubParam struct. 153 154 llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context); 155 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context); 156 /* Defined in frameworks/base/libs/rs/rs_hal.h: 157 * 158 * struct RsForEachStubParamStruct { 159 * const void *in; 160 * void *out; 161 * const void *usr; 162 * uint32_t usr_len; 163 * uint32_t x; 164 * uint32_t y; 165 * uint32_t z; 166 * uint32_t lod; 167 * enum RsAllocationCubemapFace face; 168 * uint32_t ar[16]; 169 * }; 170 */ 171 llvm::SmallVector<llvm::Type*, 16> StructTypes; 172 StructTypes.push_back(VoidPtrTy); // const void *in 173 StructTypes.push_back(VoidPtrTy); // void *out 174 StructTypes.push_back(VoidPtrTy); // const void *usr 175 StructTypes.push_back(Int32Ty); // uint32_t usr_len 176 StructTypes.push_back(Int32Ty); // uint32_t x 177 StructTypes.push_back(Int32Ty); // uint32_t y 178 StructTypes.push_back(Int32Ty); // uint32_t z 179 StructTypes.push_back(Int32Ty); // uint32_t lod 180 StructTypes.push_back(Int32Ty); // enum RsAllocationCubemapFace 181 StructTypes.push_back(llvm::ArrayType::get(Int32Ty, 16)); // uint32_t ar[16] 182 183 ForEachStubType = 184 llvm::StructType::create(StructTypes, "RsForEachStubParamStruct"); 185 186 // Create the function type for expanded kernels. 187 188 llvm::Type *ForEachStubPtrTy = ForEachStubType->getPointerTo(); 189 190 llvm::SmallVector<llvm::Type*, 8> ParamTypes; 191 ParamTypes.push_back(ForEachStubPtrTy); // const RsForEachStubParamStruct *p 192 ParamTypes.push_back(Int32Ty); // uint32_t x1 193 ParamTypes.push_back(Int32Ty); // uint32_t x2 194 ParamTypes.push_back(Int32Ty); // uint32_t instep 195 ParamTypes.push_back(Int32Ty); // uint32_t outstep 196 197 ExpandedFunctionType = llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), 198 ParamTypes, 199 false); 200 } 201 202 /// @brief Create skeleton of the expanded function. 203 /// 204 /// This creates a function with the following signature: 205 /// 206 /// void (const RsForEachStubParamStruct *p, uint32_t x1, uint32_t x2, 207 /// uint32_t instep, uint32_t outstep) 208 /// 209 llvm::Function *createEmptyExpandedFunction(llvm::StringRef OldName) { 210 llvm::Function *ExpandedFunction = 211 llvm::Function::Create(ExpandedFunctionType, 212 llvm::GlobalValue::ExternalLinkage, 213 OldName + ".expand", Module); 214 215 bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 216 217 llvm::Function::arg_iterator AI = ExpandedFunction->arg_begin(); 218 219 (AI++)->setName("p"); 220 (AI++)->setName("x1"); 221 (AI++)->setName("x2"); 222 (AI++)->setName("arg_instep"); 223 (AI++)->setName("arg_outstep"); 224 225 llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin", 226 ExpandedFunction); 227 llvm::IRBuilder<> Builder(Begin); 228 Builder.CreateRetVoid(); 229 230 return ExpandedFunction; 231 } 232 233 /// @brief Create an empty loop 234 /// 235 /// Create a loop of the form: 236 /// 237 /// for (i = LowerBound; i < UpperBound; i++) 238 /// ; 239 /// 240 /// After the loop has been created, the builder is set such that 241 /// instructions can be added to the loop body. 242 /// 243 /// @param Builder The builder to use to build this loop. The current 244 /// position of the builder is the position the loop 245 /// will be inserted. 246 /// @param LowerBound The first value of the loop iterator 247 /// @param UpperBound The maximal value of the loop iterator 248 /// @param LoopIV A reference that will be set to the loop iterator. 249 /// @return The BasicBlock that will be executed after the loop. 250 llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder, 251 llvm::Value *LowerBound, 252 llvm::Value *UpperBound, 253 llvm::PHINode **LoopIV) { 254 assert(LowerBound->getType() == UpperBound->getType()); 255 256 llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB; 257 llvm::Value *Cond, *IVNext; 258 llvm::PHINode *IV; 259 260 CondBB = Builder.GetInsertBlock(); 261 AfterBB = llvm::SplitBlock(CondBB, Builder.GetInsertPoint(), this); 262 HeaderBB = llvm::BasicBlock::Create(*Context, "Loop", CondBB->getParent()); 263 264 // if (LowerBound < Upperbound) 265 // goto LoopHeader 266 // else 267 // goto AfterBB 268 CondBB->getTerminator()->eraseFromParent(); 269 Builder.SetInsertPoint(CondBB); 270 Cond = Builder.CreateICmpULT(LowerBound, UpperBound); 271 Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 272 273 // iv = PHI [CondBB -> LowerBound], [LoopHeader -> NextIV ] 274 // iv.next = iv + 1 275 // if (iv.next < Upperbound) 276 // goto LoopHeader 277 // else 278 // goto AfterBB 279 Builder.SetInsertPoint(HeaderBB); 280 IV = Builder.CreatePHI(LowerBound->getType(), 2, "X"); 281 IV->addIncoming(LowerBound, CondBB); 282 IVNext = Builder.CreateNUWAdd(IV, Builder.getInt32(1)); 283 IV->addIncoming(IVNext, HeaderBB); 284 Cond = Builder.CreateICmpULT(IVNext, UpperBound); 285 Builder.CreateCondBr(Cond, HeaderBB, AfterBB); 286 AfterBB->setName("Exit"); 287 Builder.SetInsertPoint(HeaderBB->getFirstNonPHI()); 288 *LoopIV = IV; 289 return AfterBB; 290 } 291 292public: 293 RSForEachExpandPass(bool pEnableStepOpt) 294 : ModulePass(ID), Module(NULL), Context(NULL), 295 mEnableStepOpt(pEnableStepOpt) { 296 297 } 298 299 /* Performs the actual optimization on a selected function. On success, the 300 * Module will contain a new function of the name "<NAME>.expand" that 301 * invokes <NAME>() in a loop with the appropriate parameters. 302 */ 303 bool ExpandFunction(llvm::Function *Function, uint32_t Signature) { 304 ALOGV("Expanding ForEach-able Function %s", 305 Function->getName().str().c_str()); 306 307 if (!Signature) { 308 Signature = getRootSignature(Function); 309 if (!Signature) { 310 // We couldn't determine how to expand this function based on its 311 // function signature. 312 return false; 313 } 314 } 315 316 llvm::DataLayout DL(Module); 317 318 llvm::Function *ExpandedFunction = 319 createEmptyExpandedFunction(Function->getName()); 320 321 bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 322 323 /* 324 * Extract the expanded function's parameters. It is guaranteed by 325 * createEmptyExpandedFunction that there will be five parameters. 326 */ 327 llvm::Function::arg_iterator ExpandedFunctionArgIter = 328 ExpandedFunction->arg_begin(); 329 330 llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 331 llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 332 llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 333 llvm::Value *Arg_instep = &*(ExpandedFunctionArgIter++); 334 llvm::Value *Arg_outstep = &*ExpandedFunctionArgIter; 335 336 llvm::Value *InStep = NULL; 337 llvm::Value *OutStep = NULL; 338 339 // Construct the actual function body. 340 llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 341 342 // Collect and construct the arguments for the kernel(). 343 // Note that we load any loop-invariant arguments before entering the Loop. 344 llvm::Function::arg_iterator FunctionArgIter = Function->arg_begin(); 345 346 llvm::Type *InTy = NULL; 347 llvm::Value *InBasePtr = NULL; 348 if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) { 349 InTy = (FunctionArgIter++)->getType(); 350 InStep = getStepValue(&DL, InTy, Arg_instep); 351 InStep->setName("instep"); 352 InBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 0)); 353 } 354 355 llvm::Type *OutTy = NULL; 356 llvm::Value *OutBasePtr = NULL; 357 if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 358 OutTy = (FunctionArgIter++)->getType(); 359 OutStep = getStepValue(&DL, OutTy, Arg_outstep); 360 OutStep->setName("outstep"); 361 OutBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 1)); 362 } 363 364 llvm::Value *UsrData = NULL; 365 if (bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)) { 366 llvm::Type *UsrDataTy = (FunctionArgIter++)->getType(); 367 UsrData = Builder.CreatePointerCast(Builder.CreateLoad( 368 Builder.CreateStructGEP(Arg_p, 2)), UsrDataTy); 369 UsrData->setName("UsrData"); 370 } 371 372 if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 373 FunctionArgIter++; 374 } 375 376 llvm::Value *Y = NULL; 377 if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 378 Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 379 FunctionArgIter++; 380 } 381 382 bccAssert(FunctionArgIter == Function->arg_end()); 383 384 llvm::PHINode *IV; 385 createLoop(Builder, Arg_x1, Arg_x2, &IV); 386 387 // Populate the actual call to kernel(). 388 llvm::SmallVector<llvm::Value*, 8> RootArgs; 389 390 llvm::Value *InPtr = NULL; 391 llvm::Value *OutPtr = NULL; 392 393 // Calculate the current input and output pointers 394 // 395 // We always calculate the input/output pointers with a GEP operating on i8 396 // values and only cast at the very end to OutTy. This is because the step 397 // between two values is given in bytes. 398 // 399 // TODO: We could further optimize the output by using a GEP operation of 400 // type 'OutTy' in cases where the element type of the allocation allows. 401 if (OutBasePtr) { 402 llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 403 OutOffset = Builder.CreateMul(OutOffset, OutStep); 404 OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 405 OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 406 } 407 408 if (InBasePtr) { 409 llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1); 410 InOffset = Builder.CreateMul(InOffset, InStep); 411 InPtr = Builder.CreateGEP(InBasePtr, InOffset); 412 InPtr = Builder.CreatePointerCast(InPtr, InTy); 413 } 414 415 if (InPtr) { 416 RootArgs.push_back(InPtr); 417 } 418 419 if (OutPtr) { 420 RootArgs.push_back(OutPtr); 421 } 422 423 if (UsrData) { 424 RootArgs.push_back(UsrData); 425 } 426 427 llvm::Value *X = IV; 428 if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 429 RootArgs.push_back(X); 430 } 431 432 if (Y) { 433 RootArgs.push_back(Y); 434 } 435 436 Builder.CreateCall(Function, RootArgs); 437 438 return true; 439 } 440 441 /* Expand a pass-by-value kernel. 442 */ 443 bool ExpandKernel(llvm::Function *Function, uint32_t Signature) { 444 bccAssert(bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)); 445 ALOGV("Expanding kernel Function %s", Function->getName().str().c_str()); 446 447 // TODO: Refactor this to share functionality with ExpandFunction. 448 llvm::DataLayout DL(Module); 449 450 llvm::Function *ExpandedFunction = 451 createEmptyExpandedFunction(Function->getName()); 452 453 bccAssert(ExpandedFunction->arg_size() == NUM_EXPANDED_FUNCTION_PARAMS); 454 455 /* 456 * Extract the expanded function's parameters. It is guaranteed by 457 * createEmptyExpandedFunction that there will be five parameters. 458 */ 459 llvm::Function::arg_iterator ExpandedFunctionArgIter = 460 ExpandedFunction->arg_begin(); 461 462 llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++); 463 llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++); 464 llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++); 465 llvm::Value *Arg_instep = &*(ExpandedFunctionArgIter++); 466 llvm::Value *Arg_outstep = &*ExpandedFunctionArgIter; 467 468 llvm::Value *InStep = NULL; 469 llvm::Value *OutStep = NULL; 470 471 // Construct the actual function body. 472 llvm::IRBuilder<> Builder(ExpandedFunction->getEntryBlock().begin()); 473 474 // Create TBAA meta-data. 475 llvm::MDNode *TBAARenderScript, *TBAAAllocation, *TBAAPointer; 476 llvm::MDBuilder MDHelper(*Context); 477 478 TBAARenderScript = MDHelper.createTBAARoot("RenderScript TBAA"); 479 TBAAAllocation = MDHelper.createTBAAScalarTypeNode("allocation", TBAARenderScript); 480 TBAAAllocation = MDHelper.createTBAAStructTagNode(TBAAAllocation, TBAAAllocation, 0); 481 TBAAPointer = MDHelper.createTBAAScalarTypeNode("pointer", TBAARenderScript); 482 TBAAPointer = MDHelper.createTBAAStructTagNode(TBAAPointer, TBAAPointer, 0); 483 484 // Collect and construct the arguments for the kernel(). 485 // Note that we load any loop-invariant arguments before entering the Loop. 486 llvm::Function::arg_iterator FunctionArgIterator = Function->arg_begin(); 487 488 llvm::Type *OutTy = NULL; 489 bool PassOutByReference = false; 490 llvm::LoadInst *OutBasePtr = NULL; 491 if (bcinfo::MetadataExtractor::hasForEachSignatureOut(Signature)) { 492 llvm::Type *OutBaseTy = Function->getReturnType(); 493 if (OutBaseTy->isVoidTy()) { 494 PassOutByReference = true; 495 OutTy = (FunctionArgIterator++)->getType(); 496 } else { 497 OutTy = OutBaseTy->getPointerTo(); 498 // We don't increment Args, since we are using the actual return type. 499 } 500 OutStep = getStepValue(&DL, OutTy, Arg_outstep); 501 OutStep->setName("outstep"); 502 OutBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 1)); 503 if (gEnableRsTbaa) { 504 OutBasePtr->setMetadata("tbaa", TBAAPointer); 505 } 506 } 507 508 llvm::Type *InBaseTy = NULL; 509 llvm::Type *InTy = NULL; 510 llvm::LoadInst *InBasePtr = NULL; 511 if (bcinfo::MetadataExtractor::hasForEachSignatureIn(Signature)) { 512 InBaseTy = (FunctionArgIterator++)->getType(); 513 InTy = InBaseTy->getPointerTo(); 514 InStep = getStepValue(&DL, InTy, Arg_instep); 515 InStep->setName("instep"); 516 InBasePtr = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 0)); 517 if (gEnableRsTbaa) { 518 InBasePtr->setMetadata("tbaa", TBAAPointer); 519 } 520 } 521 522 // No usrData parameter on kernels. 523 bccAssert( 524 !bcinfo::MetadataExtractor::hasForEachSignatureUsrData(Signature)); 525 526 if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 527 FunctionArgIterator++; 528 } 529 530 llvm::Value *Y = NULL; 531 if (bcinfo::MetadataExtractor::hasForEachSignatureY(Signature)) { 532 Y = Builder.CreateLoad(Builder.CreateStructGEP(Arg_p, 5), "Y"); 533 FunctionArgIterator++; 534 } 535 536 bccAssert(FunctionArgIterator == Function->arg_end()); 537 538 llvm::PHINode *IV; 539 createLoop(Builder, Arg_x1, Arg_x2, &IV); 540 541 // Populate the actual call to kernel(). 542 llvm::SmallVector<llvm::Value*, 8> RootArgs; 543 544 llvm::Value *InPtr = NULL; 545 llvm::Value *OutPtr = NULL; 546 547 // Calculate the current input and output pointers 548 // 549 // We always calculate the input/output pointers with a GEP operating on i8 550 // values and only cast at the very end to OutTy. This is because the step 551 // between two values is given in bytes. 552 // 553 // TODO: We could further optimize the output by using a GEP operation of 554 // type 'OutTy' in cases where the element type of the allocation allows. 555 if (OutBasePtr) { 556 llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1); 557 OutOffset = Builder.CreateMul(OutOffset, OutStep); 558 OutPtr = Builder.CreateGEP(OutBasePtr, OutOffset); 559 OutPtr = Builder.CreatePointerCast(OutPtr, OutTy); 560 } 561 562 if (InBasePtr) { 563 llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1); 564 InOffset = Builder.CreateMul(InOffset, InStep); 565 InPtr = Builder.CreateGEP(InBasePtr, InOffset); 566 InPtr = Builder.CreatePointerCast(InPtr, InTy); 567 } 568 569 if (PassOutByReference) { 570 RootArgs.push_back(OutPtr); 571 } 572 573 if (InPtr) { 574 llvm::LoadInst *In = Builder.CreateLoad(InPtr, "In"); 575 if (gEnableRsTbaa) { 576 In->setMetadata("tbaa", TBAAAllocation); 577 } 578 RootArgs.push_back(In); 579 } 580 581 llvm::Value *X = IV; 582 if (bcinfo::MetadataExtractor::hasForEachSignatureX(Signature)) { 583 RootArgs.push_back(X); 584 } 585 586 if (Y) { 587 RootArgs.push_back(Y); 588 } 589 590 llvm::Value *RetVal = Builder.CreateCall(Function, RootArgs); 591 592 if (OutPtr && !PassOutByReference) { 593 llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr); 594 if (gEnableRsTbaa) { 595 Store->setMetadata("tbaa", TBAAAllocation); 596 } 597 } 598 599 return true; 600 } 601 602 /// @brief Checks if pointers to allocation internals are exposed 603 /// 604 /// This function verifies if through the parameters passed to the kernel 605 /// or through calls to the runtime library the script gains access to 606 /// pointers pointing to data within a RenderScript Allocation. 607 /// If we know we control all loads from and stores to data within 608 /// RenderScript allocations and if we know the run-time internal accesses 609 /// are all annotated with RenderScript TBAA metadata, only then we 610 /// can safely use TBAA to distinguish between generic and from-allocation 611 /// pointers. 612 bool allocPointersExposed(llvm::Module &Module) { 613 // Old style kernel function can expose pointers to elements within 614 // allocations. 615 // TODO: Extend analysis to allow simple cases of old-style kernels. 616 for (size_t i = 0; i < mExportForEachCount; ++i) { 617 const char *Name = mExportForEachNameList[i]; 618 uint32_t Signature = mExportForEachSignatureList[i]; 619 if (Module.getFunction(Name) && 620 !bcinfo::MetadataExtractor::hasForEachSignatureKernel(Signature)) { 621 return true; 622 } 623 } 624 625 // Check for library functions that expose a pointer to an Allocation or 626 // that are not yet annotated with RenderScript-specific tbaa information. 627 static std::vector<std::string> Funcs; 628 629 // rsGetElementAt(...) 630 Funcs.push_back("_Z14rsGetElementAt13rs_allocationj"); 631 Funcs.push_back("_Z14rsGetElementAt13rs_allocationjj"); 632 Funcs.push_back("_Z14rsGetElementAt13rs_allocationjjj"); 633 // rsSetElementAt() 634 Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvj"); 635 Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjj"); 636 Funcs.push_back("_Z14rsSetElementAt13rs_allocationPvjjj"); 637 // rsGetElementAtYuv_uchar_Y() 638 Funcs.push_back("_Z25rsGetElementAtYuv_uchar_Y13rs_allocationjj"); 639 // rsGetElementAtYuv_uchar_U() 640 Funcs.push_back("_Z25rsGetElementAtYuv_uchar_U13rs_allocationjj"); 641 // rsGetElementAtYuv_uchar_V() 642 Funcs.push_back("_Z25rsGetElementAtYuv_uchar_V13rs_allocationjj"); 643 644 for (std::vector<std::string>::iterator FI = Funcs.begin(), 645 FE = Funcs.end(); 646 FI != FE; ++FI) { 647 llvm::Function *Function = Module.getFunction(*FI); 648 649 if (!Function) { 650 ALOGE("Missing run-time function '%s'", FI->c_str()); 651 return true; 652 } 653 654 if (Function->getNumUses() > 0) { 655 return true; 656 } 657 } 658 659 return false; 660 } 661 662 /// @brief Connect RenderScript TBAA metadata to C/C++ metadata 663 /// 664 /// The TBAA metadata used to annotate loads/stores from RenderScript 665 /// Allocations is generated in a separate TBAA tree with a "RenderScript TBAA" 666 /// root node. LLVM does assume may-alias for all nodes in unrelated alias 667 /// analysis trees. This function makes the RenderScript TBAA a subtree of the 668 /// normal C/C++ TBAA tree aside of normal C/C++ types. With the connected trees 669 /// every access to an Allocation is resolved to must-alias if compared to 670 /// a normal C/C++ access. 671 void connectRenderScriptTBAAMetadata(llvm::Module &Module) { 672 llvm::MDBuilder MDHelper(*Context); 673 llvm::MDNode *TBAARenderScript = 674 MDHelper.createTBAARoot("RenderScript TBAA"); 675 676 llvm::MDNode *TBAARoot = MDHelper.createTBAARoot("Simple C/C++ TBAA"); 677 llvm::MDNode *TBAAMergedRS = MDHelper.createTBAANode("RenderScript", 678 TBAARoot); 679 680 TBAARenderScript->replaceAllUsesWith(TBAAMergedRS); 681 } 682 683 virtual bool runOnModule(llvm::Module &Module) { 684 bool Changed = false; 685 this->Module = &Module; 686 this->Context = &Module.getContext(); 687 688 this->buildTypes(); 689 690 bcinfo::MetadataExtractor me(&Module); 691 if (!me.extract()) { 692 ALOGE("Could not extract metadata from module!"); 693 return false; 694 } 695 mExportForEachCount = me.getExportForEachSignatureCount(); 696 mExportForEachNameList = me.getExportForEachNameList(); 697 mExportForEachSignatureList = me.getExportForEachSignatureList(); 698 699 bool AllocsExposed = allocPointersExposed(Module); 700 701 for (size_t i = 0; i < mExportForEachCount; ++i) { 702 const char *name = mExportForEachNameList[i]; 703 uint32_t signature = mExportForEachSignatureList[i]; 704 llvm::Function *kernel = Module.getFunction(name); 705 if (kernel) { 706 if (bcinfo::MetadataExtractor::hasForEachSignatureKernel(signature)) { 707 Changed |= ExpandKernel(kernel, signature); 708 kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 709 } else if (kernel->getReturnType()->isVoidTy()) { 710 Changed |= ExpandFunction(kernel, signature); 711 kernel->setLinkage(llvm::GlobalValue::InternalLinkage); 712 } else { 713 // There are some graphics root functions that are not 714 // expanded, but that will be called directly. For those 715 // functions, we can not set the linkage to internal. 716 } 717 } 718 } 719 720 if (gEnableRsTbaa && !AllocsExposed) { 721 connectRenderScriptTBAAMetadata(Module); 722 } 723 724 return Changed; 725 } 726 727 virtual const char *getPassName() const { 728 return "ForEach-able Function Expansion"; 729 } 730 731}; // end RSForEachExpandPass 732 733} // end anonymous namespace 734 735char RSForEachExpandPass::ID = 0; 736 737namespace bcc { 738 739llvm::ModulePass * 740createRSForEachExpandPass(bool pEnableStepOpt){ 741 return new RSForEachExpandPass(pEnableStepOpt); 742} 743 744} // end namespace bcc 745