Lines Matching refs:llvm
31 #include <llvm/IR/DerivedTypes.h>
32 #include <llvm/IR/Function.h>
33 #include <llvm/IR/Instructions.h>
34 #include <llvm/IR/IRBuilder.h>
35 #include <llvm/IR/MDBuilder.h>
36 #include <llvm/IR/Module.h>
37 #include <llvm/Pass.h>
38 #include <llvm/Support/raw_ostream.h>
39 #include <llvm/IR/DataLayout.h>
40 #include <llvm/IR/Function.h>
41 #include <llvm/IR/Type.h>
42 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
89 class RSKernelExpandPass : public llvm::ModulePass {
96 typedef std::unordered_set<llvm::Function *> FunctionSet;
124 llvm::Module *Module;
125 llvm::LLVMContext *Context;
132 llvm::FunctionType *ExpandedForEachType;
133 llvm::Type *RsExpandKernelDriverInfoPfxTy;
144 uint32_t getRootSignature(llvm::Function *Function) {
145 const llvm::NamedMDNode *ExportForEachMetadata =
149 llvm::SmallVector<llvm::Type*, 8> RootArgTys;
150 for (llvm::Function::arg_iterator B = Function->arg_begin(),
171 llvm::MDNode *SigNode = ExportForEachMetadata->getOperand(0);
173 llvm::Metadata *SigMD = SigNode->getOperand(0);
174 if (llvm::MDString *SigS = llvm::dyn_cast<llvm::MDString>(SigMD)) {
175 llvm::StringRef SigString = SigS->getString();
188 bool isStepOptSupported(llvm::Type *AllocType) {
190 llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType);
191 llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context);
211 if (AllocType->getStructName().find("coerce") != llvm::StringRef::npos) {
216 llvm::Type *V2xi64Ty = llvm::VectorType::get(llvm::Type::getInt64Ty(*Context), 2);
217 llvm::Type *Int128Ty = llvm::Type::getIntNTy(*Context, 128);
235 llvm::Value *getStepValue(llvm::DataLayout *DL, llvm::Type *AllocType,
236 llvm::Value *OrigStep) {
240 llvm::PointerType *PT = llvm::dyn_cast<llvm::PointerType>(AllocType);
242 llvm::Type *ET = PT->getElementType();
244 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
245 return llvm::ConstantInt::get(Int32Ty, ETSize);
255 llvm::Type *Int8Ty = llvm::Type::getInt8Ty(*Context);
256 llvm::Type *Int8PtrTy = Int8Ty->getPointerTo();
257 llvm::Type *Int8PtrArrayInputLimitTy = llvm::ArrayType::get(Int8PtrTy, RS_KERNEL_INPUT_LIMIT);
258 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
259 llvm::Type *Int32ArrayInputLimitTy = llvm::ArrayType::get(Int32Ty, RS_KERNEL_INPUT_LIMIT);
260 llvm::Type *VoidPtrTy = llvm::Type::getInt8PtrTy(*Context);
261 llvm::Type *Int32Array4Ty = llvm::ArrayType::get(Int32Ty, 4);
274 llvm::SmallVector<llvm::Type*, RsLaunchDimensionsFieldCount> RsLaunchDimensionsTypes;
281 llvm::StructType *RsLaunchDimensionsTy =
282 llvm::StructType::create(RsLaunchDimensionsTypes, "RsLaunchDimensions");
311 llvm::SmallVector<llvm::Type*, RsExpandKernelDriverInfoPfxFieldCount> RsExpandKernelDriverInfoPfxTypes;
323 llvm::StructType::create(RsExpandKernelDriverInfoPfxTypes, "RsExpandKernelDriverInfoPfx");
326 llvm::Type *VoidTy = llvm::Type::getVoidTy(*Context);
328 llvm::Type *RsExpandKernelDriverInfoPfxPtrTy = RsExpandKernelDriverInfoPfxTy->getPointerTo();
330 ExpandedForEachType = llvm::FunctionType::get(VoidTy,
341 llvm::Function *createEmptyExpandedForEachKernel(llvm::StringRef OldName) {
342 llvm::Function *ExpandedFunction =
343 llvm::Function::Create(ExpandedForEachType,
344 llvm::GlobalValue::ExternalLinkage,
347 llvm::Function::arg_iterator AI = ExpandedFunction->arg_begin();
352 llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin",
354 llvm::IRBuilder<> Builder(Begin);
366 llvm::Function *createEmptyExpandedReduceAccumulator(llvm::StringRef OldName,
367 llvm::Type *AccumArgTy) {
368 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
369 llvm::Type *VoidTy = llvm::Type::getVoidTy(*Context);
370 llvm::FunctionType *ExpandedReduceAccumulatorType =
371 llvm::FunctionType::get(VoidTy,
374 llvm::Function *FnExpandedAccumulator =
375 llvm::Function::Create(ExpandedReduceAccumulatorType,
376 llvm::GlobalValue::ExternalLinkage,
380 llvm::Function::arg_iterator AI = FnExpandedAccumulator->arg_begin();
382 using llvm::Attribute;
384 llvm::Argument *Arg_p = &(*AI++);
386 Arg_p->addAttr(llvm::AttributeSet::get(*Context, Arg_p->getArgNo() + 1,
387 llvm::makeArrayRef(Attribute::NoCapture)));
389 llvm::Argument *Arg_x1 = &(*AI++);
392 llvm::Argument *Arg_x2 = &(*AI++);
395 llvm::Argument *Arg_accum = &(*AI++);
397 Arg_accum->addAttr(llvm::AttributeSet::get(*Context, Arg_accum->getArgNo() + 1,
398 llvm::makeArrayRef(Attribute::NoCapture)));
400 llvm::BasicBlock *Begin = llvm::BasicBlock::Create(*Context, "Begin",
402 llvm::IRBuilder<> Builder(Begin);
425 llvm::BasicBlock *createLoop(llvm::IRBuilder<> &Builder,
426 llvm::Value *LowerBound,
427 llvm::Value *UpperBound,
428 llvm::Value **LoopIV) {
431 llvm::BasicBlock *CondBB, *AfterBB, *HeaderBB;
432 llvm::Value *Cond, *IVNext, *IV, *IVVar;
435 AfterBB = llvm::SplitBlock(CondBB, &*Builder.GetInsertPoint(), nullptr, nullptr);
436 HeaderBB = llvm::BasicBlock::Create(*Context, "Loop", CondBB->getParent());
470 Builder.SetInsertPoint(llvm::cast<llvm::Instruction>(IVNext));
488 void finishArgList( llvm::SmallVector<llvm::Value *, ArgVectorLen> &ArgVector,
489 const llvm::SmallVector<llvm::Value *, SpecialArgVectorLen> &SpecialArgVector,
491 const llvm::Function &CalleeFunction,
492 llvm::IRBuilder<> &CallerBuilder) {
502 llvm::Type *ContextArgType = nullptr;
520 typedef llvm::SmallVector<llvm::Value *, 3> SmallGEPIndices;
523 // SmallVector of llvm::Value*. The return value is suitable for
534 llvm::IntegerType *I32Ty = llvm::Type::getInt32Ty(*Context);
536 [I32Ty](int32_t Arg) { return llvm::ConstantInt::get(I32Ty, Arg); });
547 virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
568 llvm::Value *X,
569 llvm::Value *Arg_p,
570 llvm::IRBuilder<> &Builder,
571 llvm::SmallVector<llvm::Value*, 8> &CalleeArgs,
573 llvm::Instruction *LoopHeaderInsertionPoint) {
601 llvm::Value *YAddr = Builder.CreateInBoundsGEP(Arg_p, YValueGEP, "Y.gep");
609 llvm::Value *ZAddr = Builder.CreateInBoundsGEP(Arg_p, ZValueGEP, "Z.gep");
638 void ExpandInputsLoopInvariant(llvm::IRBuilder<> &Builder, llvm::BasicBlock *LoopHeader,
639 llvm::Value *Arg_p,
640 llvm::MDNode *TBAAPointer,
641 llvm::Function::arg_iterator ArgIter,
643 llvm::SmallVectorImpl<llvm::Type *> &InTypes,
644 llvm::SmallVectorImpl<llvm::Value *> &InBufPtrs,
645 llvm::SmallVectorImpl<llvm::Value *> &InStructTempSlots) {
654 llvm::Type *InType = ArgIter->getType();
666 if (auto PtrType = llvm::dyn_cast<llvm::PointerType>(InType)) {
667 llvm::Type *ElementType = PtrType->getElementType();
677 llvm::Value *InBufPtrAddr = Builder.CreateInBoundsGEP(Arg_p, InBufPtrGEP, "input_buf.gep");
678 llvm::LoadInst *InBufPtr = Builder.CreateLoad(InBufPtrAddr, "input_buf");
680 llvm::Value *CastInBufPtr = nullptr;
720 void ExpandInputsBody(llvm::IRBuilder<> &Builder,
721 llvm::Value *Arg_x1,
722 llvm::MDNode *TBAAAllocation,
724 const llvm::SmallVectorImpl<llvm::Type *> &InTypes,
725 const llvm::SmallVectorImpl<llvm::Value *> &InBufPtrs,
726 const llvm::SmallVectorImpl<llvm::Value *> &InStructTempSlots,
727 llvm::Value *IndVar,
728 llvm::SmallVectorImpl<llvm::Value *> &RootArgs) {
729 llvm::Value *Offset = Builder.CreateSub(IndVar, Arg_x1);
730 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
734 llvm::Value *InPtr = nullptr;
741 llvm::DataLayout DL(X86_CUSTOM_DL_STRING);
742 llvm::Type *InTy = InTypes[Index];
744 llvm::Value *OffsetInBytes = Builder.CreateMul(Offset, llvm::ConstantInt::get(Int32Ty, InStep));
749 llvm::Value *Input;
750 llvm::LoadInst *InputLoad = Builder.CreateLoad(InPtr, "input");
756 if (llvm::Value *TemporarySlot = InStructTempSlots[Index]) {
777 bool ExpandOldStyleForEach(llvm::Function *Function, uint32_t Signature) {
790 llvm::DataLayout DL(Module);
795 llvm::Function *ExpandedFunction =
805 llvm::Function::arg_iterator ExpandedFunctionArgIter =
808 llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++);
809 llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++);
810 llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++);
811 llvm::Value *Arg_outstep = &*(ExpandedFunctionArgIter);
813 llvm::Value *InStep = nullptr;
814 llvm::Value *OutStep = nullptr;
817 llvm::IRBuilder<> Builder(&*ExpandedFunction->getEntryBlock().begin());
821 llvm::Function::arg_iterator FunctionArgIter = Function->arg_begin();
823 llvm::Type *InTy = nullptr;
824 llvm::Value *InBufPtr = nullptr;
827 llvm::LoadInst *InStepArg = Builder.CreateLoad(
840 llvm::Type *OutTy = nullptr;
841 llvm::Value *OutBasePtr = nullptr;
850 llvm::Value *UsrData = nullptr;
852 llvm::Type *UsrDataTy = (FunctionArgIter++)->getType();
853 llvm::Value *UsrDataPointerAddr = Builder.CreateStructGEP(nullptr, Arg_p, RsExpandKernelDriverInfoPfxFieldUsr);
858 llvm::BasicBlock *LoopHeader = Builder.GetInsertBlock();
859 llvm::Value *IV;
862 llvm::SmallVector<llvm::Value*, 8> CalleeArgs;
870 llvm::SmallVector<llvm::Value*, 8> RootArgs;
872 llvm::Value *InPtr = nullptr;
873 llvm::Value *OutPtr = nullptr;
884 llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1);
891 llvm::Value *InOffset = Builder.CreateSub(IV, Arg_x1);
918 bool ExpandForEach(llvm::Function *Function, uint32_t Signature) {
923 llvm::DataLayout DL(Module);
927 llvm::Type *Int32Ty = llvm::Type::getInt32Ty(*Context);
929 llvm::Function *ExpandedFunction =
939 llvm::Function::arg_iterator ExpandedFunctionArgIter =
942 llvm::Value *Arg_p = &*(ExpandedFunctionArgIter++);
943 llvm::Value *Arg_x1 = &*(ExpandedFunctionArgIter++);
944 llvm::Value *Arg_x2 = &*(ExpandedFunctionArgIter++);
948 llvm::IRBuilder<> Builder(&*ExpandedFunction->getEntryBlock().begin());
951 llvm::MDNode *TBAARenderScriptDistinct, *TBAARenderScript,
953 llvm::MDBuilder MDHelper(*Context);
978 llvm::Function::arg_iterator ArgIter = Function->arg_begin();
981 llvm::Type *OutTy = nullptr;
982 llvm::LoadInst *OutBasePtr = nullptr;
983 llvm::Value *CastedOutBasePtr = nullptr;
988 llvm::Type *OutBaseTy = Function->getReturnType();
1021 llvm::SmallVector<llvm::Type*, 8> InTypes;
1022 llvm::SmallVector<llvm::Value*, 8> InBufPtrs;
1023 llvm::SmallVector<llvm::Value*, 8> InStructTempSlots;
1028 llvm::BasicBlock *LoopHeader = Builder.GetInsertBlock();
1029 llvm::Value *IV;
1032 llvm::SmallVector<llvm::Value*, 8> CalleeArgs;
1050 llvm::SmallVector<llvm::Value*, 8> RootArgs;
1056 llvm::Value *OutPtr = nullptr;
1058 llvm::Value *OutOffset = Builder.CreateSub(IV, Arg_x1);
1067 llvm::Value *OutOffsetInBytes = Builder.CreateMul(OutOffset, llvm::ConstantInt::get(Int32Ty, OutStep));
1086 llvm::Value *RetVal = Builder.CreateCall(Function, RootArgs);
1090 llvm::StoreInst *Store = Builder.CreateStore(RetVal, OutPtr);
1111 llvm::Function *Fn = Module->getFunction(Name);
1114 bccAssert(Fn->getLinkage() == llvm::GlobalValue::InternalLinkage);
1115 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1145 bool ExpandReduceAccumulator(llvm::Function *FnAccumulator, uint32_t Signature, size_t NumInputs) {
1150 llvm::MDNode *TBAARenderScriptDistinct, *TBAARenderScript,
1152 llvm::MDBuilder MDHelper(*Context);
1168 llvm::Function *FnExpandedAccumulator =
1177 llvm::Value *Arg_p = &*(ExpandedAccumulatorArgIter++);
1178 llvm::Value *Arg_x1 = &*(ExpandedAccumulatorArgIter++);
1179 llvm::Value *Arg_x2 = &*(ExpandedAccumulatorArgIter++);
1180 llvm::Value *Arg_accum = &*(ExpandedAccumulatorArgIter++);
1183 llvm::IRBuilder<> Builder(&*FnExpandedAccumulator->getEntryBlock().begin());
1186 llvm::BasicBlock *LoopHeader = Builder.GetInsertBlock();
1187 llvm::Value *IndVar;
1190 llvm::SmallVector<llvm::Value*, 8> CalleeArgs;
1195 llvm::SmallVector<llvm::Type*, 8> InTypes;
1196 llvm::SmallVector<llvm::Value*, 8> InBufPtrs;
1197 llvm::SmallVector<llvm::Value*, 8> InStructTempSlots;
1202 llvm::SmallVector<llvm::Value*, 8> RootArgs;
1225 bool CreateReduceCombinerFromAccumulator(llvm::Function *FnAccumulator) {
1229 using llvm::Attribute;
1233 llvm::Value *AccumulatorArg_accum = &*(AccumulatorArgIter++);
1234 llvm::Value *AccumulatorArg_in = &*(AccumulatorArgIter++);
1235 llvm::Type *AccumulatorArgType = AccumulatorArg_accum->getType();
1238 llvm::Type *VoidTy = llvm::Type::getVoidTy(*Context);
1239 llvm::FunctionType *CombinerType =
1240 llvm::FunctionType::get(VoidTy, { AccumulatorArgType, AccumulatorArgType }, false);
1241 llvm::Function *FnCombiner =
1242 llvm::Function::Create(CombinerType, llvm::GlobalValue::ExternalLinkage,
1248 llvm::Argument *CombinerArg_accum = &(*CombinerArgIter++);
1250 CombinerArg_accum->addAttr(llvm::AttributeSet::get(*Context, CombinerArg_accum->getArgNo() + 1,
1251 llvm::makeArrayRef(Attribute::NoCapture)));
1253 llvm::Argument *CombinerArg_other = &(*CombinerArgIter++);
1255 CombinerArg_other->addAttr(llvm::AttributeSet::get(*Context, CombinerArg_other->getArgNo() + 1,
1256 llvm::makeArrayRef(Attribute::NoCapture)));
1258 llvm::BasicBlock *BB = llvm::BasicBlock::Create(*Context, "BB", FnCombiner);
1259 llvm::IRBuilder<> Builder(BB);
1266 llvm::Type *ElementType = AccumulatorArg_in->getType()->getPointerElementType();
1267 llvm::Value *TempMem = Builder.CreateAlloca(ElementType, nullptr, "caller_copy");
1271 llvm::Value *TypeAdjustedOther = CombinerArg_other;
1278 llvm::Value *DerefOther = Builder.CreateLoad(TypeAdjustedOther);
1296 bool allocPointersExposed(llvm::Module &Module) {
1333 llvm::Function *Function = Module.getFunction(FI);
1358 void connectRenderScriptTBAAMetadata(llvm::Module &Module) {
1359 llvm::MDBuilder MDHelper(*Context);
1360 llvm::MDNode *TBAARenderScriptDistinct =
1362 llvm::MDNode *TBAARenderScript = MDHelper.createTBAANode(
1364 llvm::MDNode *TBAARoot = MDHelper.createTBAARoot("Simple C/C++ TBAA");
1368 virtual bool runOnModule(llvm::Module &Module) {
1391 llvm::Function *kernel = Module.getFunction(name);
1395 kernel->setLinkage(llvm::GlobalValue::InternalLinkage);
1398 kernel->setLinkage(llvm::GlobalValue::InternalLinkage);
1419 llvm::Function *accumulator = Module.getFunction(ExportReduceList[i].mAccumulatorName);
1447 static llvm::RegisterPass<RSKernelExpandPass> X("kernelexp", "Kernel Expand Pass");
1453 llvm::ModulePass *