16bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===// 26bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// 36bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// The LLVM Compiler Infrastructure 46bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// 56bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// This file is distributed under the University of Illinois Open Source 66bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// License. See LICENSE.TXT for details. 76bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// 86bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines//===----------------------------------------------------------------------===// 96bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// 106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// This contains code to emit OpenMP nodes as LLVM code. 116bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// 126bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines//===----------------------------------------------------------------------===// 136bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 146bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines#include "CGOpenMPRuntime.h" 156bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines#include "CodeGenFunction.h" 166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines#include "CodeGenModule.h" 170e2c34f92f00628d48968dfea096d36381f494cbStephen Hines#include "TargetInfo.h" 186bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines#include "clang/AST/Stmt.h" 196bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines#include "clang/AST/StmtOpenMP.h" 206bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesusing namespace clang; 216bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesusing namespace CodeGen; 226bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 236bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines//===----------------------------------------------------------------------===// 246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines// OpenMP Directive Emission 256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines//===----------------------------------------------------------------------===// 26176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen 27176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// function. Here is the logic: 28176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// if (Cond) { 29176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// CodeGen(true); 30176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// } else { 31176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// CodeGen(false); 32176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// } 33176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic void EmitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, 34176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines const std::function<void(bool)> &CodeGen) { 35176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); 36176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 37176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // If the condition constant folds and can be elided, try to avoid emitting 38176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // the condition and the dead arm of the if/else. 39176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines bool CondConstant; 40176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { 41176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CodeGen(CondConstant); 42176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return; 43176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 44176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 45176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Otherwise, the condition did not fold, or we couldn't elide it. Just 46176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // emit the conditional branch. 47176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto ThenBlock = CGF.createBasicBlock(/*name*/ "omp_if.then"); 48176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto ElseBlock = CGF.createBasicBlock(/*name*/ "omp_if.else"); 49176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto ContBlock = CGF.createBasicBlock(/*name*/ "omp_if.end"); 50176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount*/ 0); 516bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 52176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit the 'then' code. 53176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitBlock(ThenBlock); 54176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CodeGen(/*ThenBlock*/ true); 55176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitBranch(ContBlock); 56176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit the 'else' code if present. 576bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines { 58176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // There is no need to emit line number for unconditional branch. 590e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto NL = ApplyDebugLocation::CreateEmpty(CGF); 60176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitBlock(ElseBlock); 61176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 62176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CodeGen(/*ThenBlock*/ false); 63176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines { 64176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // There is no need to emit line number for unconditional branch. 650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto NL = ApplyDebugLocation::CreateEmpty(CGF); 66176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitBranch(ContBlock); 67176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 68176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit the continuation block for code after the if. 69176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitBlock(ContBlock, /*IsFinished*/ true); 70176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 71176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 7233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPAggregateAssign( 7333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, 7433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen) { 7533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Perform element-by-element initialization. 7633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar QualType ElementTy; 7733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto SrcBegin = SrcAddr; 7833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto DestBegin = DestAddr; 7933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); 8033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); 8133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Cast from pointer to array type to pointer to single element. 8233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SrcBegin = Builder.CreatePointerBitCastOrAddrSpaceCast(SrcBegin, 8333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DestBegin->getType()); 8433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto DestEnd = Builder.CreateGEP(DestBegin, NumElements); 8533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // The basic structure here is a while-do loop. 8633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto BodyBB = createBasicBlock("omp.arraycpy.body"); 8733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto DoneBB = createBasicBlock("omp.arraycpy.done"); 8833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IsEmpty = 8933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); 9033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); 9133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 9233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Enter the loop body, making that address the current address. 9333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto EntryBB = Builder.GetInsertBlock(); 9433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitBlock(BodyBB); 9533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto SrcElementCurrent = 9633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); 9733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SrcElementCurrent->addIncoming(SrcBegin, EntryBB); 9833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto DestElementCurrent = Builder.CreatePHI(DestBegin->getType(), 2, 9933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar "omp.arraycpy.destElementPast"); 10033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DestElementCurrent->addIncoming(DestBegin, EntryBB); 10133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 10233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit copy. 10333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CopyGen(DestElementCurrent, SrcElementCurrent); 10433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 10533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Shift the address forward by one element. 10633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto DestElementNext = Builder.CreateConstGEP1_32( 10733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DestElementCurrent, /*Idx0=*/1, "omp.arraycpy.dest.element"); 10833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto SrcElementNext = Builder.CreateConstGEP1_32( 10933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SrcElementCurrent, /*Idx0=*/1, "omp.arraycpy.src.element"); 11033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Check whether we've reached the end. 11133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto Done = 11233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); 11333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreateCondBr(Done, DoneBB, BodyBB); 11433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DestElementCurrent->addIncoming(DestElementNext, Builder.GetInsertBlock()); 11533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SrcElementCurrent->addIncoming(SrcElementNext, Builder.GetInsertBlock()); 11633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 11733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Done. 11833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitBlock(DoneBB, /*IsFinished=*/true); 11933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 120176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 12133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPCopy(CodeGenFunction &CGF, 12233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar QualType OriginalType, llvm::Value *DestAddr, 12333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::Value *SrcAddr, const VarDecl *DestVD, 12433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const VarDecl *SrcVD, const Expr *Copy) { 12533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (OriginalType->isArrayType()) { 12633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *BO = dyn_cast<BinaryOperator>(Copy); 12733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (BO && BO->getOpcode() == BO_Assign) { 12833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Perform simple memcpy for simple copying. 12933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitAggregateAssign(DestAddr, SrcAddr, OriginalType); 13033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } else { 13133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // For arrays with complex element types perform element by element 13233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // copying. 13333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPAggregateAssign( 13433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DestAddr, SrcAddr, OriginalType, 13533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar [&CGF, Copy, SrcVD, DestVD](llvm::Value *DestElement, 13633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::Value *SrcElement) { 13733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Working with the single array element, so have to remap 13833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // destination and source variables to corresponding array 13933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // elements. 14033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CodeGenFunction::OMPPrivateScope Remap(CGF); 14133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Remap.addPrivate(DestVD, [DestElement]() -> llvm::Value *{ 14233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return DestElement; 14333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 14433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Remap.addPrivate( 14533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SrcVD, [SrcElement]() -> llvm::Value *{ return SrcElement; }); 14633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)Remap.Privatize(); 14733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitIgnoredExpr(Copy); 14833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 14933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 15033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } else { 15133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Remap pseudo source variable to private copy. 15233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CodeGenFunction::OMPPrivateScope Remap(CGF); 15333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Remap.addPrivate(SrcVD, [SrcAddr]() -> llvm::Value *{ return SrcAddr; }); 15433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Remap.addPrivate(DestVD, [DestAddr]() -> llvm::Value *{ return DestAddr; }); 15533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)Remap.Privatize(); 15633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit copying of the whole variable. 15733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitIgnoredExpr(Copy); 158176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 159176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 160176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 16133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarbool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, 16233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPPrivateScope &PrivateScope) { 16333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto FirstprivateFilter = [](const OMPClause *C) -> bool { 164176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return C->getClauseKind() == OMPC_firstprivate; 165176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines }; 16633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate; 16733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (OMPExecutableDirective::filtered_clause_iterator<decltype( 16833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar FirstprivateFilter)> I(D.clauses(), FirstprivateFilter); 16933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar I; ++I) { 170176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto *C = cast<OMPFirstprivateClause>(*I); 171176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto IRef = C->varlist_begin(); 172176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto InitsRef = C->inits().begin(); 173176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines for (auto IInit : C->private_copies()) { 174176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); 17533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (EmittedAsFirstprivate.count(OrigVD) == 0) { 17633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmittedAsFirstprivate.insert(OrigVD); 17733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); 17833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl()); 17933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool IsRegistered; 18033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DeclRefExpr DRE( 18133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const_cast<VarDecl *>(OrigVD), 18233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( 18333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OrigVD) != nullptr, 18433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); 18533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *OriginalAddr = EmitLValue(&DRE).getAddress(); 18633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (OrigVD->getType()->isArrayType()) { 18733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit VarDecl with copy init for arrays. 18833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Get the address of the original variable captured in current 18933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // captured region. 19033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ 19133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto Emission = EmitAutoVarAlloca(*VD); 19233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *Init = VD->getInit(); 19333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) { 19433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Perform simple memcpy. 19533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr, 19633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (*IRef)->getType()); 19733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } else { 19833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPAggregateAssign( 19933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Emission.getAllocatedAddress(), OriginalAddr, 20033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (*IRef)->getType(), 20133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar [this, VDInit, Init](llvm::Value *DestElement, 20233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::Value *SrcElement) { 20333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Clean up any temporaries needed by the initialization. 20433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RunCleanupsScope InitScope(*this); 20533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit initialization for single element. 20633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LocalDeclMap[VDInit] = SrcElement; 20733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitAnyExprToMem(Init, DestElement, 20833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Init->getType().getQualifiers(), 20933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar /*IsInitializer*/ false); 21033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LocalDeclMap.erase(VDInit); 21133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 21233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 21333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitAutoVarCleanups(Emission); 21433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return Emission.getAllocatedAddress(); 21533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 21633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } else { 21733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ 21833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit private VarDecl with copy init. 21933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Remap temp VDInit variable to the address of the original 22033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // variable 22133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // (for proper handling of captured global variables). 22233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LocalDeclMap[VDInit] = OriginalAddr; 22333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitDecl(*VD); 22433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LocalDeclMap.erase(VDInit); 22533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return GetAddrOfLocalVar(VD); 22633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 22733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 22833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar assert(IsRegistered && 22933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar "firstprivate var already registered as private"); 23033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Silence the warning about unused variable. 23133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)IsRegistered; 23233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 233176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ++IRef, ++InitsRef; 234176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 235176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 23633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return !EmittedAsFirstprivate.empty(); 237176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 238176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 239176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPPrivateClause( 240176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines const OMPExecutableDirective &D, 241176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CodeGenFunction::OMPPrivateScope &PrivateScope) { 242176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto PrivateFilter = [](const OMPClause *C) -> bool { 243176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return C->getClauseKind() == OMPC_private; 244176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines }; 245176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines for (OMPExecutableDirective::filtered_clause_iterator<decltype(PrivateFilter)> 246176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines I(D.clauses(), PrivateFilter); I; ++I) { 247176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto *C = cast<OMPPrivateClause>(*I); 248176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto IRef = C->varlist_begin(); 249176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines for (auto IInit : C->private_copies()) { 250176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); 251176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); 252176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines bool IsRegistered = 253176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { 254176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit private VarDecl with copy init. 255176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitDecl(*VD); 256176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return GetAddrOfLocalVar(VD); 257176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines }); 2583ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar assert(IsRegistered && "private var already registered as private"); 259176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Silence the warning about unused variable. 260176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines (void)IsRegistered; 261176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ++IRef; 262176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 263176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 264176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 265176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 26633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarbool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { 26733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // threadprivate_var1 = master_threadprivate_var1; 26833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // operator=(threadprivate_var2, master_threadprivate_var2); 26933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // ... 27033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // __kmpc_barrier(&loc, global_tid); 27133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto CopyinFilter = [](const OMPClause *C) -> bool { 27233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return C->getClauseKind() == OMPC_copyin; 27333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 27433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::DenseSet<const VarDecl *> CopiedVars; 27533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr; 27633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (OMPExecutableDirective::filtered_clause_iterator<decltype(CopyinFilter)> 27733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar I(D.clauses(), CopyinFilter); 27833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar I; ++I) { 27933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *C = cast<OMPCopyinClause>(*I); 28033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IRef = C->varlist_begin(); 28133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ISrcRef = C->source_exprs().begin(); 28233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IDestRef = C->destination_exprs().begin(); 28333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto *AssignOp : C->assignment_ops()) { 28433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); 28533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (CopiedVars.insert(VD->getCanonicalDecl()).second) { 28633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Get the address of the master variable. 28733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *MasterAddr = VD->isStaticLocal() 28833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ? CGM.getStaticLocalDeclAddress(VD) 28933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar : CGM.GetAddrOfGlobal(VD); 29033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Get the address of the threadprivate variable. 29133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *PrivateAddr = EmitLValue(*IRef).getAddress(); 29233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (CopiedVars.size() == 1) { 29333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // At first check if current thread is a master thread. If it is, no 29433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // need to copy data. 29533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CopyBegin = createBasicBlock("copyin.not.master"); 29633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CopyEnd = createBasicBlock("copyin.not.master.end"); 29733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreateCondBr( 29833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreateICmpNE( 29933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreatePtrToInt(MasterAddr, CGM.IntPtrTy), 30033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreatePtrToInt(PrivateAddr, CGM.IntPtrTy)), 30133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CopyBegin, CopyEnd); 30233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitBlock(CopyBegin); 30333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 30433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); 30533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); 30633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPCopy(*this, (*IRef)->getType(), PrivateAddr, MasterAddr, DestVD, 30733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SrcVD, AssignOp); 30833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 30933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++IRef; 31033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++ISrcRef; 31133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++IDestRef; 31233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 31333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 31433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (CopyEnd) { 31533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Exit out of copying procedure for non-master thread. 31633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitBlock(CopyEnd, /*IsFinished=*/true); 31733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return true; 31833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 31933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return false; 32033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 32133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 32233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarbool CodeGenFunction::EmitOMPLastprivateClauseInit( 32333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { 32433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto LastprivateFilter = [](const OMPClause *C) -> bool { 32533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return C->getClauseKind() == OMPC_lastprivate; 32633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 32733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool HasAtLeastOneLastprivate = false; 32833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; 32933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (OMPExecutableDirective::filtered_clause_iterator<decltype( 33033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LastprivateFilter)> I(D.clauses(), LastprivateFilter); 33133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar I; ++I) { 33233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *C = cast<OMPLastprivateClause>(*I); 33333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IRef = C->varlist_begin(); 33433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IDestRef = C->destination_exprs().begin(); 33533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto *IInit : C->private_copies()) { 33633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Keep the address of the original variable for future update at the end 33733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // of the loop. 33833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); 33933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) { 34033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); 34133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> llvm::Value *{ 34233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DeclRefExpr DRE( 34333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const_cast<VarDecl *>(OrigVD), 34433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( 34533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OrigVD) != nullptr, 34633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); 34733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return EmitLValue(&DRE).getAddress(); 34833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 34933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Check if the variable is also a firstprivate: in this case IInit is 35033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // not generated. Initialization of this variable will happen in codegen 35133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // for 'firstprivate' clause. 35233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (!IInit) 35333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar continue; 35433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); 35533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool IsRegistered = 35633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ 35733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit private VarDecl with copy init. 35833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitDecl(*VD); 35933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return GetAddrOfLocalVar(VD); 36033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 36133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar assert(IsRegistered && "lastprivate var already registered as private"); 36233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar HasAtLeastOneLastprivate = HasAtLeastOneLastprivate || IsRegistered; 36333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 36433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++IRef, ++IDestRef; 36533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 36633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 36733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return HasAtLeastOneLastprivate; 36833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 36933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 37033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPLastprivateClauseFinal( 37133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPExecutableDirective &D, llvm::Value *IsLastIterCond) { 37233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit following code: 37333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // if (<IsLastIterCond>) { 37433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // orig_var1 = private_orig_var1; 37533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // ... 37633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // orig_varn = private_orig_varn; 37733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // } 37833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *ThenBB = createBasicBlock(".omp.lastprivate.then"); 37933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *DoneBB = createBasicBlock(".omp.lastprivate.done"); 38033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); 38133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitBlock(ThenBB); 38233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar { 38333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto LastprivateFilter = [](const OMPClause *C) -> bool { 38433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return C->getClauseKind() == OMPC_lastprivate; 38533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 38633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; 38733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (OMPExecutableDirective::filtered_clause_iterator<decltype( 38833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LastprivateFilter)> I(D.clauses(), LastprivateFilter); 38933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar I; ++I) { 39033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *C = cast<OMPLastprivateClause>(*I); 39133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IRef = C->varlist_begin(); 39233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ISrcRef = C->source_exprs().begin(); 39333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IDestRef = C->destination_exprs().begin(); 39433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto *AssignOp : C->assignment_ops()) { 39533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); 39633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (AlreadyEmittedVars.insert(PrivateVD->getCanonicalDecl()).second) { 39733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); 39833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); 39933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Get the address of the original variable. 40033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *OriginalAddr = GetAddrOfLocalVar(DestVD); 40133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Get the address of the private variable. 40233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *PrivateAddr = GetAddrOfLocalVar(PrivateVD); 40333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPCopy(*this, (*IRef)->getType(), OriginalAddr, PrivateAddr, 40433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DestVD, SrcVD, AssignOp); 40533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 40633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++IRef; 40733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++ISrcRef; 40833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++IDestRef; 40933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 41033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 41133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 41233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitBlock(DoneBB, /*IsFinished=*/true); 41333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 41433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 41533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPReductionClauseInit( 41633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPExecutableDirective &D, 41733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CodeGenFunction::OMPPrivateScope &PrivateScope) { 41833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ReductionFilter = [](const OMPClause *C) -> bool { 41933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return C->getClauseKind() == OMPC_reduction; 42033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 42133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (OMPExecutableDirective::filtered_clause_iterator<decltype( 42233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ReductionFilter)> I(D.clauses(), ReductionFilter); 42333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar I; ++I) { 42433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *C = cast<OMPReductionClause>(*I); 42533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ILHS = C->lhs_exprs().begin(); 42633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto IRHS = C->rhs_exprs().begin(); 42733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto IRef : C->varlists()) { 42833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl()); 42933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); 43033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); 43133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Store the address of the original variable associated with the LHS 43233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // implicit variable. 43333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> llvm::Value *{ 43433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), 43533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CapturedStmtInfo->lookup(OrigVD) != nullptr, 43633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar IRef->getType(), VK_LValue, IRef->getExprLoc()); 43733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return EmitLValue(&DRE).getAddress(); 43833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 43933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit reduction copy. 44033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool IsRegistered = 44133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> llvm::Value *{ 44233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit private VarDecl with reduction init. 44333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitDecl(*PrivateVD); 44433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return GetAddrOfLocalVar(PrivateVD); 44533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 44633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar assert(IsRegistered && "private var already registered as private"); 44733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Silence the warning about unused variable. 44833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)IsRegistered; 44933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ++ILHS, ++IRHS; 45033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 45133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 45233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 45333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 45433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPReductionClauseFinal( 45533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPExecutableDirective &D) { 45633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::SmallVector<const Expr *, 8> LHSExprs; 45733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::SmallVector<const Expr *, 8> RHSExprs; 45833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::SmallVector<const Expr *, 8> ReductionOps; 45933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ReductionFilter = [](const OMPClause *C) -> bool { 46033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return C->getClauseKind() == OMPC_reduction; 46133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 46233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool HasAtLeastOneReduction = false; 46333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (OMPExecutableDirective::filtered_clause_iterator<decltype( 46433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ReductionFilter)> I(D.clauses(), ReductionFilter); 46533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar I; ++I) { 46633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar HasAtLeastOneReduction = true; 46733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *C = cast<OMPReductionClause>(*I); 46833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end()); 46933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end()); 47033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end()); 47133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 47233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (HasAtLeastOneReduction) { 47333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit nowait reduction if nowait clause is present or directive is a 47433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // parallel directive (it always has implicit barrier). 47533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitReduction( 47633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar *this, D.getLocEnd(), LHSExprs, RHSExprs, ReductionOps, 47733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar D.getSingleClause(OMPC_nowait) || 47833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar isOpenMPParallelDirective(D.getDirectiveKind())); 47933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 48033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 48133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 482176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines/// \brief Emits code for OpenMP parallel directive in the parallel region. 48333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarstatic void emitOMPParallelCall(CodeGenFunction &CGF, 48433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPExecutableDirective &S, 485176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *OutlinedFn, 486176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *CapturedStruct) { 487176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (auto C = S.getSingleClause(/*K*/ OMPC_num_threads)) { 488176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); 489176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto NumThreadsClause = cast<OMPNumThreadsClause>(C); 490176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), 491176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines /*IgnoreResultAssign*/ true); 4920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( 493176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF, NumThreads, NumThreadsClause->getLocStart()); 494176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 4950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn, 4960e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CapturedStruct); 497176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 498176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 49933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarstatic void emitCommonOMPParallelDirective(CodeGenFunction &CGF, 50033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPExecutableDirective &S, 50133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const RegionCodeGenTy &CodeGen) { 502176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); 50333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto CapturedStruct = CGF.GenerateCapturedStmtArgument(*CS); 50433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( 50533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar S, *CS->getCapturedDecl()->param_begin(), CodeGen); 506176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (auto C = S.getSingleClause(/*K*/ OMPC_if)) { 507176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto Cond = cast<OMPIfClause>(C)->getCondition(); 50833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPIfClause(CGF, Cond, [&](bool ThenBlock) { 509176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (ThenBlock) 51033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar emitOMPParallelCall(CGF, S, OutlinedFn, CapturedStruct); 511176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines else 51233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitSerialCall(CGF, S.getLocStart(), 51333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OutlinedFn, CapturedStruct); 514176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines }); 515176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } else 51633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar emitOMPParallelCall(CGF, S, OutlinedFn, CapturedStruct); 51733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 51833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 51933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { 52033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 52133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit parallel region as a standalone region. 52233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 52333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPPrivateScope PrivateScope(CGF); 52433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool Copyins = CGF.EmitOMPCopyinClause(S); 52533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool Firstprivates = CGF.EmitOMPFirstprivateClause(S, PrivateScope); 52633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (Copyins || Firstprivates) { 52733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit implicit barrier to synchronize threads and avoid data races on 52833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // initialization of firstprivate variables or propagation master's thread 52933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // values of threadprivate variables to local instances of that variables 53033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // of all other implicit threads. 53133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), 53233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPD_unknown); 53333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 53433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPPrivateClause(S, PrivateScope); 53533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPReductionClauseInit(S, PrivateScope); 53633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)PrivateScope.Privatize(); 53733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); 53833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPReductionClauseFinal(S); 53933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit implicit barrier at the end of the 'parallel' directive. 54033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), 54133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPD_unknown); 54233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 54333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar emitCommonOMPParallelDirective(*this, S, CodeGen); 544176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 545176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 546176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, 547176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines bool SeparateIter) { 548176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines RunCleanupsScope BodyScope(*this); 549176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Update counters values on current iteration. 550176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines for (auto I : S.updates()) { 551176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitIgnoredExpr(I); 552176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 5533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Update the linear variables. 5543ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { 5553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar for (auto U : C->updates()) { 5563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(U); 5573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 5583ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 5593ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 560176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // On a continue in the body, jump to the end. 561176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto Continue = getJumpDestInCurrentScope("omp.body.continue"); 562176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue)); 563176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit loop body. 564176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitStmt(S.getBody()); 565176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // The end (updates/cleanups). 566176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBlock(Continue.getBlock()); 567176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines BreakContinueStack.pop_back(); 568176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (SeparateIter) { 569176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // TODO: Update lastprivates if the SeparateIter flag is true. 570176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // This will be implemented in a follow-up OMPLastprivateClause patch, but 571176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // result should be still correct without it, as we do not make these 572176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // variables private yet. 573176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 574176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 575176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 57633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPInnerLoop( 57733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, 57833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const Expr *IncExpr, 57933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const llvm::function_ref<void(CodeGenFunction &)> &BodyGen) { 580176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end"); 581176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto Cnt = getPGORegionCounter(&S); 582176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 583176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Start the loop with a block that tests the condition. 584176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto CondBlock = createBasicBlock("omp.inner.for.cond"); 585176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBlock(CondBlock); 586176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines LoopStack.push(CondBlock); 587176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 588176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // If there are any cleanups between here and the loop-exit scope, 589176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // create a block to stage a loop exit along. 590176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto ExitBlock = LoopExit.getBlock(); 5913ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (RequiresCleanup) 592176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup"); 593176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 594176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto LoopBody = createBasicBlock("omp.inner.for.body"); 595176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 5963ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit condition. 5973ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, Cnt.getCount()); 598176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (ExitBlock != LoopExit.getBlock()) { 599176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBlock(ExitBlock); 600176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBranchThroughCleanup(LoopExit); 601176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 602176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 603176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBlock(LoopBody); 604176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Cnt.beginRegion(Builder); 605176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 606176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Create a block for the increment. 607176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); 608176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); 609176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 61033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar BodyGen(*this); 611176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 612176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit "IV = IV + 1" and a back-edge to the condition block. 613176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBlock(Continue.getBlock()); 6143ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(IncExpr); 615176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines BreakContinueStack.pop_back(); 616176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBranch(CondBlock); 617176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines LoopStack.pop(); 618176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit the fall-through block. 619176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitBlock(LoopExit.getBlock()); 620176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 621176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 622176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &S) { 623176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto IC = S.counters().begin(); 624176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines for (auto F : S.finals()) { 625176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (LocalDeclMap.lookup(cast<DeclRefExpr>((*IC))->getDecl())) { 626176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines EmitIgnoredExpr(F); 627176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 628176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ++IC; 629176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 6303ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit the final values of the linear variables. 6313ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { 6323ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar for (auto F : C->finals()) { 6333ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(F); 6343ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 6353ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 636176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 637176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 638176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM, 639176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines const OMPAlignedClause &Clause) { 640176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned ClauseAlignment = 0; 641176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (auto AlignmentExpr = Clause.getAlignment()) { 642176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto AlignmentCI = 643176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); 644176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue()); 6456bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 646176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines for (auto E : Clause.varlists()) { 647176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines unsigned Alignment = ClauseAlignment; 648176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (Alignment == 0) { 649176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // OpenMP [2.8.1, Description] 650176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // If no optional parameter is specified, implementation-defined default 651176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // alignments for SIMD instructions on the target platforms are assumed. 652176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Alignment = CGM.getTargetCodeGenInfo().getOpenMPSimdDefaultAlignment( 653176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines E->getType()); 654176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 655176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && 656176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines "alignment is not power of 2"); 657176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines if (Alignment != 0) { 658176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Value *PtrValue = CGF.EmitScalarExpr(E); 659176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitAlignmentAssumption(PtrValue, Alignment); 660176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 661176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 662176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 6636bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 664176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesstatic void EmitPrivateLoopCounters(CodeGenFunction &CGF, 665176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CodeGenFunction::OMPPrivateScope &LoopScope, 666176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ArrayRef<Expr *> Counters) { 667176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines for (auto *E : Counters) { 668176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 669176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines bool IsRegistered = LoopScope.addPrivate(VD, [&]() -> llvm::Value * { 670176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Emit var without initialization. 671176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines auto VarEmission = CGF.EmitAutoVarAlloca(*VD); 672176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.EmitAutoVarCleanups(VarEmission); 673176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines return VarEmission.getAllocatedAddress(); 674176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines }); 675176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines assert(IsRegistered && "counter already registered as private"); 676176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Silence the warning about unused variable. 677176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines (void)IsRegistered; 678176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 6793ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 6803ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 6813ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarstatic void 6823ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga NainarEmitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, 6833ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CodeGenFunction::OMPPrivateScope &PrivateScope) { 6843ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar for (auto Clause : OMPExecutableDirective::linear_filter(D.clauses())) { 6853ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar for (auto *E : Clause->varlists()) { 6863ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 6873ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool IsRegistered = PrivateScope.addPrivate(VD, [&]()->llvm::Value * { 6883ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit var without initialization. 6893ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto VarEmission = CGF.EmitAutoVarAlloca(*VD); 6903ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.EmitAutoVarCleanups(VarEmission); 6913ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return VarEmission.getAllocatedAddress(); 6923ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar }); 6933ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar assert(IsRegistered && "linear var already registered as private"); 6943ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Silence the warning about unused variable. 6953ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar (void)IsRegistered; 6963ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 6973ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 6986bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 6996bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 7006bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesvoid CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { 70133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 70233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Pragma 'simd' code depends on presence of 'lastprivate'. 70333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // If present, we have to separate last iteration of the loop: 70433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // 70533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // if (LastIteration != 0) { 70633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // for (IV in 0..LastIteration-1) BODY; 70733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // BODY with updates of lastprivate vars; 70833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // <Final counter/linear vars updates>; 70933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // } 71033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // 71133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // otherwise (when there's no lastprivate): 71233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // 71333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // for (IV in 0..LastIteration) BODY; 71433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // <Final counter/linear vars updates>; 71533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // 71633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 71733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Walk clauses and process safelen/lastprivate. 71833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool SeparateIter = false; 71933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.LoopStack.setParallel(); 72033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.LoopStack.setVectorizerEnable(true); 72133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto C : S.clauses()) { 72233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar switch (C->getClauseKind()) { 72333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case OMPC_safelen: { 72433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RValue Len = CGF.EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(), 72533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar AggValueSlot::ignored(), true); 72633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); 72733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.LoopStack.setVectorizerWidth(Val->getZExtValue()); 72833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // In presence of finite 'safelen', it may be unsafe to mark all 72933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // the memory instructions parallel, because loop-carried 73033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // dependences of 'safelen' iterations are possible. 73133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.LoopStack.setParallel(false); 73233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 73333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 73433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case OMPC_aligned: 73533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPAlignedClause(CGF, CGF.CGM, cast<OMPAlignedClause>(*C)); 73633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 73733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case OMPC_lastprivate: 73833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SeparateIter = true; 73933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 74033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar default: 74133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Not handled yet 74233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ; 74333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 7446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines } 745176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 74633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit inits for the linear variables. 74733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { 74833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto Init : C->inits()) { 74933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *D = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); 75033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitVarDecl(*D); 75133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 7523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 753176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 75433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit the loop iteration variable. 75533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const Expr *IVExpr = S.getIterationVariable(); 75633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); 75733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitVarDecl(*IVDecl); 75833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitIgnoredExpr(S.getInit()); 75933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 76033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit the iterations count variable. 76133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // If it is not a variable, Sema decided to calculate iterations count on 76233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // each 76333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // iteration (e.g., it is foldable into a constant). 76433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { 76533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); 76633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit calculation of the iterations count. 76733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitIgnoredExpr(S.getCalcLastIteration()); 76833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 769176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 77033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit the linear steps for the linear clauses. 77133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // If a step is not constant, it is pre-calculated before the loop. 77233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) { 77333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep())) 77433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) { 77533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl())); 77633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit calculation of the linear step. 77733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitIgnoredExpr(CS); 77833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 77933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 780176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 78133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (SeparateIter) { 78233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit: if (LastIteration > 0) - begin. 78333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RegionCounter Cnt = CGF.getPGORegionCounter(&S); 78433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ThenBlock = CGF.createBasicBlock("simd.if.then"); 78533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto ContBlock = CGF.createBasicBlock("simd.if.end"); 78633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, 78733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Cnt.getCount()); 78833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitBlock(ThenBlock); 78933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Cnt.beginRegion(CGF.Builder); 79033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit 'then' code. 79133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar { 79233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPPrivateScope LoopScope(CGF); 79333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitPrivateLoopCounters(CGF, LoopScope, S.counters()); 79433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitPrivateLinearVars(CGF, S, LoopScope); 79533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPPrivateClause(S, LoopScope); 79633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)LoopScope.Privatize(); 79733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), 79833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar S.getCond(/*SeparateIter=*/true), S.getInc(), 79933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar [&S](CodeGenFunction &CGF) { 80033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPLoopBody(S); 80133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStopPoint(&S); 80233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 80333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPLoopBody(S, /* SeparateIter */ true); 8043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 80533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPSimdFinal(S); 80633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit: if (LastIteration != 0) - end. 80733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitBranch(ContBlock); 80833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitBlock(ContBlock, true); 80933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } else { 81033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar { 81133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPPrivateScope LoopScope(CGF); 81233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitPrivateLoopCounters(CGF, LoopScope, S.counters()); 81333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitPrivateLinearVars(CGF, S, LoopScope); 81433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPPrivateClause(S, LoopScope); 81533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)LoopScope.Privatize(); 81633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), 81733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar S.getCond(/*SeparateIter=*/false), S.getInc(), 81833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar [&S](CodeGenFunction &CGF) { 81933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPLoopBody(S); 82033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStopPoint(&S); 82133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }); 82233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 82333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPSimdFinal(S); 824176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines } 82533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 82633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); 8276bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines} 8286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines 8290e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, 8300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const OMPLoopDirective &S, 8310e2c34f92f00628d48968dfea096d36381f494cbStephen Hines OMPPrivateScope &LoopScope, 8320e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *LB, llvm::Value *UB, 8330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *ST, llvm::Value *IL, 8340e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *Chunk) { 8350e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto &RT = CGM.getOpenMPRuntime(); 8363ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 8373ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). 8383ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar const bool Dynamic = RT.isDynamic(ScheduleKind); 8393ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 8400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(!RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr) && 8410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines "static non-chunked schedule does not need outer loop"); 8420e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 8430e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit outer loop. 8440e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // 8450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // OpenMP [2.7.1, Loop Construct, Description, table 2-1] 8463ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // When schedule(dynamic,chunk_size) is specified, the iterations are 8473ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // distributed to threads in the team in chunks as the threads request them. 8483ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Each thread executes a chunk of iterations, then requests another chunk, 8493ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // until no chunks remain to be distributed. Each chunk contains chunk_size 8503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // iterations, except for the last chunk to be distributed, which may have 8513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // fewer iterations. When no chunk_size is specified, it defaults to 1. 8523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // 8533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // When schedule(guided,chunk_size) is specified, the iterations are assigned 8543ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // to threads in the team in chunks as the executing threads request them. 8553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Each thread executes a chunk of iterations, then requests another chunk, 8563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // until no chunks remain to be assigned. For a chunk_size of 1, the size of 8573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // each chunk is proportional to the number of unassigned iterations divided 8583ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // by the number of threads in the team, decreasing to 1. For a chunk_size 8593ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // with value k (greater than 1), the size of each chunk is determined in the 8603ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // same way, with the restriction that the chunks do not contain fewer than k 8613ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // iterations (except for the last chunk to be assigned, which may have fewer 8623ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // than k iterations). 8633ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // 8643ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // When schedule(auto) is specified, the decision regarding scheduling is 8653ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // delegated to the compiler and/or runtime system. The programmer gives the 8663ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // implementation the freedom to choose any possible mapping of iterations to 8673ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // threads in the team. 8683ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // 8693ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // When schedule(runtime) is specified, the decision regarding scheduling is 8703ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // deferred until run time, and the schedule and chunk size are taken from the 8713ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // run-sched-var ICV. If the ICV is set to auto, the schedule is 8723ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // implementation defined 8733ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // 8743ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // while(__kmpc_dispatch_next(&LB, &UB)) { 8753ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // idx = LB; 8763ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // while (idx <= UB) { BODY; ++idx; } // inner loop 8773ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // } 8783ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // 8793ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // OpenMP [2.7.1, Loop Construct, Description, table 2-1] 8800e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // When schedule(static, chunk_size) is specified, iterations are divided into 8810e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // chunks of size chunk_size, and the chunks are assigned to the threads in 8820e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // the team in a round-robin fashion in the order of the thread number. 8830e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // 8840e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { 8850e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // while (idx <= UB) { BODY; ++idx; } // inner loop 8860e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // LB = LB + ST; 8870e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // UB = UB + ST; 8880e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // } 8890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // 8903ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 8910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const Expr *IVExpr = S.getIterationVariable(); 8920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); 8930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); 8940e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 8953ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar RT.emitForInit( 8963ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar *this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, IL, LB, 8973ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar (Dynamic ? EmitAnyExpr(S.getLastIteration()).getScalarVal() : UB), ST, 8983ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Chunk); 8993ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 9000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); 9010e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9020e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Start the loop with a block that tests the condition. 9030e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto CondBlock = createBasicBlock("omp.dispatch.cond"); 9040e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBlock(CondBlock); 9050e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LoopStack.push(CondBlock); 9060e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9070e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *BoolCondVal = nullptr; 9083ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (!Dynamic) { 9093ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // UB = min(UB, GlobalUB) 9103ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(S.getEnsureUpperBound()); 9113ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // IV = LB 9123ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(S.getInit()); 9133ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // IV < UB 9143ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar BoolCondVal = EvaluateExprAsBool(S.getCond(false)); 9153ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } else { 9163ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, 9173ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar IL, LB, UB, ST); 9183ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 9190e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9200e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // If there are any cleanups between here and the loop-exit scope, 9210e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // create a block to stage a loop exit along. 9220e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto ExitBlock = LoopExit.getBlock(); 9230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (LoopScope.requiresCleanups()) 9240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ExitBlock = createBasicBlock("omp.dispatch.cleanup"); 9250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto LoopBody = createBasicBlock("omp.dispatch.body"); 9270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); 9280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (ExitBlock != LoopExit.getBlock()) { 9290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBlock(ExitBlock); 9300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBranchThroughCleanup(LoopExit); 9310e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 9320e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBlock(LoopBody); 9330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9343ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit "IV = LB" (in case of static schedule, we have already calculated new 9353ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // LB for loop condition and emitted it above). 9363ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (Dynamic) 9373ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(S.getInit()); 9383ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 9390e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Create a block for the increment. 9400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); 9410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); 9420e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9433ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), 94433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar S.getCond(/*SeparateIter=*/false), S.getInc(), 94533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar [&S](CodeGenFunction &CGF) { 94633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPLoopBody(S); 94733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStopPoint(&S); 9483ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar }); 9490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9500e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBlock(Continue.getBlock()); 9510e2c34f92f00628d48968dfea096d36381f494cbStephen Hines BreakContinueStack.pop_back(); 9523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (!Dynamic) { 9533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit "LB = LB + Stride", "UB = UB + Stride". 9543ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(S.getNextLowerBound()); 9553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitIgnoredExpr(S.getNextUpperBound()); 9563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 9570e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9580e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBranch(CondBlock); 9590e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LoopStack.pop(); 9600e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit the fall-through block. 9610e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBlock(LoopExit.getBlock()); 9620e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9630e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Tell the runtime we are done. 9643ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // FIXME: Also call fini for ordered loops with dynamic scheduling. 9653ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (!Dynamic) 9663ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar RT.emitForFinish(*this, S.getLocStart(), ScheduleKind); 9670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 9680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines/// \brief Emit a helper variable and return corresponding lvalue. 9700e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesstatic LValue EmitOMPHelperVar(CodeGenFunction &CGF, 9710e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const DeclRefExpr *Helper) { 9720e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto VDecl = cast<VarDecl>(Helper->getDecl()); 9730e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.EmitVarDecl(*VDecl); 9740e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return CGF.EmitLValue(Helper); 9750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 9760e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 97733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarbool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { 9780e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit the loop iteration variable. 9790e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable()); 9800e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto IVDecl = cast<VarDecl>(IVExpr->getDecl()); 9810e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitVarDecl(*IVDecl); 9820e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9830e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit the iterations count variable. 9840e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // If it is not a variable, Sema decided to calculate iterations count on each 9850e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // iteration (e.g., it is foldable into a constant). 9860e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { 9870e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); 9880e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit calculation of the iterations count. 9890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitIgnoredExpr(S.getCalcLastIteration()); 9900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 9910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 9920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto &RT = CGM.getOpenMPRuntime(); 9930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 99433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool HasLastprivateClause; 9950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Check pre-condition. 9960e2c34f92f00628d48968dfea096d36381f494cbStephen Hines { 9970e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Skip the entire loop if we don't meet the precondition. 9980e2c34f92f00628d48968dfea096d36381f494cbStephen Hines RegionCounter Cnt = getPGORegionCounter(&S); 9990e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto ThenBlock = createBasicBlock("omp.precond.then"); 10000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto ContBlock = createBasicBlock("omp.precond.end"); 10010e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); 10020e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBlock(ThenBlock); 10030e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Cnt.beginRegion(Builder); 10040e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit 'then' code. 10050e2c34f92f00628d48968dfea096d36381f494cbStephen Hines { 10060e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit helper vars inits. 10070e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue LB = 10080e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getLowerBoundVariable())); 10090e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue UB = 10100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getUpperBoundVariable())); 10110e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue ST = 10120e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable())); 10130e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue IL = 10140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); 10150e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 10160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines OMPPrivateScope LoopScope(*this); 101733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (EmitOMPFirstprivateClause(S, LoopScope)) { 101833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit implicit barrier to synchronize threads and avoid data races on 101933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // initialization of firstprivate variables. 102033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), 102133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPD_unknown); 102233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 102333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); 10240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitPrivateLoopCounters(*this, LoopScope, S.counters()); 10253ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar (void)LoopScope.Privatize(); 10260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 10270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Detect the loop schedule kind and chunk. 10280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto ScheduleKind = OMPC_SCHEDULE_unknown; 10290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *Chunk = nullptr; 10300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (auto C = cast_or_null<OMPScheduleClause>( 10310e2c34f92f00628d48968dfea096d36381f494cbStephen Hines S.getSingleClause(OMPC_schedule))) { 10320e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ScheduleKind = C->getScheduleKind(); 10330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (auto Ch = C->getChunkSize()) { 10340e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Chunk = EmitScalarExpr(Ch); 10350e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Chunk = EmitScalarConversion(Chunk, Ch->getType(), 10360e2c34f92f00628d48968dfea096d36381f494cbStephen Hines S.getIterationVariable()->getType()); 10370e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 10380e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 10390e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); 10400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); 10410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (RT.isStaticNonchunked(ScheduleKind, 10420e2c34f92f00628d48968dfea096d36381f494cbStephen Hines /* Chunked */ Chunk != nullptr)) { 10430e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // OpenMP [2.7.1, Loop Construct, Description, table 2-1] 10440e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // When no chunk_size is specified, the iteration space is divided into 10450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // chunks that are approximately equal in size, and at most one chunk is 10460e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // distributed to each thread. Note that the size of the chunks is 10470e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // unspecified in this case. 10480e2c34f92f00628d48968dfea096d36381f494cbStephen Hines RT.emitForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, 10490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines IL.getAddress(), LB.getAddress(), UB.getAddress(), 10500e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ST.getAddress()); 10510e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // UB = min(UB, GlobalUB); 10520e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitIgnoredExpr(S.getEnsureUpperBound()); 10530e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // IV = LB; 10540e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitIgnoredExpr(S.getInit()); 10550e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // while (idx <= UB) { BODY; ++idx; } 10563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), 10573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar S.getCond(/*SeparateIter=*/false), S.getInc(), 105833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar [&S](CodeGenFunction &CGF) { 105933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPLoopBody(S); 106033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStopPoint(&S); 10613ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar }); 10620e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Tell the runtime we are done. 10630e2c34f92f00628d48968dfea096d36381f494cbStephen Hines RT.emitForFinish(*this, S.getLocStart(), ScheduleKind); 10640e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else { 10650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit the outer loop, which requests its work chunk [LB..UB] from 10660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // runtime and runs the inner loop to process it. 10670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitOMPForOuterLoop(ScheduleKind, S, LoopScope, LB.getAddress(), 10680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines UB.getAddress(), ST.getAddress(), IL.getAddress(), 10690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Chunk); 10700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 107133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit final copy of the lastprivate variables if IsLastIter != 0. 107233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (HasLastprivateClause) 107333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPLastprivateClauseFinal( 107433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar S, Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); 10750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 10760e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // We're now done with the loop, so jump to the continuation block. 10770e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBranch(ContBlock); 10780e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitBlock(ContBlock, true); 10790e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 108033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return HasLastprivateClause; 10810e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 10820e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 10830e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { 108433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 108533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool HasLastprivates = false; 108633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { 108733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar HasLastprivates = CGF.EmitOMPWorksharingLoop(S); 108833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 108933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); 10900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 10910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Emit an implicit barrier at the end. 109233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) { 109333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); 109433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 1095c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines} 1096c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1097176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) { 1098176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm_unreachable("CodeGen for 'omp for simd' is not supported yet."); 1099176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1100176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 11013ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarstatic LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, 11023ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar const Twine &Name, 11033ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Value *Init = nullptr) { 11043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto LVal = CGF.MakeNaturalAlignAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); 11053ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (Init) 11063ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.EmitScalarInit(Init, LVal); 11073ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return LVal; 1108c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines} 1109c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 111033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarstatic OpenMPDirectiveKind emitSections(CodeGenFunction &CGF, 111133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPExecutableDirective &S) { 11123ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt(); 11133ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *CS = dyn_cast<CompoundStmt>(Stmt); 11143ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (CS && CS->size() > 1) { 111533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S, CS](CodeGenFunction &CGF) { 111633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &C = CGF.CGM.getContext(); 111733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); 111833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit helper vars inits. 111933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", 112033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.Builder.getInt32(0)); 112133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *GlobalUBVal = CGF.Builder.getInt32(CS->size() - 1); 112233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LValue UB = 112333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); 112433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", 112533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.Builder.getInt32(1)); 112633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", 112733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.Builder.getInt32(0)); 112833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Loop counter. 112933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); 113033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); 113133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); 113233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); 113333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); 113433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Generate condition for loop. 113533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, 113633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OK_Ordinary, S.getLocStart(), 113733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar /*fpContractable=*/false); 113833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Increment for loop counter. 113933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, 114033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OK_Ordinary, S.getLocStart()); 114133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto BodyGen = [CS, &S, &IV](CodeGenFunction &CGF) { 114233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Iterate through all sections and emit a switch construct: 114333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // switch (IV) { 114433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // case 0: 114533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // <SectionStmt[0]>; 114633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // break; 114733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // ... 114833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // case <NumSection> - 1: 114933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // <SectionStmt[<NumSection> - 1]>; 115033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // break; 115133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // } 115233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // .omp.sections.exit: 115333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); 115433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *SwitchStmt = CGF.Builder.CreateSwitch( 115533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB, 115633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CS->size()); 115733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar unsigned CaseNumber = 0; 115833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar for (auto C = CS->children(); C; ++C, ++CaseNumber) { 115933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); 116033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitBlock(CaseBB); 116133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); 116233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(*C); 116333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitBranch(ExitBB); 116433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 116533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 116633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 116733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit static non-chunked loop. 116833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitForInit( 116933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32, 117033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar /*IVSigned=*/true, IL.getAddress(), LB.getAddress(), UB.getAddress(), 117133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ST.getAddress()); 117233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // UB = min(UB, GlobalUB); 117333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart()); 117433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *MinUBGlobalUB = CGF.Builder.CreateSelect( 117533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); 117633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); 117733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // IV = LB; 117833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV); 117933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // while (idx <= UB) { BODY; ++idx; } 118033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen); 118133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Tell the runtime we are done. 118233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitForFinish(CGF, S.getLocStart(), 118333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPC_SCHEDULE_static); 11843ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar }; 118533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 118633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, CodeGen); 118733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return OMPD_sections; 11883ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 118933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // If only one section is found - no need to generate loop, emit as a single 119033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // region. 119133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [Stmt](CodeGenFunction &CGF) { 119233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(Stmt); 119333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EnsureInsertPoint(); 119433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 119533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitSingleRegion(CGF, CodeGen, S.getLocStart(), 119633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::None, llvm::None, 119733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::None, llvm::None); 119833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return OMPD_single; 119933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 12003ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 120133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { 120233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 120333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OpenMPDirectiveKind EmittedAs = emitSections(*this, S); 12043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit an implicit barrier at the end. 120533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (!S.getSingleClause(OMPC_nowait)) { 120633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), EmittedAs); 120733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 12083ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 12093ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 12103ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { 121133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 121233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 121333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); 121433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EnsureInsertPoint(); 121533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 121633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); 1217c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines} 1218c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 12190e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { 12203ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::SmallVector<const Expr *, 8> CopyprivateVars; 122133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::SmallVector<const Expr *, 8> DestExprs; 12223ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::SmallVector<const Expr *, 8> SrcExprs; 12233ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::SmallVector<const Expr *, 8> AssignmentOps; 122433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Check if there are any 'copyprivate' clauses associated with this 122533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // 'single' 12263ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // construct. 12273ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto CopyprivateFilter = [](const OMPClause *C) -> bool { 12283ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return C->getClauseKind() == OMPC_copyprivate; 12293ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar }; 12303ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Build a list of copyprivate variables along with helper expressions 12313ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // (<source>, <destination>, <destination>=<source> expressions) 12323ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar typedef OMPExecutableDirective::filtered_clause_iterator<decltype( 12333ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CopyprivateFilter)> CopyprivateIter; 12343ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar for (CopyprivateIter I(S.clauses(), CopyprivateFilter); I; ++I) { 12353ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *C = cast<OMPCopyprivateClause>(*I); 12363ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CopyprivateVars.append(C->varlists().begin(), C->varlists().end()); 123733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar DestExprs.append(C->destination_exprs().begin(), 123833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar C->destination_exprs().end()); 12393ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end()); 12403ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AssignmentOps.append(C->assignment_ops().begin(), 12413ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar C->assignment_ops().end()); 12423ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 124333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 12443ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit code for 'single' region along with 'copyprivate' clauses 124533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 124633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); 124733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EnsureInsertPoint(); 124833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 124933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), 125033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CopyprivateVars, DestExprs, SrcExprs, 125133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar AssignmentOps); 12523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit an implicit barrier at the end. 125333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (!S.getSingleClause(OMPC_nowait)) { 125433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_single); 125533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 1256176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1257176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 12580e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { 125933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 126033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 126133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); 126233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EnsureInsertPoint(); 126333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 126433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart()); 12650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 12660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 12670e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { 126833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 126933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 127033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); 127133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EnsureInsertPoint(); 127233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 12730e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGM.getOpenMPRuntime().emitCriticalRegion( 127433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar *this, S.getDirectiveName().getAsString(), CodeGen, S.getLocStart()); 1275176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1276176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 127733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPParallelForDirective( 127833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPParallelForDirective &S) { 127933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit directive as a combined directive that consists of two implicit 128033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // directives: 'parallel' with 'for' directive. 128133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 128233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 128333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPWorksharingLoop(S); 128433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit implicit barrier at the end of parallel region, but this barrier 128533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // is at the end of 'for' directive, so emit it as the implicit barrier for 128633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // this 'for' directive. 128733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), 128833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPD_parallel); 128933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 129033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar emitCommonOMPParallelDirective(*this, S, CodeGen); 1291c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines} 1292c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 1293176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPParallelForSimdDirective( 1294176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines const OMPParallelForSimdDirective &) { 1295176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm_unreachable("CodeGen for 'omp parallel for simd' is not supported yet."); 1296176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1297176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1298c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hinesvoid CodeGenFunction::EmitOMPParallelSectionsDirective( 129933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const OMPParallelSectionsDirective &S) { 130033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit directive as a combined directive that consists of two implicit 130133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // directives: 'parallel' with 'sections' directive. 130233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 130333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S](CodeGenFunction &CGF) { 130433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (void)emitSections(CGF, S); 130533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit implicit barrier at the end of parallel region. 130633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), 130733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar OMPD_parallel); 130833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 130933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar emitCommonOMPParallelDirective(*this, S, CodeGen); 1310c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines} 1311c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 13123ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { 13133ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Emit outlined function for task construct. 131433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 13153ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); 13163ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto CapturedStruct = GenerateCapturedStmtArgument(*CS); 13173ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *I = CS->getCapturedDecl()->param_begin(); 131833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *PartId = std::next(I); 13193ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // The first function argument for tasks is a thread id, the second one is a 13203ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // part id (0 for tied tasks, >=0 for untied task). 132133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [PartId, &S](CodeGenFunction &CGF) { 132233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (*PartId) { 132333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // TODO: emit code for untied tasks. 132433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 132533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); 132633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 13273ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto OutlinedFn = 132833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitTaskOutlinedFunction(S, *I, CodeGen); 13293ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Check if we should emit tied or untied task. 13303ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool Tied = !S.getSingleClause(OMPC_untied); 13313ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Check if the task is final 13323ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::PointerIntPair<llvm::Value *, 1, bool> Final; 13333ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (auto *Clause = S.getSingleClause(OMPC_final)) { 13343ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // If the condition constant folds and can be elided, try to avoid emitting 13353ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // the condition and the dead arm of the if/else. 13363ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *Cond = cast<OMPFinalClause>(Clause)->getCondition(); 13373ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool CondConstant; 13383ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) 13393ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Final.setInt(CondConstant); 13403ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar else 13413ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Final.setPointer(EvaluateExprAsBool(Cond)); 13423ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } else { 13433ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // By default the task is not final. 13443ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Final.setInt(/*IntVal=*/false); 13453ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 13463ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); 13473ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), Tied, Final, 13483ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar OutlinedFn, SharedsTy, CapturedStruct); 1349176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1350176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 13510e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPTaskyieldDirective( 13520e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const OMPTaskyieldDirective &S) { 13530e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart()); 1354176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1355176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 13560e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { 135733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier); 1358176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1359176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1360176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &) { 1361176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm_unreachable("CodeGen for 'omp taskwait' is not supported yet."); 1362176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1363176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1364176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { 13650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> { 13660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { 13670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto FlushClause = cast<OMPFlushClause>(C); 13680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return llvm::makeArrayRef(FlushClause->varlist_begin(), 13690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines FlushClause->varlist_end()); 13700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 13710e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return llvm::None; 13720e2c34f92f00628d48968dfea096d36381f494cbStephen Hines }(), S.getLocStart()); 1373176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1374176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1375176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &) { 1376176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm_unreachable("CodeGen for 'omp ordered' is not supported yet."); 1377176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1378176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 13790e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesstatic llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, 13800e2c34f92f00628d48968dfea096d36381f494cbStephen Hines QualType SrcType, QualType DestType) { 13810e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(CGF.hasScalarEvaluationKind(DestType) && 13820e2c34f92f00628d48968dfea096d36381f494cbStephen Hines "DestType must have scalar evaluation kind."); 13830e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(!Val.isAggregate() && "Must be a scalar or complex."); 13840e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return Val.isScalar() 13850e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType) 13860e2c34f92f00628d48968dfea096d36381f494cbStephen Hines : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType, 13870e2c34f92f00628d48968dfea096d36381f494cbStephen Hines DestType); 13880e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 13890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 13900e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesstatic CodeGenFunction::ComplexPairTy 13910e2c34f92f00628d48968dfea096d36381f494cbStephen HinesconvertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, 13920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines QualType DestType) { 13930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(CGF.getEvaluationKind(DestType) == TEK_Complex && 13940e2c34f92f00628d48968dfea096d36381f494cbStephen Hines "DestType must have complex evaluation kind."); 13950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CodeGenFunction::ComplexPairTy ComplexVal; 13960e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (Val.isScalar()) { 13970e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Convert the input element to the element type of the complex. 13980e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); 13990e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto ScalarVal = 14000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType); 14010e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ComplexVal = CodeGenFunction::ComplexPairTy( 14020e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType())); 14030e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else { 14040e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(Val.isComplex() && "Must be a scalar or complex."); 14050e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType(); 14060e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); 14070e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ComplexVal.first = CGF.EmitScalarConversion( 14080e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Val.getComplexVal().first, SrcElementType, DestElementType); 14090e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ComplexVal.second = CGF.EmitScalarConversion( 14100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Val.getComplexVal().second, SrcElementType, DestElementType); 14110e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 14120e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return ComplexVal; 14130e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 14140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 14150e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesstatic void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, 14160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const Expr *X, const Expr *V, 14170e2c34f92f00628d48968dfea096d36381f494cbStephen Hines SourceLocation Loc) { 14180e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // v = x; 14190e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(V->isLValue() && "V of 'omp atomic read' is not lvalue"); 14200e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(X->isLValue() && "X of 'omp atomic read' is not lvalue"); 14210e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue XLValue = CGF.EmitLValue(X); 14220e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue VLValue = CGF.EmitLValue(V); 14230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines RValue Res = XLValue.isGlobalReg() 14240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ? CGF.EmitLoadOfLValue(XLValue, Loc) 14250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines : CGF.EmitAtomicLoad(XLValue, Loc, 14260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines IsSeqCst ? llvm::SequentiallyConsistent 14273ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar : llvm::Monotonic, 14283ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar XLValue.isVolatile()); 14290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // OpenMP, 2.12.6, atomic Construct 14300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Any atomic construct with a seq_cst clause forces the atomically 14310e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // performed operation to include an implicit flush operation without a 14320e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // list. 14330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (IsSeqCst) 14340e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); 14350e2c34f92f00628d48968dfea096d36381f494cbStephen Hines switch (CGF.getEvaluationKind(V->getType())) { 14360e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case TEK_Scalar: 14370e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.EmitStoreOfScalar( 14380e2c34f92f00628d48968dfea096d36381f494cbStephen Hines convertToScalarValue(CGF, Res, X->getType(), V->getType()), VLValue); 14390e2c34f92f00628d48968dfea096d36381f494cbStephen Hines break; 14400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case TEK_Complex: 14410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.EmitStoreOfComplex( 14420e2c34f92f00628d48968dfea096d36381f494cbStephen Hines convertToComplexValue(CGF, Res, X->getType(), V->getType()), VLValue, 14430e2c34f92f00628d48968dfea096d36381f494cbStephen Hines /*isInit=*/false); 14440e2c34f92f00628d48968dfea096d36381f494cbStephen Hines break; 14450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case TEK_Aggregate: 14460e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm_unreachable("Must be a scalar or complex."); 14470e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 14480e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 14490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 14503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarstatic void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, 14513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar const Expr *X, const Expr *E, 14523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar SourceLocation Loc) { 14533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // x = expr; 14543ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar assert(X->isLValue() && "X of 'omp atomic write' is not lvalue"); 14553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar LValue XLValue = CGF.EmitLValue(X); 14563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar RValue ExprRValue = CGF.EmitAnyExpr(E); 14573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (XLValue.isGlobalReg()) 14583ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.EmitStoreThroughGlobalRegLValue(ExprRValue, XLValue); 14593ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar else 14603ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.EmitAtomicStore(ExprRValue, XLValue, 14613ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar IsSeqCst ? llvm::SequentiallyConsistent 14623ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar : llvm::Monotonic, 14633ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar XLValue.isVolatile(), /*IsInit=*/false); 14643ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // OpenMP, 2.12.6, atomic Construct 14653ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Any atomic construct with a seq_cst clause forces the atomically 14663ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // performed operation to include an implicit flush operation without a 14673ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // list. 14683ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (IsSeqCst) 14693ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); 14703ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 14713ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 147233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarbool emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, 147333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar BinaryOperatorKind BO, llvm::AtomicOrdering AO, 147433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool IsXLHSInRHSPart) { 147533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &Context = CGF.CGM.getContext(); 147633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x' 147733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // expression is simple and atomic is allowed for the given type for the 147833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // target platform. 147933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (BO == BO_Comma || !Update.isScalar() || 148033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() || 148133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (!isa<llvm::ConstantInt>(Update.getScalarVal()) && 148233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar (Update.getScalarVal()->getType() != 148333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar X.getAddress()->getType()->getPointerElementType())) || 148433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar !Context.getTargetInfo().hasBuiltinAtomic( 148533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) 148633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return false; 148733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 148833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::AtomicRMWInst::BinOp RMWOp; 148933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar switch (BO) { 149033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Add: 149133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RMWOp = llvm::AtomicRMWInst::Add; 149233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 149333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Sub: 149433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (!IsXLHSInRHSPart) 149533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return false; 149633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RMWOp = llvm::AtomicRMWInst::Sub; 149733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 149833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_And: 149933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RMWOp = llvm::AtomicRMWInst::And; 150033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 150133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Or: 150233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RMWOp = llvm::AtomicRMWInst::Or; 150333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 150433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Xor: 150533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RMWOp = llvm::AtomicRMWInst::Xor; 150633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 150733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_LT: 150833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RMWOp = X.getType()->hasSignedIntegerRepresentation() 150933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min 151033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar : llvm::AtomicRMWInst::Max) 151133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin 151233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar : llvm::AtomicRMWInst::UMax); 151333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 151433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_GT: 151533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RMWOp = X.getType()->hasSignedIntegerRepresentation() 151633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max 151733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar : llvm::AtomicRMWInst::Min) 151833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax 151933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar : llvm::AtomicRMWInst::UMin); 152033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 152133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Mul: 152233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Div: 152333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Rem: 152433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Shl: 152533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Shr: 152633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_LAnd: 152733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_LOr: 152833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return false; 152933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_PtrMemD: 153033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_PtrMemI: 153133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_LE: 153233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_GE: 153333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_EQ: 153433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_NE: 153533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Assign: 153633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_AddAssign: 153733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_SubAssign: 153833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_AndAssign: 153933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_OrAssign: 154033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_XorAssign: 154133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_MulAssign: 154233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_DivAssign: 154333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_RemAssign: 154433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_ShlAssign: 154533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_ShrAssign: 154633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case BO_Comma: 154733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm_unreachable("Unsupported atomic update operation"); 154833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 154933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *UpdateVal = Update.getScalarVal(); 155033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) { 155133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar UpdateVal = CGF.Builder.CreateIntCast( 155233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar IC, X.getAddress()->getType()->getPointerElementType(), 155333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar X.getType()->hasSignedIntegerRepresentation()); 155433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 155533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO); 155633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return true; 155733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 155833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 155933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarvoid CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr( 156033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, 156133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar llvm::AtomicOrdering AO, SourceLocation Loc, 156233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const llvm::function_ref<RValue(RValue)> &CommonGen) { 156333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Update expressions are allowed to have the following forms: 156433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x binop= expr; -> xrval + expr; 156533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x++, ++x -> xrval + 1; 156633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x--, --x -> xrval - 1; 156733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x = x binop expr; -> xrval binop expr 156833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x = expr Op x; - > expr binop xrval; 156933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (!emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart)) { 157033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (X.isGlobalReg()) { 157133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop 157233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // 'xrval'. 157333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X); 157433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } else { 157533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Perform compare-and-swap procedure. 157633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified()); 157733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 157833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar } 157933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 158033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 158133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainarstatic void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, 158233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const Expr *X, const Expr *E, 158333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const Expr *UE, bool IsXLHSInRHSPart, 158433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar SourceLocation Loc) { 158533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) && 158633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar "Update expr in 'atomic update' must be a binary operator."); 158733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); 158833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Update expressions are allowed to have the following forms: 158933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x binop= expr; -> xrval + expr; 159033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x++, ++x -> xrval + 1; 159133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x--, --x -> xrval - 1; 159233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x = x binop expr; -> xrval binop expr 159333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // x = expr Op x; - > expr binop xrval; 159433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar assert(X->isLValue() && "X of 'omp atomic update' is not lvalue"); 159533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LValue XLValue = CGF.EmitLValue(X); 159633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar RValue ExprRValue = CGF.EmitAnyExpr(E); 159733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic; 159833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); 159933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); 160033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; 160133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; 160233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto Gen = 160333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue { 160433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); 160533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); 160633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar return CGF.EmitAnyExpr(UE); 160733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 160833337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.EmitOMPAtomicSimpleUpdateExpr(XLValue, ExprRValue, BOUE->getOpcode(), 160933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar IsXLHSInRHSPart, AO, Loc, Gen); 161033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // OpenMP, 2.12.6, atomic Construct 161133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // Any atomic construct with a seq_cst clause forces the atomically 161233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // performed operation to include an implicit flush operation without a 161333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar // list. 161433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar if (IsSeqCst) 161533337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); 161633337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar} 161733337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar 16180e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesstatic void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, 16190e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool IsSeqCst, const Expr *X, const Expr *V, 162033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar const Expr *E, const Expr *UE, 162133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar bool IsXLHSInRHSPart, SourceLocation Loc) { 16220e2c34f92f00628d48968dfea096d36381f494cbStephen Hines switch (Kind) { 16230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_read: 16240e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); 16250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines break; 16260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_write: 16273ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); 16283ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar break; 162933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar case OMPC_unknown: 16300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_update: 163133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); 163233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar break; 16330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_capture: 16340e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm_unreachable("CodeGen for 'omp atomic clause' is not supported yet."); 16350e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_if: 16360e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_final: 16370e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_num_threads: 16380e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_private: 16390e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_firstprivate: 16400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_lastprivate: 16410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_reduction: 16420e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_safelen: 16430e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_collapse: 16440e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_default: 16450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_seq_cst: 16460e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_shared: 16470e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_linear: 16480e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_aligned: 16490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_copyin: 16500e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_copyprivate: 16510e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_flush: 16520e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_proc_bind: 16530e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_schedule: 16540e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_ordered: 16550e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_nowait: 16560e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_untied: 16570e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_threadprivate: 16580e2c34f92f00628d48968dfea096d36381f494cbStephen Hines case OMPC_mergeable: 16590e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm_unreachable("Clause is not allowed in 'omp atomic'."); 16600e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 16610e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 16620e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 16630e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { 16640e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool IsSeqCst = S.getSingleClause(/*K=*/OMPC_seq_cst); 16650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines OpenMPClauseKind Kind = OMPC_unknown; 16660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines for (auto *C : S.clauses()) { 16670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Find first clause (skip seq_cst clause, if it is first). 16680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (C->getClauseKind() != OMPC_seq_cst) { 16690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Kind = C->getClauseKind(); 16700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines break; 16710e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 16720e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 16733ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 16743ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar const auto *CS = 16753ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); 16763ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) 16773ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar enterFullExpression(EWC); 16783ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 167933337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar LexicalScope Scope(*this, S.getSourceRange()); 168033337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar auto &&CodeGen = [&S, Kind, IsSeqCst](CodeGenFunction &CGF) { 168133337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.getX(), S.getV(), S.getExpr(), 168233337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar S.getUpdateExpr(), S.isXLHSInRHSPart(), S.getLocStart()); 168333337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar }; 168433337ca4d89605025818daf83390ab4271d598d9Pirama Arumuga Nainar CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); 1685176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1686176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1687176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) { 1688176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm_unreachable("CodeGen for 'omp target' is not supported yet."); 1689176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1690176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1691176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hinesvoid CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) { 1692176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm_unreachable("CodeGen for 'omp teams' is not supported yet."); 1693176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines} 1694176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines 1695