Lines Matching refs:llvm
19 #include "llvm/ADT/StringExtras.h"
20 #include "llvm/IR/DataLayout.h"
21 #include "llvm/IR/Intrinsics.h"
22 #include "llvm/IR/Operator.h"
89 llvm::Value *getAtomicSizeValue() const {
96 llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
99 RValue convertTempToRValue(llvm::Value *addr,
108 llvm::Value *addr = lvalue.getAddress();
117 llvm::Value *materializeRValue(RValue rvalue) const;
120 bool requiresMemSetZero(llvm::Type *type) const;
131 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
132 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
137 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
145 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
167 llvm::Value *addr = dest.getAddress();
171 CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
178 llvm::Value *Dest, llvm::Value *Ptr,
179 llvm::Value *Val1, llvm::Value *Val2,
181 llvm::AtomicOrdering SuccessOrder,
182 llvm::AtomicOrdering FailureOrder) {
184 llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
186 llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
189 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
196 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
197 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
201 llvm::BasicBlock *StoreExpectedBB =
206 llvm::BasicBlock *ContinueBB =
215 llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
230 bool IsWeak, llvm::Value *Dest,
231 llvm::Value *Ptr, llvm::Value *Val1,
232 llvm::Value *Val2,
233 llvm::Value *FailureOrderVal,
235 llvm::AtomicOrdering SuccessOrder) {
236 llvm::AtomicOrdering FailureOrder;
237 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
240 FailureOrder = llvm::Monotonic;
244 FailureOrder = llvm::Acquire;
247 FailureOrder = llvm::SequentiallyConsistent;
253 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
261 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
264 if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
266 if (SuccessOrder == llvm::SequentiallyConsistent)
269 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
271 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
280 Size, Align, SuccessOrder, llvm::Monotonic);
286 Size, Align, SuccessOrder, llvm::Acquire);
296 Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
305 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
306 llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
307 llvm::Value *IsWeak, llvm::Value *FailureOrder,
309 llvm::AtomicOrdering Order) {
310 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
311 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
327 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
332 llvm::BasicBlock *StrongBB =
334 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
335 llvm::BasicBlock *ContBB =
338 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
358 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
362 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
371 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
373 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
383 Op = llvm::AtomicRMWInst::Xchg;
387 PostOp = llvm::Instruction::Add;
391 Op = llvm::AtomicRMWInst::Add;
395 PostOp = llvm::Instruction::Sub;
399 Op = llvm::AtomicRMWInst::Sub;
403 PostOp = llvm::Instruction::And;
407 Op = llvm::AtomicRMWInst::And;
411 PostOp = llvm::Instruction::Or;
415 Op = llvm::AtomicRMWInst::Or;
419 PostOp = llvm::Instruction::Xor;
423 Op = llvm::AtomicRMWInst::Xor;
427 PostOp = llvm::Instruction::And;
430 Op = llvm::AtomicRMWInst::Nand;
434 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
436 llvm::AtomicRMWInst *RMWI =
442 llvm::Value *Result = RMWI;
447 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
453 static llvm::Value *
455 llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
463 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
477 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
491 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
493 llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
502 llvm::Value *Order = EmitScalarExpr(E->getOrder());
547 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
609 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
709 LibCallName += "_" + llvm::utostr(Size);
729 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
730 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
746 llvm::Type *IPtrTy =
747 llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
748 llvm::Value *OrigDest = Dest;
754 if (isa<llvm::ConstantInt>(Order)) {
755 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
759 Size, Align, llvm::Monotonic);
766 Size, Align, llvm::Acquire);
772 Size, Align, llvm::Release);
778 Size, Align, llvm::AcquireRelease);
782 Size, Align, llvm::SequentiallyConsistent);
797 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
808 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
815 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
820 Size, Align, llvm::Monotonic);
825 Size, Align, llvm::Acquire);
835 Size, Align, llvm::Release);
843 Size, Align, llvm::AcquireRelease);
850 Size, Align, llvm::SequentiallyConsistent);
862 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
864 cast<llvm::PointerType>(addr->getType())->getAddressSpace();
865 llvm::IntegerType *ty =
866 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
870 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
893 llvm::Value *tempAddr;
909 args.add(RValue::get(llvm::ConstantInt::get(
919 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
920 llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
921 load->setAtomic(llvm::SequentiallyConsistent);
932 llvm::Value *result = load;
941 llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType);
942 if (isa<llvm::IntegerType>(resultTy)) {
945 } else if (isa<llvm::PointerType>(resultTy)) {
955 llvm::Value *temp;
969 llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp);
1013 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
1020 llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp");
1049 llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
1059 args.add(RValue::get(llvm::ConstantInt::get(
1067 llvm::Value *intValue;
1072 llvm::Value *value = rvalue.getScalarVal();
1073 if (isa<llvm::IntegerType>(value->getType())) {
1076 llvm::IntegerType *inputIntTy =
1077 llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits());
1078 if (isa<llvm::PointerType>(value->getType())) {
1088 llvm::Value *addr = atomics.materializeRValue(rvalue);
1097 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
1098 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1101 if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
1116 llvm::Value *value = EmitScalarExpr(init);