1//===-- ThreadSanitizer.cpp - race detector -------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of ThreadSanitizer, a race detector. 11// 12// The tool is under development, for the details about previous versions see 13// http://code.google.com/p/data-race-test 14// 15// The instrumentation phase is quite simple: 16// - Insert calls to run-time library before every memory access. 17// - Optimizations may apply to avoid instrumenting some of the accesses. 18// - Insert calls at function entry/exit. 19// The rest is handled by the run-time library. 20//===----------------------------------------------------------------------===// 21 22#include "llvm/Transforms/Instrumentation.h" 23#include "llvm/ADT/SmallSet.h" 24#include "llvm/ADT/SmallString.h" 25#include "llvm/ADT/SmallVector.h" 26#include "llvm/ADT/Statistic.h" 27#include "llvm/ADT/StringExtras.h" 28#include "llvm/Analysis/CaptureTracking.h" 29#include "llvm/Analysis/ValueTracking.h" 30#include "llvm/IR/DataLayout.h" 31#include "llvm/IR/Function.h" 32#include "llvm/IR/IRBuilder.h" 33#include "llvm/IR/IntrinsicInst.h" 34#include "llvm/IR/Intrinsics.h" 35#include "llvm/IR/LLVMContext.h" 36#include "llvm/IR/Metadata.h" 37#include "llvm/IR/Module.h" 38#include "llvm/IR/Type.h" 39#include "llvm/Support/CommandLine.h" 40#include "llvm/Support/Debug.h" 41#include "llvm/Support/MathExtras.h" 42#include "llvm/Support/raw_ostream.h" 43#include "llvm/Transforms/Utils/BasicBlockUtils.h" 44#include "llvm/Transforms/Utils/ModuleUtils.h" 45 46using namespace llvm; 47 48#define DEBUG_TYPE "tsan" 49 50static cl::opt<bool> ClInstrumentMemoryAccesses( 51 "tsan-instrument-memory-accesses", cl::init(true), 52 cl::desc("Instrument memory accesses"), cl::Hidden); 53static cl::opt<bool> ClInstrumentFuncEntryExit( 54 "tsan-instrument-func-entry-exit", cl::init(true), 55 cl::desc("Instrument function entry and exit"), cl::Hidden); 56static cl::opt<bool> ClInstrumentAtomics( 57 "tsan-instrument-atomics", cl::init(true), 58 cl::desc("Instrument atomics"), cl::Hidden); 59static cl::opt<bool> ClInstrumentMemIntrinsics( 60 "tsan-instrument-memintrinsics", cl::init(true), 61 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 62 63STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 64STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 65STATISTIC(NumOmittedReadsBeforeWrite, 66 "Number of reads ignored due to following writes"); 67STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 68STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 69STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 70STATISTIC(NumOmittedReadsFromConstantGlobals, 71 "Number of reads from constant globals"); 72STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 73STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 74 75namespace { 76 77/// ThreadSanitizer: instrument the code in module to find races. 78struct ThreadSanitizer : public FunctionPass { 79 ThreadSanitizer() : FunctionPass(ID) {} 80 const char *getPassName() const override; 81 bool runOnFunction(Function &F) override; 82 bool doInitialization(Module &M) override; 83 static char ID; // Pass identification, replacement for typeid. 84 85 private: 86 void initializeCallbacks(Module &M); 87 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); 88 bool instrumentAtomic(Instruction *I, const DataLayout &DL); 89 bool instrumentMemIntrinsic(Instruction *I); 90 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 91 SmallVectorImpl<Instruction *> &All, 92 const DataLayout &DL); 93 bool addrPointsToConstantData(Value *Addr); 94 int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); 95 96 Type *IntptrTy; 97 IntegerType *OrdTy; 98 // Callbacks to run-time library are computed in doInitialization. 99 Function *TsanFuncEntry; 100 Function *TsanFuncExit; 101 // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 102 static const size_t kNumberOfAccessSizes = 5; 103 Function *TsanRead[kNumberOfAccessSizes]; 104 Function *TsanWrite[kNumberOfAccessSizes]; 105 Function *TsanUnalignedRead[kNumberOfAccessSizes]; 106 Function *TsanUnalignedWrite[kNumberOfAccessSizes]; 107 Function *TsanAtomicLoad[kNumberOfAccessSizes]; 108 Function *TsanAtomicStore[kNumberOfAccessSizes]; 109 Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; 110 Function *TsanAtomicCAS[kNumberOfAccessSizes]; 111 Function *TsanAtomicThreadFence; 112 Function *TsanAtomicSignalFence; 113 Function *TsanVptrUpdate; 114 Function *TsanVptrLoad; 115 Function *MemmoveFn, *MemcpyFn, *MemsetFn; 116}; 117} // namespace 118 119char ThreadSanitizer::ID = 0; 120INITIALIZE_PASS(ThreadSanitizer, "tsan", 121 "ThreadSanitizer: detects data races.", 122 false, false) 123 124const char *ThreadSanitizer::getPassName() const { 125 return "ThreadSanitizer"; 126} 127 128FunctionPass *llvm::createThreadSanitizerPass() { 129 return new ThreadSanitizer(); 130} 131 132void ThreadSanitizer::initializeCallbacks(Module &M) { 133 IRBuilder<> IRB(M.getContext()); 134 // Initialize the callbacks. 135 TsanFuncEntry = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 136 "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 137 TsanFuncExit = checkSanitizerInterfaceFunction( 138 M.getOrInsertFunction("__tsan_func_exit", IRB.getVoidTy(), nullptr)); 139 OrdTy = IRB.getInt32Ty(); 140 for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 141 const size_t ByteSize = 1 << i; 142 const size_t BitSize = ByteSize * 8; 143 SmallString<32> ReadName("__tsan_read" + itostr(ByteSize)); 144 TsanRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 145 ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 146 147 SmallString<32> WriteName("__tsan_write" + itostr(ByteSize)); 148 TsanWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 149 WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 150 151 SmallString<64> UnalignedReadName("__tsan_unaligned_read" + 152 itostr(ByteSize)); 153 TsanUnalignedRead[i] = 154 checkSanitizerInterfaceFunction(M.getOrInsertFunction( 155 UnalignedReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 156 157 SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + 158 itostr(ByteSize)); 159 TsanUnalignedWrite[i] = 160 checkSanitizerInterfaceFunction(M.getOrInsertFunction( 161 UnalignedWriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 162 163 Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 164 Type *PtrTy = Ty->getPointerTo(); 165 SmallString<32> AtomicLoadName("__tsan_atomic" + itostr(BitSize) + 166 "_load"); 167 TsanAtomicLoad[i] = checkSanitizerInterfaceFunction( 168 M.getOrInsertFunction(AtomicLoadName, Ty, PtrTy, OrdTy, nullptr)); 169 170 SmallString<32> AtomicStoreName("__tsan_atomic" + itostr(BitSize) + 171 "_store"); 172 TsanAtomicStore[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 173 AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy, nullptr)); 174 175 for (int op = AtomicRMWInst::FIRST_BINOP; 176 op <= AtomicRMWInst::LAST_BINOP; ++op) { 177 TsanAtomicRMW[op][i] = nullptr; 178 const char *NamePart = nullptr; 179 if (op == AtomicRMWInst::Xchg) 180 NamePart = "_exchange"; 181 else if (op == AtomicRMWInst::Add) 182 NamePart = "_fetch_add"; 183 else if (op == AtomicRMWInst::Sub) 184 NamePart = "_fetch_sub"; 185 else if (op == AtomicRMWInst::And) 186 NamePart = "_fetch_and"; 187 else if (op == AtomicRMWInst::Or) 188 NamePart = "_fetch_or"; 189 else if (op == AtomicRMWInst::Xor) 190 NamePart = "_fetch_xor"; 191 else if (op == AtomicRMWInst::Nand) 192 NamePart = "_fetch_nand"; 193 else 194 continue; 195 SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 196 TsanAtomicRMW[op][i] = checkSanitizerInterfaceFunction( 197 M.getOrInsertFunction(RMWName, Ty, PtrTy, Ty, OrdTy, nullptr)); 198 } 199 200 SmallString<32> AtomicCASName("__tsan_atomic" + itostr(BitSize) + 201 "_compare_exchange_val"); 202 TsanAtomicCAS[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 203 AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, nullptr)); 204 } 205 TsanVptrUpdate = checkSanitizerInterfaceFunction( 206 M.getOrInsertFunction("__tsan_vptr_update", IRB.getVoidTy(), 207 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), nullptr)); 208 TsanVptrLoad = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 209 "__tsan_vptr_read", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 210 TsanAtomicThreadFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 211 "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, nullptr)); 212 TsanAtomicSignalFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 213 "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, nullptr)); 214 215 MemmoveFn = checkSanitizerInterfaceFunction( 216 M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 217 IRB.getInt8PtrTy(), IntptrTy, nullptr)); 218 MemcpyFn = checkSanitizerInterfaceFunction( 219 M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 220 IRB.getInt8PtrTy(), IntptrTy, nullptr)); 221 MemsetFn = checkSanitizerInterfaceFunction( 222 M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 223 IRB.getInt32Ty(), IntptrTy, nullptr)); 224} 225 226bool ThreadSanitizer::doInitialization(Module &M) { 227 const DataLayout &DL = M.getDataLayout(); 228 229 // Always insert a call to __tsan_init into the module's CTORs. 230 IRBuilder<> IRB(M.getContext()); 231 IntptrTy = IRB.getIntPtrTy(DL); 232 Value *TsanInit = M.getOrInsertFunction("__tsan_init", 233 IRB.getVoidTy(), nullptr); 234 appendToGlobalCtors(M, cast<Function>(TsanInit), 0); 235 236 return true; 237} 238 239static bool isVtableAccess(Instruction *I) { 240 if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 241 return Tag->isTBAAVtableAccess(); 242 return false; 243} 244 245bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 246 // If this is a GEP, just analyze its pointer operand. 247 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 248 Addr = GEP->getPointerOperand(); 249 250 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 251 if (GV->isConstant()) { 252 // Reads from constant globals can not race with any writes. 253 NumOmittedReadsFromConstantGlobals++; 254 return true; 255 } 256 } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 257 if (isVtableAccess(L)) { 258 // Reads from a vtable pointer can not race with any writes. 259 NumOmittedReadsFromVtable++; 260 return true; 261 } 262 } 263 return false; 264} 265 266// Instrumenting some of the accesses may be proven redundant. 267// Currently handled: 268// - read-before-write (within same BB, no calls between) 269// - not captured variables 270// 271// We do not handle some of the patterns that should not survive 272// after the classic compiler optimizations. 273// E.g. two reads from the same temp should be eliminated by CSE, 274// two writes should be eliminated by DSE, etc. 275// 276// 'Local' is a vector of insns within the same BB (no calls between). 277// 'All' is a vector of insns that will be instrumented. 278void ThreadSanitizer::chooseInstructionsToInstrument( 279 SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All, 280 const DataLayout &DL) { 281 SmallSet<Value*, 8> WriteTargets; 282 // Iterate from the end. 283 for (SmallVectorImpl<Instruction*>::reverse_iterator It = Local.rbegin(), 284 E = Local.rend(); It != E; ++It) { 285 Instruction *I = *It; 286 if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 287 WriteTargets.insert(Store->getPointerOperand()); 288 } else { 289 LoadInst *Load = cast<LoadInst>(I); 290 Value *Addr = Load->getPointerOperand(); 291 if (WriteTargets.count(Addr)) { 292 // We will write to this temp, so no reason to analyze the read. 293 NumOmittedReadsBeforeWrite++; 294 continue; 295 } 296 if (addrPointsToConstantData(Addr)) { 297 // Addr points to some constant data -- it can not race with any writes. 298 continue; 299 } 300 } 301 Value *Addr = isa<StoreInst>(*I) 302 ? cast<StoreInst>(I)->getPointerOperand() 303 : cast<LoadInst>(I)->getPointerOperand(); 304 if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) && 305 !PointerMayBeCaptured(Addr, true, true)) { 306 // The variable is addressable but not captured, so it cannot be 307 // referenced from a different thread and participate in a data race 308 // (see llvm/Analysis/CaptureTracking.h for details). 309 NumOmittedNonCaptured++; 310 continue; 311 } 312 All.push_back(I); 313 } 314 Local.clear(); 315} 316 317static bool isAtomic(Instruction *I) { 318 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 319 return LI->isAtomic() && LI->getSynchScope() == CrossThread; 320 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 321 return SI->isAtomic() && SI->getSynchScope() == CrossThread; 322 if (isa<AtomicRMWInst>(I)) 323 return true; 324 if (isa<AtomicCmpXchgInst>(I)) 325 return true; 326 if (isa<FenceInst>(I)) 327 return true; 328 return false; 329} 330 331bool ThreadSanitizer::runOnFunction(Function &F) { 332 initializeCallbacks(*F.getParent()); 333 SmallVector<Instruction*, 8> RetVec; 334 SmallVector<Instruction*, 8> AllLoadsAndStores; 335 SmallVector<Instruction*, 8> LocalLoadsAndStores; 336 SmallVector<Instruction*, 8> AtomicAccesses; 337 SmallVector<Instruction*, 8> MemIntrinCalls; 338 bool Res = false; 339 bool HasCalls = false; 340 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 341 const DataLayout &DL = F.getParent()->getDataLayout(); 342 343 // Traverse all instructions, collect loads/stores/returns, check for calls. 344 for (auto &BB : F) { 345 for (auto &Inst : BB) { 346 if (isAtomic(&Inst)) 347 AtomicAccesses.push_back(&Inst); 348 else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 349 LocalLoadsAndStores.push_back(&Inst); 350 else if (isa<ReturnInst>(Inst)) 351 RetVec.push_back(&Inst); 352 else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { 353 if (isa<MemIntrinsic>(Inst)) 354 MemIntrinCalls.push_back(&Inst); 355 HasCalls = true; 356 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 357 DL); 358 } 359 } 360 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 361 } 362 363 // We have collected all loads and stores. 364 // FIXME: many of these accesses do not need to be checked for races 365 // (e.g. variables that do not escape, etc). 366 367 // Instrument memory accesses only if we want to report bugs in the function. 368 if (ClInstrumentMemoryAccesses && SanitizeFunction) 369 for (auto Inst : AllLoadsAndStores) { 370 Res |= instrumentLoadOrStore(Inst, DL); 371 } 372 373 // Instrument atomic memory accesses in any case (they can be used to 374 // implement synchronization). 375 if (ClInstrumentAtomics) 376 for (auto Inst : AtomicAccesses) { 377 Res |= instrumentAtomic(Inst, DL); 378 } 379 380 if (ClInstrumentMemIntrinsics && SanitizeFunction) 381 for (auto Inst : MemIntrinCalls) { 382 Res |= instrumentMemIntrinsic(Inst); 383 } 384 385 // Instrument function entry/exit points if there were instrumented accesses. 386 if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 387 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 388 Value *ReturnAddress = IRB.CreateCall( 389 Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 390 IRB.getInt32(0)); 391 IRB.CreateCall(TsanFuncEntry, ReturnAddress); 392 for (auto RetInst : RetVec) { 393 IRBuilder<> IRBRet(RetInst); 394 IRBRet.CreateCall(TsanFuncExit); 395 } 396 Res = true; 397 } 398 return Res; 399} 400 401bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, 402 const DataLayout &DL) { 403 IRBuilder<> IRB(I); 404 bool IsWrite = isa<StoreInst>(*I); 405 Value *Addr = IsWrite 406 ? cast<StoreInst>(I)->getPointerOperand() 407 : cast<LoadInst>(I)->getPointerOperand(); 408 int Idx = getMemoryAccessFuncIndex(Addr, DL); 409 if (Idx < 0) 410 return false; 411 if (IsWrite && isVtableAccess(I)) { 412 DEBUG(dbgs() << " VPTR : " << *I << "\n"); 413 Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 414 // StoredValue may be a vector type if we are storing several vptrs at once. 415 // In this case, just take the first element of the vector since this is 416 // enough to find vptr races. 417 if (isa<VectorType>(StoredValue->getType())) 418 StoredValue = IRB.CreateExtractElement( 419 StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 420 if (StoredValue->getType()->isIntegerTy()) 421 StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 422 // Call TsanVptrUpdate. 423 IRB.CreateCall2(TsanVptrUpdate, 424 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 425 IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())); 426 NumInstrumentedVtableWrites++; 427 return true; 428 } 429 if (!IsWrite && isVtableAccess(I)) { 430 IRB.CreateCall(TsanVptrLoad, 431 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 432 NumInstrumentedVtableReads++; 433 return true; 434 } 435 const unsigned Alignment = IsWrite 436 ? cast<StoreInst>(I)->getAlignment() 437 : cast<LoadInst>(I)->getAlignment(); 438 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 439 const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 440 Value *OnAccessFunc = nullptr; 441 if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) 442 OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 443 else 444 OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 445 IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 446 if (IsWrite) NumInstrumentedWrites++; 447 else NumInstrumentedReads++; 448 return true; 449} 450 451static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 452 uint32_t v = 0; 453 switch (ord) { 454 case NotAtomic: llvm_unreachable("unexpected atomic ordering!"); 455 case Unordered: // Fall-through. 456 case Monotonic: v = 0; break; 457 // case Consume: v = 1; break; // Not specified yet. 458 case Acquire: v = 2; break; 459 case Release: v = 3; break; 460 case AcquireRelease: v = 4; break; 461 case SequentiallyConsistent: v = 5; break; 462 } 463 return IRB->getInt32(v); 464} 465 466// If a memset intrinsic gets inlined by the code gen, we will miss races on it. 467// So, we either need to ensure the intrinsic is not inlined, or instrument it. 468// We do not instrument memset/memmove/memcpy intrinsics (too complicated), 469// instead we simply replace them with regular function calls, which are then 470// intercepted by the run-time. 471// Since tsan is running after everyone else, the calls should not be 472// replaced back with intrinsics. If that becomes wrong at some point, 473// we will need to call e.g. __tsan_memset to avoid the intrinsics. 474bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 475 IRBuilder<> IRB(I); 476 if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 477 IRB.CreateCall3(MemsetFn, 478 IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 479 IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), 480 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)); 481 I->eraseFromParent(); 482 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 483 IRB.CreateCall3(isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 484 IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 485 IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), 486 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)); 487 I->eraseFromParent(); 488 } 489 return false; 490} 491 492// Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 493// standards. For background see C++11 standard. A slightly older, publicly 494// available draft of the standard (not entirely up-to-date, but close enough 495// for casual browsing) is available here: 496// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 497// The following page contains more background information: 498// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 499 500bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 501 IRBuilder<> IRB(I); 502 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 503 Value *Addr = LI->getPointerOperand(); 504 int Idx = getMemoryAccessFuncIndex(Addr, DL); 505 if (Idx < 0) 506 return false; 507 const size_t ByteSize = 1 << Idx; 508 const size_t BitSize = ByteSize * 8; 509 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 510 Type *PtrTy = Ty->getPointerTo(); 511 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 512 createOrdering(&IRB, LI->getOrdering())}; 513 CallInst *C = CallInst::Create(TsanAtomicLoad[Idx], Args); 514 ReplaceInstWithInst(I, C); 515 516 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 517 Value *Addr = SI->getPointerOperand(); 518 int Idx = getMemoryAccessFuncIndex(Addr, DL); 519 if (Idx < 0) 520 return false; 521 const size_t ByteSize = 1 << Idx; 522 const size_t BitSize = ByteSize * 8; 523 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 524 Type *PtrTy = Ty->getPointerTo(); 525 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 526 IRB.CreateIntCast(SI->getValueOperand(), Ty, false), 527 createOrdering(&IRB, SI->getOrdering())}; 528 CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); 529 ReplaceInstWithInst(I, C); 530 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 531 Value *Addr = RMWI->getPointerOperand(); 532 int Idx = getMemoryAccessFuncIndex(Addr, DL); 533 if (Idx < 0) 534 return false; 535 Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 536 if (!F) 537 return false; 538 const size_t ByteSize = 1 << Idx; 539 const size_t BitSize = ByteSize * 8; 540 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 541 Type *PtrTy = Ty->getPointerTo(); 542 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 543 IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 544 createOrdering(&IRB, RMWI->getOrdering())}; 545 CallInst *C = CallInst::Create(F, Args); 546 ReplaceInstWithInst(I, C); 547 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 548 Value *Addr = CASI->getPointerOperand(); 549 int Idx = getMemoryAccessFuncIndex(Addr, DL); 550 if (Idx < 0) 551 return false; 552 const size_t ByteSize = 1 << Idx; 553 const size_t BitSize = ByteSize * 8; 554 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 555 Type *PtrTy = Ty->getPointerTo(); 556 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 557 IRB.CreateIntCast(CASI->getCompareOperand(), Ty, false), 558 IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false), 559 createOrdering(&IRB, CASI->getSuccessOrdering()), 560 createOrdering(&IRB, CASI->getFailureOrdering())}; 561 CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 562 Value *Success = IRB.CreateICmpEQ(C, CASI->getCompareOperand()); 563 564 Value *Res = IRB.CreateInsertValue(UndefValue::get(CASI->getType()), C, 0); 565 Res = IRB.CreateInsertValue(Res, Success, 1); 566 567 I->replaceAllUsesWith(Res); 568 I->eraseFromParent(); 569 } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 570 Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 571 Function *F = FI->getSynchScope() == SingleThread ? 572 TsanAtomicSignalFence : TsanAtomicThreadFence; 573 CallInst *C = CallInst::Create(F, Args); 574 ReplaceInstWithInst(I, C); 575 } 576 return true; 577} 578 579int ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, 580 const DataLayout &DL) { 581 Type *OrigPtrTy = Addr->getType(); 582 Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 583 assert(OrigTy->isSized()); 584 uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 585 if (TypeSize != 8 && TypeSize != 16 && 586 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 587 NumAccessesWithBadSize++; 588 // Ignore all unusual sizes. 589 return -1; 590 } 591 size_t Idx = countTrailingZeros(TypeSize / 8); 592 assert(Idx < kNumberOfAccessSizes); 593 return Idx; 594} 595