ThreadSanitizer.cpp revision 6702e53926c2ce04ab4c049930d24a9c42996941
160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===-- ThreadSanitizer.cpp - race detector -------------------------------===// 260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The LLVM Compiler Infrastructure 460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is distributed under the University of Illinois Open Source 660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// License. See LICENSE.TXT for details. 760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is a part of ThreadSanitizer, a race detector. 1160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The tool is under development, for the details about previous versions see 1360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// http://code.google.com/p/data-race-test 1460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The instrumentation phase is quite simple: 1660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls to run-time library before every memory access. 1760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Optimizations may apply to avoid instrumenting some of the accesses. 1860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls at function entry/exit. 1960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The rest is handled by the run-time library. 2060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 2160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 2260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#define DEBUG_TYPE "tsan" 2360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 24b5b86d263a651566cb25c0f406f75ceffb771029Kostya Serebryany#include "BlackList.h" 2506cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/Function.h" 2606cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/IRBuilder.h" 2706cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/Intrinsics.h" 2806cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/LLVMContext.h" 2906cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/Metadata.h" 3006cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/Module.h" 3106cb8ed00696eb14d1b831921452e50ec0568ea2Chandler Carruth#include "llvm/Type.h" 322076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany#include "llvm/ADT/SmallSet.h" 3360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/ADT/SmallString.h" 3460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/ADT/SmallVector.h" 352d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany#include "llvm/ADT/Statistic.h" 3660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/ADT/StringExtras.h" 376e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany#include "llvm/Support/CommandLine.h" 3860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/Debug.h" 3960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/MathExtras.h" 4052eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany#include "llvm/Support/raw_ostream.h" 413574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow#include "llvm/DataLayout.h" 4260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Transforms/Instrumentation.h" 43e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany#include "llvm/Transforms/Utils/BasicBlockUtils.h" 4460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Transforms/Utils/ModuleUtils.h" 4560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 4660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanyusing namespace llvm; 4760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 486e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryanystatic cl::opt<std::string> ClBlackListFile("tsan-blacklist", 496e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany cl::desc("Blacklist file"), cl::Hidden); 5041d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentMemoryAccesses( 5141d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-memory-accesses", cl::init(true), 5241d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument memory accesses"), cl::Hidden); 5341d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentFuncEntryExit( 5441d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-func-entry-exit", cl::init(true), 5541d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument function entry and exit"), cl::Hidden); 5641d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentAtomics( 5741d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-atomics", cl::init(true), 5841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument atomics"), cl::Hidden); 596e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany 602d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 612d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 621dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey SamsonovSTATISTIC(NumOmittedReadsBeforeWrite, 632d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads ignored due to following writes"); 642d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 652d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 662d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromConstantGlobals, 672d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads from constant globals"); 682d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 692076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 7060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanynamespace { 712076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 7260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany/// ThreadSanitizer: instrument the code in module to find races. 7360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanystruct ThreadSanitizer : public FunctionPass { 7460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany ThreadSanitizer(); 75e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const char *getPassName() const; 7660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool runOnFunction(Function &F); 7760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool doInitialization(Module &M); 7860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany static char ID; // Pass identification, replacement for typeid. 7960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 8060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany private: 81e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany bool instrumentLoadOrStore(Instruction *I); 82e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany bool instrumentAtomic(Instruction *I); 8337cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany void chooseInstructionsToInstrument(SmallVectorImpl<Instruction*> &Local, 8437cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany SmallVectorImpl<Instruction*> &All); 85cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany bool addrPointsToConstantData(Value *Addr); 86e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int getMemoryAccessFuncIndex(Value *Addr); 872076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 883574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow DataLayout *TD; 89b5b86d263a651566cb25c0f406f75ceffb771029Kostya Serebryany OwningPtr<BlackList> BL; 90e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IntegerType *OrdTy; 9160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Callbacks to run-time library are computed in doInitialization. 92e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncEntry; 93e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncExit; 9460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 953eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany static const size_t kNumberOfAccessSizes = 5; 96e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanRead[kNumberOfAccessSizes]; 97e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanWrite[kNumberOfAccessSizes]; 98e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicLoad[kNumberOfAccessSizes]; 99e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicStore[kNumberOfAccessSizes]; 1009f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; 1019f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicCAS[kNumberOfAccessSizes]; 1029f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicThreadFence; 1039f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicSignalFence; 104e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanVptrUpdate; 10560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany}; 10660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} // namespace 10760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 10860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanychar ThreadSanitizer::ID = 0; 10960ebb1947faed42e493179e569c5db0c01d38a2aKostya SerebryanyINITIALIZE_PASS(ThreadSanitizer, "tsan", 11060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany "ThreadSanitizer: detects data races.", 11160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany false, false) 11260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 113e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanyconst char *ThreadSanitizer::getPassName() const { 114e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return "ThreadSanitizer"; 115e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 116e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 11760ebb1947faed42e493179e569c5db0c01d38a2aKostya SerebryanyThreadSanitizer::ThreadSanitizer() 11860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany : FunctionPass(ID), 11960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany TD(NULL) { 12060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 12160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 12260ebb1947faed42e493179e569c5db0c01d38a2aKostya SerebryanyFunctionPass *llvm::createThreadSanitizerPass() { 12360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return new ThreadSanitizer(); 12460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 12560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 126e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic Function *checkInterfaceFunction(Constant *FuncOrBitcast) { 127e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Function *F = dyn_cast<Function>(FuncOrBitcast)) 128e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return F; 129e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany FuncOrBitcast->dump(); 130e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany report_fatal_error("ThreadSanitizer interface function redefined"); 131e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 132e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 13360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanybool ThreadSanitizer::doInitialization(Module &M) { 1343574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow TD = getAnalysisIfAvailable<DataLayout>(); 13560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany if (!TD) 13660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return false; 137b5b86d263a651566cb25c0f406f75ceffb771029Kostya Serebryany BL.reset(new BlackList(ClBlackListFile)); 1386e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany 13960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Always insert a call to __tsan_init into the module's CTORs. 14060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(M.getContext()); 14160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *TsanInit = M.getOrInsertFunction("__tsan_init", 14260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.getVoidTy(), NULL); 14360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany appendToGlobalCtors(M, cast<Function>(TsanInit), 0); 14460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 14560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Initialize the callbacks. 146e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanFuncEntry = checkInterfaceFunction(M.getOrInsertFunction( 147e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL)); 148e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanFuncExit = checkInterfaceFunction(M.getOrInsertFunction( 149e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "__tsan_func_exit", IRB.getVoidTy(), NULL)); 150e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany OrdTy = IRB.getInt32Ty(); 1513eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 152e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << i; 153e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 154e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> ReadName("__tsan_read" + itostr(ByteSize)); 155e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanRead[i] = checkInterfaceFunction(M.getOrInsertFunction( 156e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL)); 157e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 158e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> WriteName("__tsan_write" + itostr(ByteSize)); 159e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanWrite[i] = checkInterfaceFunction(M.getOrInsertFunction( 160e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL)); 161e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 162e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 163e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *PtrTy = Ty->getPointerTo(); 164e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> AtomicLoadName("__tsan_atomic" + itostr(BitSize) + 165e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "_load"); 166e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanAtomicLoad[i] = checkInterfaceFunction(M.getOrInsertFunction( 167e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany AtomicLoadName, Ty, PtrTy, OrdTy, NULL)); 168e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 169e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> AtomicStoreName("__tsan_atomic" + itostr(BitSize) + 170e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "_store"); 171e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanAtomicStore[i] = checkInterfaceFunction(M.getOrInsertFunction( 172e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy, 173e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany NULL)); 1749f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 1759f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov for (int op = AtomicRMWInst::FIRST_BINOP; 1769f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov op <= AtomicRMWInst::LAST_BINOP; ++op) { 1779f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicRMW[op][i] = NULL; 1789f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const char *NamePart = NULL; 1799f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (op == AtomicRMWInst::Xchg) 1809f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_exchange"; 1819f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Add) 1829f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_add"; 1839f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Sub) 1849f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_sub"; 1859f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::And) 1869f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_and"; 1879f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Or) 1889f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_or"; 1899f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Xor) 1909f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_xor"; 1919f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else 1929f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov continue; 1939f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 1949f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicRMW[op][i] = checkInterfaceFunction(M.getOrInsertFunction( 1959f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov RMWName, Ty, PtrTy, Ty, OrdTy, NULL)); 1969f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } 1979f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 1989f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov SmallString<32> AtomicCASName("__tsan_atomic" + itostr(BitSize) + 1999f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "_compare_exchange_val"); 2009f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicCAS[i] = checkInterfaceFunction(M.getOrInsertFunction( 2016702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, NULL)); 20260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 203e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanVptrUpdate = checkInterfaceFunction(M.getOrInsertFunction( 204e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "__tsan_vptr_update", IRB.getVoidTy(), IRB.getInt8PtrTy(), 205e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRB.getInt8PtrTy(), NULL)); 2069f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicThreadFence = checkInterfaceFunction(M.getOrInsertFunction( 2079f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, NULL)); 2089f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicSignalFence = checkInterfaceFunction(M.getOrInsertFunction( 2099f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, NULL)); 21060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 21160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 21260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 213cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanystatic bool isVtableAccess(Instruction *I) { 214cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) { 215cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (Tag->getNumOperands() < 1) return false; 216cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (MDString *Tag1 = dyn_cast<MDString>(Tag->getOperand(0))) { 217cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (Tag1->getString() == "vtable pointer") return true; 218cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 219cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 220cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 221cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 222cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 223cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanybool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 224cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // If this is a GEP, just analyze its pointer operand. 225cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 226cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Addr = GEP->getPointerOperand(); 227cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 228cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 229cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GV->isConstant()) { 230cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from constant globals can not race with any writes. 2312d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromConstantGlobals++; 232cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 233cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 2341dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey Samsonov } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 235cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (isVtableAccess(L)) { 236cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from a vtable pointer can not race with any writes. 2372d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromVtable++; 238cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 239cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 240cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 241cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 242cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 243cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 2442076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Instrumenting some of the accesses may be proven redundant. 2452076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Currently handled: 2462076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// - read-before-write (within same BB, no calls between) 2472076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 2482076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// We do not handle some of the patterns that should not survive 2492076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// after the classic compiler optimizations. 2502076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// E.g. two reads from the same temp should be eliminated by CSE, 2512076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// two writes should be eliminated by DSE, etc. 2522076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 2532076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'Local' is a vector of insns within the same BB (no calls between). 2542076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'All' is a vector of insns that will be instrumented. 25537cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryanyvoid ThreadSanitizer::chooseInstructionsToInstrument( 2562076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVectorImpl<Instruction*> &Local, 2572076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVectorImpl<Instruction*> &All) { 2582076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallSet<Value*, 8> WriteTargets; 2592076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // Iterate from the end. 2602076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany for (SmallVectorImpl<Instruction*>::reverse_iterator It = Local.rbegin(), 2612076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany E = Local.rend(); It != E; ++It) { 2622076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany Instruction *I = *It; 2632076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 2642076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany WriteTargets.insert(Store->getPointerOperand()); 2652076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } else { 2662076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany LoadInst *Load = cast<LoadInst>(I); 267cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Value *Addr = Load->getPointerOperand(); 268cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (WriteTargets.count(Addr)) { 2692076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // We will write to this temp, so no reason to analyze the read. 2702d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsBeforeWrite++; 2712076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany continue; 2722076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 273cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (addrPointsToConstantData(Addr)) { 274cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Addr points to some constant data -- it can not race with any writes. 275cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany continue; 276cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 2772076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 2782076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany All.push_back(I); 2792076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 2802076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany Local.clear(); 2812076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany} 2822076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 283e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic bool isAtomic(Instruction *I) { 284e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) 285e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return LI->isAtomic() && LI->getSynchScope() == CrossThread; 28637cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (StoreInst *SI = dyn_cast<StoreInst>(I)) 287e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return SI->isAtomic() && SI->getSynchScope() == CrossThread; 28837cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicRMWInst>(I)) 289e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 29037cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicCmpXchgInst>(I)) 291e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 2929f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (isa<FenceInst>(I)) 2939f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return true; 294e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 295e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 296e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 29760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanybool ThreadSanitizer::runOnFunction(Function &F) { 29860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany if (!TD) return false; 2996e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany if (BL->isIn(F)) return false; 30060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany SmallVector<Instruction*, 8> RetVec; 3012076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> AllLoadsAndStores; 3022076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> LocalLoadsAndStores; 303e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallVector<Instruction*, 8> AtomicAccesses; 30460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool Res = false; 30560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool HasCalls = false; 30660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 30760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Traverse all instructions, collect loads/stores/returns, check for calls. 30860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany for (Function::iterator FI = F.begin(), FE = F.end(); 30960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany FI != FE; ++FI) { 31060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany BasicBlock &BB = *FI; 31160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); 31260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany BI != BE; ++BI) { 313e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (isAtomic(BI)) 314e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany AtomicAccesses.push_back(BI); 315e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany else if (isa<LoadInst>(BI) || isa<StoreInst>(BI)) 3162076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany LocalLoadsAndStores.push_back(BI); 31760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany else if (isa<ReturnInst>(BI)) 31860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany RetVec.push_back(BI); 3192076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany else if (isa<CallInst>(BI) || isa<InvokeInst>(BI)) { 32060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany HasCalls = true; 32137cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores); 3222076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 32360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 32437cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores); 32560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 32660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 32760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // We have collected all loads and stores. 32860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // FIXME: many of these accesses do not need to be checked for races 32960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // (e.g. variables that do not escape, etc). 33060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 33160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Instrument memory accesses. 33241d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if (ClInstrumentMemoryAccesses) 33341d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany for (size_t i = 0, n = AllLoadsAndStores.size(); i < n; ++i) { 33441d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany Res |= instrumentLoadOrStore(AllLoadsAndStores[i]); 33541d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 33660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 337e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany // Instrument atomic memory accesses. 33841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if (ClInstrumentAtomics) 33941d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany for (size_t i = 0, n = AtomicAccesses.size(); i < n; ++i) { 34041d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany Res |= instrumentAtomic(AtomicAccesses[i]); 34141d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 342e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 34360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Instrument function entry/exit points if there were instrumented accesses. 34441d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 34560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 34660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *ReturnAddress = IRB.CreateCall( 34760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 34860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.getInt32(0)); 34960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(TsanFuncEntry, ReturnAddress); 35060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany for (size_t i = 0, n = RetVec.size(); i < n; ++i) { 35160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRBRet(RetVec[i]); 35260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBRet.CreateCall(TsanFuncExit); 35360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 35452eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Res = true; 35560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 35660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return Res; 35760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 35860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 35960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanybool ThreadSanitizer::instrumentLoadOrStore(Instruction *I) { 36060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(I); 36160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool IsWrite = isa<StoreInst>(*I); 36260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *Addr = IsWrite 36360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany ? cast<StoreInst>(I)->getPointerOperand() 36460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany : cast<LoadInst>(I)->getPointerOperand(); 365e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int Idx = getMemoryAccessFuncIndex(Addr); 366e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 36760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return false; 36852eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany if (IsWrite && isVtableAccess(I)) { 3694a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany DEBUG(dbgs() << " VPTR : " << *I << "\n"); 37052eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 3714a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany // StoredValue does not necessary have a pointer type. 3724a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany if (isa<IntegerType>(StoredValue->getType())) 3734a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 3744a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany // Call TsanVptrUpdate. 37552eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreateCall2(TsanVptrUpdate, 37652eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 37752eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())); 3782d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumInstrumentedVtableWrites++; 37952eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany return true; 38052eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany } 38160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 38260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 3832d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany if (IsWrite) NumInstrumentedWrites++; 3842d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany else NumInstrumentedReads++; 38560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 38660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 387e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 388e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 389e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany uint32_t v = 0; 390e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany switch (ord) { 391e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany case NotAtomic: assert(false); 392e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany case Unordered: // Fall-through. 393c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Monotonic: v = 0; break; 3946702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov // case Consume: v = 1; break; // Not specified yet. 395c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Acquire: v = 2; break; 396c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Release: v = 3; break; 397c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case AcquireRelease: v = 4; break; 398c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case SequentiallyConsistent: v = 5; break; 399e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 400c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov return IRB->getInt32(v); 401e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 402e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 4036702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukovstatic ConstantInt *createFailOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 4046702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov uint32_t v = 0; 4056702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov switch (ord) { 4066702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case NotAtomic: assert(false); 4076702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Unordered: // Fall-through. 4086702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Monotonic: v = 0; break; 4096702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov // case Consume: v = 1; break; // Not specified yet. 4106702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Acquire: v = 2; break; 4116702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Release: v = 0; break; 4126702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case AcquireRelease: v = 2; break; 4136702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case SequentiallyConsistent: v = 5; break; 4146702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov } 4156702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov return IRB->getInt32(v); 4166702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov} 4176702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov 4186702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 4196702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// standards. For background see C++11 standard. A slightly older, publically 4206702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// available draft of the standard (not entirely up-to-date, but close enough 4216702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// for casual browsing) is available here: 4226702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf\ 4236702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// The following page contains more background information: 4246702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 4256702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov 426e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanybool ThreadSanitizer::instrumentAtomic(Instruction *I) { 427e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRBuilder<> IRB(I); 428e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 429e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = LI->getPointerOperand(); 430e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int Idx = getMemoryAccessFuncIndex(Addr); 431e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 432e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 433e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << Idx; 434e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 435e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 436b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 437e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 438e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, LI->getOrdering())}; 439e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany CallInst *C = CallInst::Create(TsanAtomicLoad[Idx], 440e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ArrayRef<Value*>(Args)); 441e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReplaceInstWithInst(I, C); 442e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 443e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 444e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = SI->getPointerOperand(); 445e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int Idx = getMemoryAccessFuncIndex(Addr); 446e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 447e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 448e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << Idx; 449e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 450e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 451b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 452e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 453e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRB.CreateIntCast(SI->getValueOperand(), Ty, false), 454e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, SI->getOrdering())}; 455e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany CallInst *C = CallInst::Create(TsanAtomicStore[Idx], 456e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ArrayRef<Value*>(Args)); 457e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReplaceInstWithInst(I, C); 4589f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 4599f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = RMWI->getPointerOperand(); 4609f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov int Idx = getMemoryAccessFuncIndex(Addr); 4619f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 4629f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 4639f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 4649f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (F == NULL) 4659f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 4669f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t ByteSize = 1 << Idx; 4679f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t BitSize = ByteSize * 8; 4689f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 4699f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 4709f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 4719f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 4729f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov createOrdering(&IRB, RMWI->getOrdering())}; 4739f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args)); 4749f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 4759f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 4769f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = CASI->getPointerOperand(); 4779f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov int Idx = getMemoryAccessFuncIndex(Addr); 4789f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 4799f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 4809f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t ByteSize = 1 << Idx; 4819f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t BitSize = ByteSize * 8; 4829f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 4839f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 4849f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 4859f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(CASI->getCompareOperand(), Ty, false), 4869f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false), 4876702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov createOrdering(&IRB, CASI->getOrdering()), 4886702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov createFailOrdering(&IRB, CASI->getOrdering())}; 4899f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov CallInst *C = CallInst::Create(TsanAtomicCAS[Idx], ArrayRef<Value*>(Args)); 4909f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 4919f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 4929f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 4939f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = FI->getSynchScope() == SingleThread ? 4949f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicSignalFence : TsanAtomicThreadFence; 4959f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args)); 4969f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 497e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 498e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 499e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 500e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 501e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanyint ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr) { 502e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigPtrTy = Addr->getType(); 503e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 504e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(OrigTy->isSized()); 505e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany uint32_t TypeSize = TD->getTypeStoreSizeInBits(OrigTy); 506e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (TypeSize != 8 && TypeSize != 16 && 507e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 508e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany NumAccessesWithBadSize++; 509e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany // Ignore all unusual sizes. 510e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return -1; 511e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 512e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany size_t Idx = CountTrailingZeros_32(TypeSize / 8); 513e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(Idx < kNumberOfAccessSizes); 514e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return Idx; 515e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 516