160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===-- ThreadSanitizer.cpp - race detector -------------------------------===// 260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The LLVM Compiler Infrastructure 460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is distributed under the University of Illinois Open Source 660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// License. See LICENSE.TXT for details. 760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is a part of ThreadSanitizer, a race detector. 1160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The tool is under development, for the details about previous versions see 1360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// http://code.google.com/p/data-race-test 1460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The instrumentation phase is quite simple: 1660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls to run-time library before every memory access. 1760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Optimizations may apply to avoid instrumenting some of the accesses. 1860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls at function entry/exit. 1960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The rest is handled by the run-time library. 2060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 2160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 2260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#define DEBUG_TYPE "tsan" 2360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 24d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Transforms/Instrumentation.h" 25d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallSet.h" 26d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallString.h" 27d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallVector.h" 28d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/Statistic.h" 29d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/StringExtras.h" 300b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DataLayout.h" 310b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Function.h" 320b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/IRBuilder.h" 330b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Intrinsics.h" 340b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/LLVMContext.h" 350b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Metadata.h" 360b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Module.h" 370b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Type.h" 386e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany#include "llvm/Support/CommandLine.h" 3960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/Debug.h" 4060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/MathExtras.h" 4152eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany#include "llvm/Support/raw_ostream.h" 42e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany#include "llvm/Transforms/Utils/BasicBlockUtils.h" 4390230c84668269fbd53d163e398cd16486d5d414Chandler Carruth#include "llvm/Transforms/Utils/BlackList.h" 4460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Transforms/Utils/ModuleUtils.h" 4560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 4660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanyusing namespace llvm; 4760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 48f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonovstatic cl::opt<std::string> ClBlacklistFile("tsan-blacklist", 496e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany cl::desc("Blacklist file"), cl::Hidden); 5041d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentMemoryAccesses( 5141d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-memory-accesses", cl::init(true), 5241d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument memory accesses"), cl::Hidden); 5341d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentFuncEntryExit( 5441d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-func-entry-exit", cl::init(true), 5541d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument function entry and exit"), cl::Hidden); 5641d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentAtomics( 5741d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-atomics", cl::init(true), 5841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument atomics"), cl::Hidden); 596e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany 602d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 612d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 621dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey SamsonovSTATISTIC(NumOmittedReadsBeforeWrite, 632d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads ignored due to following writes"); 642d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 652d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 662d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromConstantGlobals, 672d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads from constant globals"); 682d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 692076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 7060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanynamespace { 712076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 7260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany/// ThreadSanitizer: instrument the code in module to find races. 7360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanystruct ThreadSanitizer : public FunctionPass { 74f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov ThreadSanitizer(StringRef BlacklistFile = StringRef()) 75f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov : FunctionPass(ID), 76f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov TD(0), 77f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov BlacklistFile(BlacklistFile.empty() ? ClBlacklistFile 78f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov : BlacklistFile) { } 79e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const char *getPassName() const; 8060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool runOnFunction(Function &F); 8160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool doInitialization(Module &M); 8260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany static char ID; // Pass identification, replacement for typeid. 8360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 8460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany private: 858b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany void initializeCallbacks(Module &M); 86e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany bool instrumentLoadOrStore(Instruction *I); 87e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany bool instrumentAtomic(Instruction *I); 8837cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany void chooseInstructionsToInstrument(SmallVectorImpl<Instruction*> &Local, 8937cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany SmallVectorImpl<Instruction*> &All); 90cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany bool addrPointsToConstantData(Value *Addr); 91e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int getMemoryAccessFuncIndex(Value *Addr); 922076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 933574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow DataLayout *TD; 94f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov SmallString<64> BlacklistFile; 95b5b86d263a651566cb25c0f406f75ceffb771029Kostya Serebryany OwningPtr<BlackList> BL; 96e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IntegerType *OrdTy; 9760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Callbacks to run-time library are computed in doInitialization. 98e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncEntry; 99e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncExit; 10060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 1013eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany static const size_t kNumberOfAccessSizes = 5; 102e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanRead[kNumberOfAccessSizes]; 103e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanWrite[kNumberOfAccessSizes]; 104e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicLoad[kNumberOfAccessSizes]; 105e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicStore[kNumberOfAccessSizes]; 1069f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; 1079f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicCAS[kNumberOfAccessSizes]; 1089f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicThreadFence; 1099f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicSignalFence; 110e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanVptrUpdate; 11160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany}; 11260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} // namespace 11360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 11460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanychar ThreadSanitizer::ID = 0; 11560ebb1947faed42e493179e569c5db0c01d38a2aKostya SerebryanyINITIALIZE_PASS(ThreadSanitizer, "tsan", 11660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany "ThreadSanitizer: detects data races.", 11760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany false, false) 11860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 119e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanyconst char *ThreadSanitizer::getPassName() const { 120e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return "ThreadSanitizer"; 121e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 122e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 123f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey SamsonovFunctionPass *llvm::createThreadSanitizerPass(StringRef BlacklistFile) { 124f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov return new ThreadSanitizer(BlacklistFile); 12560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 12660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 127e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic Function *checkInterfaceFunction(Constant *FuncOrBitcast) { 128e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Function *F = dyn_cast<Function>(FuncOrBitcast)) 129e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return F; 130e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany FuncOrBitcast->dump(); 131e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany report_fatal_error("ThreadSanitizer interface function redefined"); 132e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 133e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 1348b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryanyvoid ThreadSanitizer::initializeCallbacks(Module &M) { 13560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(M.getContext()); 13660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Initialize the callbacks. 137e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanFuncEntry = checkInterfaceFunction(M.getOrInsertFunction( 138e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL)); 139e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanFuncExit = checkInterfaceFunction(M.getOrInsertFunction( 140e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "__tsan_func_exit", IRB.getVoidTy(), NULL)); 141e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany OrdTy = IRB.getInt32Ty(); 1423eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 143e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << i; 144e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 145e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> ReadName("__tsan_read" + itostr(ByteSize)); 146e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanRead[i] = checkInterfaceFunction(M.getOrInsertFunction( 147e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL)); 148e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 149e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> WriteName("__tsan_write" + itostr(ByteSize)); 150e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanWrite[i] = checkInterfaceFunction(M.getOrInsertFunction( 151e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL)); 152e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 153e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 154e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *PtrTy = Ty->getPointerTo(); 155e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> AtomicLoadName("__tsan_atomic" + itostr(BitSize) + 156e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "_load"); 157e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanAtomicLoad[i] = checkInterfaceFunction(M.getOrInsertFunction( 158e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany AtomicLoadName, Ty, PtrTy, OrdTy, NULL)); 159e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 160e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> AtomicStoreName("__tsan_atomic" + itostr(BitSize) + 161e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "_store"); 162e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanAtomicStore[i] = checkInterfaceFunction(M.getOrInsertFunction( 163e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy, 164e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany NULL)); 1659f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 1669f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov for (int op = AtomicRMWInst::FIRST_BINOP; 1679f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov op <= AtomicRMWInst::LAST_BINOP; ++op) { 1689f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicRMW[op][i] = NULL; 1699f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const char *NamePart = NULL; 1709f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (op == AtomicRMWInst::Xchg) 1719f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_exchange"; 1729f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Add) 1739f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_add"; 1749f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Sub) 1759f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_sub"; 1769f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::And) 1779f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_and"; 1789f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Or) 1799f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_or"; 1809f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Xor) 1819f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_xor"; 182b10675ef14bea530551172547e2111bf707a408eDmitry Vyukov else if (op == AtomicRMWInst::Nand) 183b10675ef14bea530551172547e2111bf707a408eDmitry Vyukov NamePart = "_fetch_nand"; 1849f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else 1859f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov continue; 1869f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 1879f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicRMW[op][i] = checkInterfaceFunction(M.getOrInsertFunction( 1889f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov RMWName, Ty, PtrTy, Ty, OrdTy, NULL)); 1899f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } 1909f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 1919f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov SmallString<32> AtomicCASName("__tsan_atomic" + itostr(BitSize) + 1929f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "_compare_exchange_val"); 1939f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicCAS[i] = checkInterfaceFunction(M.getOrInsertFunction( 1946702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, NULL)); 19560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 196e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TsanVptrUpdate = checkInterfaceFunction(M.getOrInsertFunction( 197e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "__tsan_vptr_update", IRB.getVoidTy(), IRB.getInt8PtrTy(), 198e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRB.getInt8PtrTy(), NULL)); 1999f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicThreadFence = checkInterfaceFunction(M.getOrInsertFunction( 2009f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, NULL)); 2019f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicSignalFence = checkInterfaceFunction(M.getOrInsertFunction( 2029f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, NULL)); 2038b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany} 2048b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 2058b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryanybool ThreadSanitizer::doInitialization(Module &M) { 2068b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany TD = getAnalysisIfAvailable<DataLayout>(); 2078b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany if (!TD) 2088b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany return false; 209f045df1b8b7f80e17e34c2b5639082a1d0e289aeAlexey Samsonov BL.reset(new BlackList(BlacklistFile)); 2108b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 2118b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany // Always insert a call to __tsan_init into the module's CTORs. 2128b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany IRBuilder<> IRB(M.getContext()); 2138b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany Value *TsanInit = M.getOrInsertFunction("__tsan_init", 2148b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany IRB.getVoidTy(), NULL); 2158b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany appendToGlobalCtors(M, cast<Function>(TsanInit), 0); 2168b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 21760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 21860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 21960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 220cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanystatic bool isVtableAccess(Instruction *I) { 221cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) { 222cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (Tag->getNumOperands() < 1) return false; 223cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (MDString *Tag1 = dyn_cast<MDString>(Tag->getOperand(0))) { 224cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (Tag1->getString() == "vtable pointer") return true; 225cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 226cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 227cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 228cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 229cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 230cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanybool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 231cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // If this is a GEP, just analyze its pointer operand. 232cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 233cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Addr = GEP->getPointerOperand(); 234cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 235cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 236cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GV->isConstant()) { 237cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from constant globals can not race with any writes. 2382d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromConstantGlobals++; 239cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 240cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 2411dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey Samsonov } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 242cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (isVtableAccess(L)) { 243cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from a vtable pointer can not race with any writes. 2442d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromVtable++; 245cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 246cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 247cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 248cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 249cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 250cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 2512076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Instrumenting some of the accesses may be proven redundant. 2522076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Currently handled: 2532076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// - read-before-write (within same BB, no calls between) 2542076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 2552076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// We do not handle some of the patterns that should not survive 2562076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// after the classic compiler optimizations. 2572076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// E.g. two reads from the same temp should be eliminated by CSE, 2582076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// two writes should be eliminated by DSE, etc. 2592076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 2602076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'Local' is a vector of insns within the same BB (no calls between). 2612076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'All' is a vector of insns that will be instrumented. 26237cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryanyvoid ThreadSanitizer::chooseInstructionsToInstrument( 2632076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVectorImpl<Instruction*> &Local, 2642076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVectorImpl<Instruction*> &All) { 2652076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallSet<Value*, 8> WriteTargets; 2662076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // Iterate from the end. 2672076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany for (SmallVectorImpl<Instruction*>::reverse_iterator It = Local.rbegin(), 2682076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany E = Local.rend(); It != E; ++It) { 2692076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany Instruction *I = *It; 2702076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 2712076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany WriteTargets.insert(Store->getPointerOperand()); 2722076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } else { 2732076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany LoadInst *Load = cast<LoadInst>(I); 274cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Value *Addr = Load->getPointerOperand(); 275cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (WriteTargets.count(Addr)) { 2762076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // We will write to this temp, so no reason to analyze the read. 2772d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsBeforeWrite++; 2782076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany continue; 2792076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 280cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (addrPointsToConstantData(Addr)) { 281cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Addr points to some constant data -- it can not race with any writes. 282cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany continue; 283cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 2842076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 2852076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany All.push_back(I); 2862076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 2872076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany Local.clear(); 2882076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany} 2892076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 290e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic bool isAtomic(Instruction *I) { 291e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) 292e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return LI->isAtomic() && LI->getSynchScope() == CrossThread; 29337cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (StoreInst *SI = dyn_cast<StoreInst>(I)) 294e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return SI->isAtomic() && SI->getSynchScope() == CrossThread; 29537cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicRMWInst>(I)) 296e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 29737cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicCmpXchgInst>(I)) 298e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 2999f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (isa<FenceInst>(I)) 3009f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return true; 301e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 302e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 303e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 30460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanybool ThreadSanitizer::runOnFunction(Function &F) { 30560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany if (!TD) return false; 3066e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany if (BL->isIn(F)) return false; 3078b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany initializeCallbacks(*F.getParent()); 30860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany SmallVector<Instruction*, 8> RetVec; 3092076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> AllLoadsAndStores; 3102076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> LocalLoadsAndStores; 311e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallVector<Instruction*, 8> AtomicAccesses; 31260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool Res = false; 31360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool HasCalls = false; 31460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 31560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Traverse all instructions, collect loads/stores/returns, check for calls. 31660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany for (Function::iterator FI = F.begin(), FE = F.end(); 31760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany FI != FE; ++FI) { 31860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany BasicBlock &BB = *FI; 31960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); 32060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany BI != BE; ++BI) { 321e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (isAtomic(BI)) 322e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany AtomicAccesses.push_back(BI); 323e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany else if (isa<LoadInst>(BI) || isa<StoreInst>(BI)) 3242076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany LocalLoadsAndStores.push_back(BI); 32560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany else if (isa<ReturnInst>(BI)) 32660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany RetVec.push_back(BI); 3272076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany else if (isa<CallInst>(BI) || isa<InvokeInst>(BI)) { 32860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany HasCalls = true; 32937cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores); 3302076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 33160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 33237cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores); 33360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 33460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 33560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // We have collected all loads and stores. 33660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // FIXME: many of these accesses do not need to be checked for races 33760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // (e.g. variables that do not escape, etc). 33860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 33960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Instrument memory accesses. 34041d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if (ClInstrumentMemoryAccesses) 34141d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany for (size_t i = 0, n = AllLoadsAndStores.size(); i < n; ++i) { 34241d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany Res |= instrumentLoadOrStore(AllLoadsAndStores[i]); 34341d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 34460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 345e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany // Instrument atomic memory accesses. 34641d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if (ClInstrumentAtomics) 34741d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany for (size_t i = 0, n = AtomicAccesses.size(); i < n; ++i) { 34841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany Res |= instrumentAtomic(AtomicAccesses[i]); 34941d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 350e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 35160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Instrument function entry/exit points if there were instrumented accesses. 35241d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 35360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 35460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *ReturnAddress = IRB.CreateCall( 35560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 35660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.getInt32(0)); 35760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(TsanFuncEntry, ReturnAddress); 35860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany for (size_t i = 0, n = RetVec.size(); i < n; ++i) { 35960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRBRet(RetVec[i]); 36060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBRet.CreateCall(TsanFuncExit); 36160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 36252eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Res = true; 36360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 36460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return Res; 36560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 36660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 36760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanybool ThreadSanitizer::instrumentLoadOrStore(Instruction *I) { 36860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(I); 36960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool IsWrite = isa<StoreInst>(*I); 37060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *Addr = IsWrite 37160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany ? cast<StoreInst>(I)->getPointerOperand() 37260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany : cast<LoadInst>(I)->getPointerOperand(); 373e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int Idx = getMemoryAccessFuncIndex(Addr); 374e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 37560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return false; 37652eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany if (IsWrite && isVtableAccess(I)) { 3774a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany DEBUG(dbgs() << " VPTR : " << *I << "\n"); 37852eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 3794a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany // StoredValue does not necessary have a pointer type. 3804a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany if (isa<IntegerType>(StoredValue->getType())) 3814a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 3824a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany // Call TsanVptrUpdate. 38352eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreateCall2(TsanVptrUpdate, 38452eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 38552eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())); 3862d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumInstrumentedVtableWrites++; 38752eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany return true; 38852eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany } 38960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 39060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 3912d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany if (IsWrite) NumInstrumentedWrites++; 3922d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany else NumInstrumentedReads++; 39360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 39460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 395e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 396e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 397e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany uint32_t v = 0; 398e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany switch (ord) { 399e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany case NotAtomic: assert(false); 400e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany case Unordered: // Fall-through. 401c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Monotonic: v = 0; break; 4029a33f9f9804126661e87edfe491288b81f5c51e6Dmitry Vyukov // case Consume: v = 1; break; // Not specified yet. 403c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Acquire: v = 2; break; 404c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Release: v = 3; break; 405c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case AcquireRelease: v = 4; break; 406c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case SequentiallyConsistent: v = 5; break; 407e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 408c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov return IRB->getInt32(v); 409e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 410e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 4116702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukovstatic ConstantInt *createFailOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 4126702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov uint32_t v = 0; 4136702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov switch (ord) { 4146702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case NotAtomic: assert(false); 4156702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Unordered: // Fall-through. 4166702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Monotonic: v = 0; break; 4179a33f9f9804126661e87edfe491288b81f5c51e6Dmitry Vyukov // case Consume: v = 1; break; // Not specified yet. 4186702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Acquire: v = 2; break; 4196702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case Release: v = 0; break; 4206702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case AcquireRelease: v = 2; break; 4216702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov case SequentiallyConsistent: v = 5; break; 4226702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov } 4236702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov return IRB->getInt32(v); 4246702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov} 4256702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov 4266702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 4276702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// standards. For background see C++11 standard. A slightly older, publically 4286702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// available draft of the standard (not entirely up-to-date, but close enough 4296702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// for casual browsing) is available here: 43070af909f67565753cbd61886da2b0b7e191d32bfMatt Beaumont-Gay// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 4316702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// The following page contains more background information: 4326702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 4336702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov 434e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanybool ThreadSanitizer::instrumentAtomic(Instruction *I) { 435e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRBuilder<> IRB(I); 436e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 437e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = LI->getPointerOperand(); 438e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int Idx = getMemoryAccessFuncIndex(Addr); 439e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 440e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 441e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << Idx; 442e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 443e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 444b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 445e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 446e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, LI->getOrdering())}; 447e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany CallInst *C = CallInst::Create(TsanAtomicLoad[Idx], 448e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ArrayRef<Value*>(Args)); 449e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReplaceInstWithInst(I, C); 450e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 451e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 452e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = SI->getPointerOperand(); 453e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany int Idx = getMemoryAccessFuncIndex(Addr); 454e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 455e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 456e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << Idx; 457e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 458e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 459b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 460e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 461e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRB.CreateIntCast(SI->getValueOperand(), Ty, false), 462e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, SI->getOrdering())}; 463e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany CallInst *C = CallInst::Create(TsanAtomicStore[Idx], 464e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ArrayRef<Value*>(Args)); 465e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReplaceInstWithInst(I, C); 4669f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 4679f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = RMWI->getPointerOperand(); 4689f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov int Idx = getMemoryAccessFuncIndex(Addr); 4699f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 4709f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 4719f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 4729f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (F == NULL) 4739f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 4749f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t ByteSize = 1 << Idx; 4759f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t BitSize = ByteSize * 8; 4769f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 4779f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 4789f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 4799f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 4809f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov createOrdering(&IRB, RMWI->getOrdering())}; 4819f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args)); 4829f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 4839f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 4849f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = CASI->getPointerOperand(); 4859f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov int Idx = getMemoryAccessFuncIndex(Addr); 4869f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 4879f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 4889f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t ByteSize = 1 << Idx; 4899f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t BitSize = ByteSize * 8; 4909f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 4919f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 4929f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 4939f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(CASI->getCompareOperand(), Ty, false), 4949f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false), 4956702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov createOrdering(&IRB, CASI->getOrdering()), 4966702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov createFailOrdering(&IRB, CASI->getOrdering())}; 4979f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov CallInst *C = CallInst::Create(TsanAtomicCAS[Idx], ArrayRef<Value*>(Args)); 4989f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 4999f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 5009f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 5019f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = FI->getSynchScope() == SingleThread ? 5029f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicSignalFence : TsanAtomicThreadFence; 5039f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args)); 5049f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 505e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 506e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 507e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 508e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 509e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanyint ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr) { 510e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigPtrTy = Addr->getType(); 511e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 512e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(OrigTy->isSized()); 513e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany uint32_t TypeSize = TD->getTypeStoreSizeInBits(OrigTy); 514e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (TypeSize != 8 && TypeSize != 16 && 515e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 516e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany NumAccessesWithBadSize++; 517e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany // Ignore all unusual sizes. 518e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return -1; 519e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 520e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany size_t Idx = CountTrailingZeros_32(TypeSize / 8); 521e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(Idx < kNumberOfAccessSizes); 522e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return Idx; 523e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 524