160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===-- ThreadSanitizer.cpp - race detector -------------------------------===// 260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The LLVM Compiler Infrastructure 460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is distributed under the University of Illinois Open Source 660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// License. See LICENSE.TXT for details. 760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is a part of ThreadSanitizer, a race detector. 1160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The tool is under development, for the details about previous versions see 1360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// http://code.google.com/p/data-race-test 1460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The instrumentation phase is quite simple: 1660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls to run-time library before every memory access. 1760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Optimizations may apply to avoid instrumenting some of the accesses. 1860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls at function entry/exit. 1960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The rest is handled by the run-time library. 2060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 2160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 22d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Transforms/Instrumentation.h" 23d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallSet.h" 24d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallString.h" 25d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallVector.h" 26d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/Statistic.h" 27d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/StringExtras.h" 284c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar#include "llvm/Analysis/CaptureTracking.h" 29de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar#include "llvm/Analysis/TargetLibraryInfo.h" 304c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar#include "llvm/Analysis/ValueTracking.h" 310b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DataLayout.h" 320b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Function.h" 330b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/IRBuilder.h" 34f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany#include "llvm/IR/IntrinsicInst.h" 350b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Intrinsics.h" 360b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/LLVMContext.h" 370b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Metadata.h" 380b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Module.h" 390b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Type.h" 40de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar#include "llvm/ProfileData/InstrProf.h" 416e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany#include "llvm/Support/CommandLine.h" 4260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/Debug.h" 4360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/MathExtras.h" 4452eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany#include "llvm/Support/raw_ostream.h" 45e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany#include "llvm/Transforms/Utils/BasicBlockUtils.h" 46de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar#include "llvm/Transforms/Utils/Local.h" 4760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Transforms/Utils/ModuleUtils.h" 4860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 4960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanyusing namespace llvm; 5060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 51dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "tsan" 52dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5341d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentMemoryAccesses( 5441d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-memory-accesses", cl::init(true), 5541d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument memory accesses"), cl::Hidden); 5641d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentFuncEntryExit( 5741d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-func-entry-exit", cl::init(true), 5841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument function entry and exit"), cl::Hidden); 5941d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentAtomics( 6041d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-atomics", cl::init(true), 6141d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument atomics"), cl::Hidden); 62f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryanystatic cl::opt<bool> ClInstrumentMemIntrinsics( 63f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany "tsan-instrument-memintrinsics", cl::init(true), 64f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 656e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany 662d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 672d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 681dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey SamsonovSTATISTIC(NumOmittedReadsBeforeWrite, 692d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads ignored due to following writes"); 702d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 712d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 72ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry VyukovSTATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 732d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromConstantGlobals, 742d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads from constant globals"); 752d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 76ebe69fe11e48d322045d5949c83283927a0d790bStephen HinesSTATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 772076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 786948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic const char *const kTsanModuleCtorName = "tsan.module_ctor"; 796948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainarstatic const char *const kTsanInitName = "__tsan_init"; 806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar 8160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanynamespace { 822076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 8360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany/// ThreadSanitizer: instrument the code in module to find races. 8460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanystruct ThreadSanitizer : public FunctionPass { 854c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar ThreadSanitizer() : FunctionPass(ID) {} 8636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const char *getPassName() const override; 87de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar void getAnalysisUsage(AnalysisUsage &AU) const override; 8836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool runOnFunction(Function &F) override; 8936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool doInitialization(Module &M) override; 9060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany static char ID; // Pass identification, replacement for typeid. 9160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 9260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany private: 938b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany void initializeCallbacks(Module &M); 944c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); 954c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool instrumentAtomic(Instruction *I, const DataLayout &DL); 96f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany bool instrumentMemIntrinsic(Instruction *I); 974c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 984c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> &All, 994c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL); 100cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany bool addrPointsToConstantData(Value *Addr); 1014c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); 1022076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 103f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany Type *IntptrTy; 104e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IntegerType *OrdTy; 10560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Callbacks to run-time library are computed in doInitialization. 106e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncEntry; 107e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncExit; 10860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 1093eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany static const size_t kNumberOfAccessSizes = 5; 110e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanRead[kNumberOfAccessSizes]; 111e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanWrite[kNumberOfAccessSizes]; 112ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Function *TsanUnalignedRead[kNumberOfAccessSizes]; 113ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Function *TsanUnalignedWrite[kNumberOfAccessSizes]; 114e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicLoad[kNumberOfAccessSizes]; 115e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicStore[kNumberOfAccessSizes]; 1169f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; 1179f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicCAS[kNumberOfAccessSizes]; 1189f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicThreadFence; 1199f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicSignalFence; 120e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanVptrUpdate; 121ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov Function *TsanVptrLoad; 122f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany Function *MemmoveFn, *MemcpyFn, *MemsetFn; 1236948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar Function *TsanCtorFunction; 12460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany}; 12560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} // namespace 12660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 12760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanychar ThreadSanitizer::ID = 0; 128de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarINITIALIZE_PASS_BEGIN( 129de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ThreadSanitizer, "tsan", 130de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar "ThreadSanitizer: detects data races.", 131de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar false, false) 132de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarINITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 133de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga NainarINITIALIZE_PASS_END( 134de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ThreadSanitizer, "tsan", 13560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany "ThreadSanitizer: detects data races.", 13660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany false, false) 13760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 138e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanyconst char *ThreadSanitizer::getPassName() const { 139e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return "ThreadSanitizer"; 140e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 141e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 142de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarvoid ThreadSanitizer::getAnalysisUsage(AnalysisUsage &AU) const { 143de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar AU.addRequired<TargetLibraryInfoWrapperPass>(); 144de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 145de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 146c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesFunctionPass *llvm::createThreadSanitizerPass() { 147c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return new ThreadSanitizer(); 14860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 14960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 1508b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryanyvoid ThreadSanitizer::initializeCallbacks(Module &M) { 15160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(M.getContext()); 15260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Initialize the callbacks. 1530c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanFuncEntry = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 15437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 1550c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanFuncExit = checkSanitizerInterfaceFunction( 1560c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar M.getOrInsertFunction("__tsan_func_exit", IRB.getVoidTy(), nullptr)); 157e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany OrdTy = IRB.getInt32Ty(); 1583eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 159f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned ByteSize = 1U << i; 160f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned BitSize = ByteSize * 8; 161f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::string ByteSizeStr = utostr(ByteSize); 162f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar std::string BitSizeStr = utostr(BitSize); 163f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallString<32> ReadName("__tsan_read" + ByteSizeStr); 1640c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 16537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 166e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 167f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallString<32> WriteName("__tsan_write" + ByteSizeStr); 1680c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 16937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 170e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 171f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr); 1720c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanUnalignedRead[i] = 1730c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar checkSanitizerInterfaceFunction(M.getOrInsertFunction( 1740c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar UnalignedReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 175ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 176f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr); 1770c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanUnalignedWrite[i] = 1780c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar checkSanitizerInterfaceFunction(M.getOrInsertFunction( 1790c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar UnalignedWriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 180ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 181e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 182e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *PtrTy = Ty->getPointerTo(); 183f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load"); 1840c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanAtomicLoad[i] = checkSanitizerInterfaceFunction( 1850c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar M.getOrInsertFunction(AtomicLoadName, Ty, PtrTy, OrdTy, nullptr)); 186e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 187f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store"); 1880c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanAtomicStore[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 1890c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy, nullptr)); 1909f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 1919f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov for (int op = AtomicRMWInst::FIRST_BINOP; 1929f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov op <= AtomicRMWInst::LAST_BINOP; ++op) { 193dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TsanAtomicRMW[op][i] = nullptr; 194dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const char *NamePart = nullptr; 1959f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (op == AtomicRMWInst::Xchg) 1969f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_exchange"; 1979f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Add) 1989f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_add"; 1999f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Sub) 2009f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_sub"; 2019f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::And) 2029f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_and"; 2039f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Or) 2049f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_or"; 2059f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Xor) 2069f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_xor"; 207b10675ef14bea530551172547e2111bf707a408eDmitry Vyukov else if (op == AtomicRMWInst::Nand) 208b10675ef14bea530551172547e2111bf707a408eDmitry Vyukov NamePart = "_fetch_nand"; 2099f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else 2109f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov continue; 2119f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 2120c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanAtomicRMW[op][i] = checkSanitizerInterfaceFunction( 2130c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar M.getOrInsertFunction(RMWName, Ty, PtrTy, Ty, OrdTy, nullptr)); 2149f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } 2159f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 216f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr + 2179f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "_compare_exchange_val"); 2180c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanAtomicCAS[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 21937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, nullptr)); 22060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 2210c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanVptrUpdate = checkSanitizerInterfaceFunction( 2220c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar M.getOrInsertFunction("__tsan_vptr_update", IRB.getVoidTy(), 2230c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), nullptr)); 2240c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanVptrLoad = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 22537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_vptr_read", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 2260c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanAtomicThreadFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 22737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, nullptr)); 2280c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar TsanAtomicSignalFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 22937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, nullptr)); 230f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany 2310c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar MemmoveFn = checkSanitizerInterfaceFunction( 2320c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 2330c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar IRB.getInt8PtrTy(), IntptrTy, nullptr)); 2340c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar MemcpyFn = checkSanitizerInterfaceFunction( 2350c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 2360c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar IRB.getInt8PtrTy(), IntptrTy, nullptr)); 2370c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar MemsetFn = checkSanitizerInterfaceFunction( 2380c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 2390c7f116bb6950ef819323d855415b2f2b0aad987Pirama Arumuga Nainar IRB.getInt32Ty(), IntptrTy, nullptr)); 2408b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany} 2418b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 2428b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryanybool ThreadSanitizer::doInitialization(Module &M) { 2434c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL = M.getDataLayout(); 2446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IntptrTy = DL.getIntPtrType(M.getContext()); 2456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar std::tie(TsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions( 2466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, 2476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar /*InitArgs=*/{}); 2488b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 2496948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar appendToGlobalCtors(M, TsanCtorFunction, 0); 2508b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 25160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 25260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 25360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 254cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanystatic bool isVtableAccess(Instruction *I) { 2550b3d39235aaed8bc66ccffb3942bf7b5f185329cManman Ren if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 2560b3d39235aaed8bc66ccffb3942bf7b5f185329cManman Ren return Tag->isTBAAVtableAccess(); 257cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 258cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 259cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 260de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// Do not instrument known races/"benign races" that come from compiler 261de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar// instrumentatin. The user has no way of suppressing them. 262de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic bool shouldInstrumentReadWriteFromAddress(Value *Addr) { 263de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Peel off GEPs and BitCasts. 264de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Addr = Addr->stripInBoundsOffsets(); 265de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 266de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 267de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (GV->hasSection()) { 268de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar StringRef SectionName = GV->getSection(); 269de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Check if the global is in the PGO counters section. 270de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (SectionName.endswith(getInstrProfCountersSectionName( 271de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar /*AddSegment=*/false))) 272de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 273de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 274de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 275de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Check if the global is in a GCOV counter array. 276de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (GV->getName().startswith("__llvm_gcov_ctr")) 277de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 278de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 279de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 280de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Do not instrument acesses from different address spaces; we cannot deal 281de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // with them. 282de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Addr) { 283de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType()); 284de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (PtrTy->getPointerAddressSpace() != 0) 285de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return false; 286de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 287de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 288de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return true; 289de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 290de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 291cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanybool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 292cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // If this is a GEP, just analyze its pointer operand. 293cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 294cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Addr = GEP->getPointerOperand(); 295cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 296cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 297cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GV->isConstant()) { 298cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from constant globals can not race with any writes. 2992d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromConstantGlobals++; 300cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 301cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 3021dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey Samsonov } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 303cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (isVtableAccess(L)) { 304cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from a vtable pointer can not race with any writes. 3052d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromVtable++; 306cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 307cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 308cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 309cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 310cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 311cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 3122076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Instrumenting some of the accesses may be proven redundant. 3132076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Currently handled: 3142076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// - read-before-write (within same BB, no calls between) 315ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// - not captured variables 3162076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 3172076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// We do not handle some of the patterns that should not survive 3182076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// after the classic compiler optimizations. 3192076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// E.g. two reads from the same temp should be eliminated by CSE, 3202076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// two writes should be eliminated by DSE, etc. 3212076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 3222076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'Local' is a vector of insns within the same BB (no calls between). 3232076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'All' is a vector of insns that will be instrumented. 32437cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryanyvoid ThreadSanitizer::chooseInstructionsToInstrument( 3254c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All, 3264c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL) { 3272076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallSet<Value*, 8> WriteTargets; 3282076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // Iterate from the end. 329de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar for (Instruction *I : reverse(Local)) { 3302076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 331de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Addr = Store->getPointerOperand(); 332de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!shouldInstrumentReadWriteFromAddress(Addr)) 333de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar continue; 334de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar WriteTargets.insert(Addr); 3352076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } else { 3362076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany LoadInst *Load = cast<LoadInst>(I); 337cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Value *Addr = Load->getPointerOperand(); 338de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (!shouldInstrumentReadWriteFromAddress(Addr)) 339de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar continue; 340cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (WriteTargets.count(Addr)) { 3412076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // We will write to this temp, so no reason to analyze the read. 3422d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsBeforeWrite++; 3432076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany continue; 3442076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 345cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (addrPointsToConstantData(Addr)) { 346cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Addr points to some constant data -- it can not race with any writes. 347cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany continue; 348cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 3492076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 350ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Addr = isa<StoreInst>(*I) 351ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ? cast<StoreInst>(I)->getPointerOperand() 352ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : cast<LoadInst>(I)->getPointerOperand(); 3534c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) && 354ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !PointerMayBeCaptured(Addr, true, true)) { 355ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The variable is addressable but not captured, so it cannot be 356ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // referenced from a different thread and participate in a data race 357ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // (see llvm/Analysis/CaptureTracking.h for details). 358ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NumOmittedNonCaptured++; 359ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 360ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 3612076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany All.push_back(I); 3622076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 3632076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany Local.clear(); 3642076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany} 3652076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 366e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic bool isAtomic(Instruction *I) { 367e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) 368e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return LI->isAtomic() && LI->getSynchScope() == CrossThread; 36937cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (StoreInst *SI = dyn_cast<StoreInst>(I)) 370e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return SI->isAtomic() && SI->getSynchScope() == CrossThread; 37137cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicRMWInst>(I)) 372e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 37337cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicCmpXchgInst>(I)) 374e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 3759f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (isa<FenceInst>(I)) 3769f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return true; 377e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 378e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 379e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 38060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanybool ThreadSanitizer::runOnFunction(Function &F) { 3816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // This is required to prevent instrumenting call to __tsan_init from within 3826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar // the module constructor. 3836948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar if (&F == TsanCtorFunction) 3846948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar return false; 3858b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany initializeCallbacks(*F.getParent()); 38660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany SmallVector<Instruction*, 8> RetVec; 3872076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> AllLoadsAndStores; 3882076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> LocalLoadsAndStores; 389e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallVector<Instruction*, 8> AtomicAccesses; 390f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany SmallVector<Instruction*, 8> MemIntrinCalls; 39160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool Res = false; 39260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool HasCalls = false; 393c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 3944c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL = F.getParent()->getDataLayout(); 395de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar const TargetLibraryInfo *TLI = 396de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); 39760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 39860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Traverse all instructions, collect loads/stores/returns, check for calls. 399c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto &BB : F) { 400c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto &Inst : BB) { 401c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (isAtomic(&Inst)) 402c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines AtomicAccesses.push_back(&Inst); 403c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 404c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LocalLoadsAndStores.push_back(&Inst); 405c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines else if (isa<ReturnInst>(Inst)) 406c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines RetVec.push_back(&Inst); 407c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { 408de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (CallInst *CI = dyn_cast<CallInst>(&Inst)) 409de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI); 410c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (isa<MemIntrinsic>(Inst)) 411c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines MemIntrinCalls.push_back(&Inst); 41260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany HasCalls = true; 4134c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 4144c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DL); 4152076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 41660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 4174c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 41860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 41960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 42060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // We have collected all loads and stores. 42160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // FIXME: many of these accesses do not need to be checked for races 42260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // (e.g. variables that do not escape, etc). 42360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 424c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Instrument memory accesses only if we want to report bugs in the function. 425c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (ClInstrumentMemoryAccesses && SanitizeFunction) 426c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto Inst : AllLoadsAndStores) { 4274c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Res |= instrumentLoadOrStore(Inst, DL); 42841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 42960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 430c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Instrument atomic memory accesses in any case (they can be used to 431c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // implement synchronization). 43241d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if (ClInstrumentAtomics) 433c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto Inst : AtomicAccesses) { 4344c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Res |= instrumentAtomic(Inst, DL); 43541d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 436e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 437c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (ClInstrumentMemIntrinsics && SanitizeFunction) 438c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto Inst : MemIntrinCalls) { 439c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Res |= instrumentMemIntrinsic(Inst); 440f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany } 441f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany 44260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Instrument function entry/exit points if there were instrumented accesses. 44341d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 44460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 44560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *ReturnAddress = IRB.CreateCall( 44660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 44760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.getInt32(0)); 44860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(TsanFuncEntry, ReturnAddress); 449c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto RetInst : RetVec) { 450c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines IRBuilder<> IRBRet(RetInst); 4516948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRBRet.CreateCall(TsanFuncExit, {}); 45260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 45352eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Res = true; 45460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 45560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return Res; 45660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 45760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 4584c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarbool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, 4594c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL) { 46060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(I); 46160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool IsWrite = isa<StoreInst>(*I); 46260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *Addr = IsWrite 46360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany ? cast<StoreInst>(I)->getPointerOperand() 46460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany : cast<LoadInst>(I)->getPointerOperand(); 4654c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 466e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 46760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return false; 46852eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany if (IsWrite && isVtableAccess(I)) { 4694a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany DEBUG(dbgs() << " VPTR : " << *I << "\n"); 47052eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 47136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // StoredValue may be a vector type if we are storing several vptrs at once. 47236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // In this case, just take the first element of the vector since this is 47336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // enough to find vptr races. 47436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<VectorType>(StoredValue->getType())) 47536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines StoredValue = IRB.CreateExtractElement( 47636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 47736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StoredValue->getType()->isIntegerTy()) 4784a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 4794a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany // Call TsanVptrUpdate. 4806948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreateCall(TsanVptrUpdate, 4816948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 4826948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())}); 4832d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumInstrumentedVtableWrites++; 48452eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany return true; 48552eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany } 486ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov if (!IsWrite && isVtableAccess(I)) { 487ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov IRB.CreateCall(TsanVptrLoad, 488ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 489ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov NumInstrumentedVtableReads++; 490ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov return true; 491ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov } 492ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const unsigned Alignment = IsWrite 493ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ? cast<StoreInst>(I)->getAlignment() 494ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : cast<LoadInst>(I)->getAlignment(); 495ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 4964c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 497ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *OnAccessFunc = nullptr; 498ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) 499ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 500ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines else 501ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 50260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 5032d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany if (IsWrite) NumInstrumentedWrites++; 5042d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany else NumInstrumentedReads++; 50560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 50660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 507e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 508e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 509e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany uint32_t v = 0; 510e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany switch (ord) { 511de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicOrdering::NotAtomic: 512de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar llvm_unreachable("unexpected atomic ordering!"); 513de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicOrdering::Unordered: // Fall-through. 514de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicOrdering::Monotonic: v = 0; break; 515de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // Not specified yet: 516de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // case AtomicOrdering::Consume: v = 1; break; 517de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicOrdering::Acquire: v = 2; break; 518de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicOrdering::Release: v = 3; break; 519de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicOrdering::AcquireRelease: v = 4; break; 520de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar case AtomicOrdering::SequentiallyConsistent: v = 5; break; 521e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 522c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov return IRB->getInt32(v); 523e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 524e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 525f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// If a memset intrinsic gets inlined by the code gen, we will miss races on it. 526f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// So, we either need to ensure the intrinsic is not inlined, or instrument it. 527f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// We do not instrument memset/memmove/memcpy intrinsics (too complicated), 528f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// instead we simply replace them with regular function calls, which are then 529f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// intercepted by the run-time. 530f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// Since tsan is running after everyone else, the calls should not be 531f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// replaced back with intrinsics. If that becomes wrong at some point, 532f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// we will need to call e.g. __tsan_memset to avoid the intrinsics. 533f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryanybool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 534f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRBuilder<> IRB(I); 535f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 5366948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreateCall( 5376948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar MemsetFn, 5386948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 5396948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), 5406948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 541f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany I->eraseFromParent(); 542f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 5436948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreateCall( 5446948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 5456948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 5466948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), 5476948897e478cbd66626159776a8017b3c18579b9Pirama Arumuga Nainar IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 548f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany I->eraseFromParent(); 549f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany } 550f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany return false; 551f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany} 552f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany 553de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainarstatic Value *createIntOrPtrToIntCast(Value *V, Type* Ty, IRBuilder<> &IRB) { 554de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar return isa<PointerType>(V->getType()) ? 555de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRB.CreatePtrToInt(V, Ty) : IRB.CreateIntCast(V, Ty, false); 556de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar} 557de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar 5586702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 55936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// standards. For background see C++11 standard. A slightly older, publicly 5606702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// available draft of the standard (not entirely up-to-date, but close enough 5616702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// for casual browsing) is available here: 56270af909f67565753cbd61886da2b0b7e191d32bfMatt Beaumont-Gay// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 5636702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// The following page contains more background information: 5646702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 5656702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov 5664c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarbool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 567e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRBuilder<> IRB(I); 568e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 569e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = LI->getPointerOperand(); 5704c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 571e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 572e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 573f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned ByteSize = 1U << Idx; 574f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned BitSize = ByteSize * 8; 575e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 576b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 577e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 578e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, LI->getOrdering())}; 579de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 580de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Ty == OrigTy) { 581de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Instruction *C = CallInst::Create(TsanAtomicLoad[Idx], Args); 582de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ReplaceInstWithInst(I, C); 583de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } else { 584de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // We are loading a pointer, so we need to cast the return value. 585de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *C = IRB.CreateCall(TsanAtomicLoad[Idx], Args); 586de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Instruction *Cast = CastInst::Create(Instruction::IntToPtr, C, OrigTy); 587de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar ReplaceInstWithInst(I, Cast); 588de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 589e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 590e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = SI->getPointerOperand(); 5914c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 592e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 593e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 594f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned ByteSize = 1U << Idx; 595f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned BitSize = ByteSize * 8; 596e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 597b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 598e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 599de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar createIntOrPtrToIntCast(SI->getValueOperand(), Ty, IRB), 600e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, SI->getOrdering())}; 60137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); 602e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReplaceInstWithInst(I, C); 6039f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 6049f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = RMWI->getPointerOperand(); 6054c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 6069f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 6079f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 6089f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 609dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!F) 6109f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 611f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned ByteSize = 1U << Idx; 612f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned BitSize = ByteSize * 8; 6139f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 6149f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 6159f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 6169f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 6179f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov createOrdering(&IRB, RMWI->getOrdering())}; 61837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CallInst *C = CallInst::Create(F, Args); 6199f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 6209f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 6219f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = CASI->getPointerOperand(); 6224c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 6239f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 6249f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 625f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned ByteSize = 1U << Idx; 626f3ef5332fa3f4d5ec72c178a2b19dac363a19383Pirama Arumuga Nainar const unsigned BitSize = ByteSize * 8; 6279f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 6289f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 629de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *CmpOperand = 630de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar createIntOrPtrToIntCast(CASI->getCompareOperand(), Ty, IRB); 631de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *NewOperand = 632de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar createIntOrPtrToIntCast(CASI->getNewValOperand(), Ty, IRB); 6339f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 634de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar CmpOperand, 635de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar NewOperand, 63636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines createOrdering(&IRB, CASI->getSuccessOrdering()), 63736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines createOrdering(&IRB, CASI->getFailureOrdering())}; 638c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 639de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Success = IRB.CreateICmpEQ(C, CmpOperand); 640de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *OldVal = C; 641de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Type *OrigOldValTy = CASI->getNewValOperand()->getType(); 642de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar if (Ty != OrigOldValTy) { 643de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar // The value is a pointer, so we need to cast the return value. 644de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar OldVal = IRB.CreateIntToPtr(C, OrigOldValTy); 645de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar } 646c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 647de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar Value *Res = 648de2d8694e25a814696358e95141f4b1aa4d8847ePirama Arumuga Nainar IRB.CreateInsertValue(UndefValue::get(CASI->getType()), OldVal, 0); 649c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Res = IRB.CreateInsertValue(Res, Success, 1); 650c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 651c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines I->replaceAllUsesWith(Res); 652c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines I->eraseFromParent(); 6539f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 6549f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 6559f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = FI->getSynchScope() == SingleThread ? 6569f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicSignalFence : TsanAtomicThreadFence; 65737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CallInst *C = CallInst::Create(F, Args); 6589f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 659e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 660e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 661e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 662e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 6634c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarint ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, 6644c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL) { 665e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigPtrTy = Addr->getType(); 666e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 667e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(OrigTy->isSized()); 6684c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 669e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (TypeSize != 8 && TypeSize != 16 && 670e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 671e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany NumAccessesWithBadSize++; 672e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany // Ignore all unusual sizes. 673e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return -1; 674e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 675c6af2432c802d241c8fffbe0371c023e6c58844eMichael J. Spencer size_t Idx = countTrailingZeros(TypeSize / 8); 676e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(Idx < kNumberOfAccessSizes); 677e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return Idx; 678e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 679