160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===-- ThreadSanitizer.cpp - race detector -------------------------------===// 260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The LLVM Compiler Infrastructure 460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is distributed under the University of Illinois Open Source 660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// License. See LICENSE.TXT for details. 760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// This file is a part of ThreadSanitizer, a race detector. 1160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The tool is under development, for the details about previous versions see 1360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// http://code.google.com/p/data-race-test 1460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// 1560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The instrumentation phase is quite simple: 1660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls to run-time library before every memory access. 1760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Optimizations may apply to avoid instrumenting some of the accesses. 1860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// - Insert calls at function entry/exit. 1960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany// The rest is handled by the run-time library. 2060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany//===----------------------------------------------------------------------===// 2160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 22d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/Transforms/Instrumentation.h" 23d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallSet.h" 24d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallString.h" 25d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/SmallVector.h" 26d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/Statistic.h" 27d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/StringExtras.h" 284c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar#include "llvm/Analysis/CaptureTracking.h" 294c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar#include "llvm/Analysis/ValueTracking.h" 300b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/DataLayout.h" 310b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Function.h" 320b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/IRBuilder.h" 33f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany#include "llvm/IR/IntrinsicInst.h" 340b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Intrinsics.h" 350b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/LLVMContext.h" 360b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Metadata.h" 370b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Module.h" 380b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/IR/Type.h" 396e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany#include "llvm/Support/CommandLine.h" 4060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/Debug.h" 4160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Support/MathExtras.h" 4252eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany#include "llvm/Support/raw_ostream.h" 43e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany#include "llvm/Transforms/Utils/BasicBlockUtils.h" 4460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany#include "llvm/Transforms/Utils/ModuleUtils.h" 4560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 4660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanyusing namespace llvm; 4760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 48dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "tsan" 49dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 5041d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentMemoryAccesses( 5141d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-memory-accesses", cl::init(true), 5241d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument memory accesses"), cl::Hidden); 5341d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentFuncEntryExit( 5441d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-func-entry-exit", cl::init(true), 5541d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument function entry and exit"), cl::Hidden); 5641d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryanystatic cl::opt<bool> ClInstrumentAtomics( 5741d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany "tsan-instrument-atomics", cl::init(true), 5841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany cl::desc("Instrument atomics"), cl::Hidden); 59f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryanystatic cl::opt<bool> ClInstrumentMemIntrinsics( 60f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany "tsan-instrument-memintrinsics", cl::init(true), 61f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 626e590e3f61b101554f97d6ab257c802c9ec49862Kostya Serebryany 632d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 642d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 651dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey SamsonovSTATISTIC(NumOmittedReadsBeforeWrite, 662d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads ignored due to following writes"); 672d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 682d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 69ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry VyukovSTATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 702d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromConstantGlobals, 712d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany "Number of reads from constant globals"); 722d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya SerebryanySTATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 73ebe69fe11e48d322045d5949c83283927a0d790bStephen HinesSTATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 742076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 7560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanynamespace { 762076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 7760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany/// ThreadSanitizer: instrument the code in module to find races. 7860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanystruct ThreadSanitizer : public FunctionPass { 794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar ThreadSanitizer() : FunctionPass(ID) {} 8036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines const char *getPassName() const override; 8136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool runOnFunction(Function &F) override; 8236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines bool doInitialization(Module &M) override; 8360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany static char ID; // Pass identification, replacement for typeid. 8460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 8560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany private: 868b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany void initializeCallbacks(Module &M); 874c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); 884c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar bool instrumentAtomic(Instruction *I, const DataLayout &DL); 89f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany bool instrumentMemIntrinsic(Instruction *I); 904c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 914c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> &All, 924c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL); 93cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany bool addrPointsToConstantData(Value *Addr); 944c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); 952076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 96f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany Type *IntptrTy; 97e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IntegerType *OrdTy; 9860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Callbacks to run-time library are computed in doInitialization. 99e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncEntry; 100e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanFuncExit; 10160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 1023eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany static const size_t kNumberOfAccessSizes = 5; 103e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanRead[kNumberOfAccessSizes]; 104e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanWrite[kNumberOfAccessSizes]; 105ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Function *TsanUnalignedRead[kNumberOfAccessSizes]; 106ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Function *TsanUnalignedWrite[kNumberOfAccessSizes]; 107e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicLoad[kNumberOfAccessSizes]; 108e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanAtomicStore[kNumberOfAccessSizes]; 1099f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes]; 1109f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicCAS[kNumberOfAccessSizes]; 1119f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicThreadFence; 1129f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *TsanAtomicSignalFence; 113e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Function *TsanVptrUpdate; 114ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov Function *TsanVptrLoad; 115f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany Function *MemmoveFn, *MemcpyFn, *MemsetFn; 11660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany}; 11760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} // namespace 11860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 11960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanychar ThreadSanitizer::ID = 0; 12060ebb1947faed42e493179e569c5db0c01d38a2aKostya SerebryanyINITIALIZE_PASS(ThreadSanitizer, "tsan", 12160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany "ThreadSanitizer: detects data races.", 12260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany false, false) 12360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 124e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanyconst char *ThreadSanitizer::getPassName() const { 125e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return "ThreadSanitizer"; 126e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 127e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 128c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen HinesFunctionPass *llvm::createThreadSanitizerPass() { 129c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines return new ThreadSanitizer(); 13060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 13160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 1328b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryanyvoid ThreadSanitizer::initializeCallbacks(Module &M) { 13360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(M.getContext()); 13460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Initialize the callbacks. 1352c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanFuncEntry = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 13637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 1372c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanFuncExit = checkSanitizerInterfaceFunction( 1382c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar M.getOrInsertFunction("__tsan_func_exit", IRB.getVoidTy(), nullptr)); 139e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany OrdTy = IRB.getInt32Ty(); 1403eccaa6625a8032bee6b84706cb46ed8eb915acfKostya Serebryany for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 141e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << i; 142e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 143e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> ReadName("__tsan_read" + itostr(ByteSize)); 1442c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 14537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 146e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 147e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> WriteName("__tsan_write" + itostr(ByteSize)); 1482c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 14937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 150e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 151ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallString<64> UnalignedReadName("__tsan_unaligned_read" + 152ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines itostr(ByteSize)); 1532c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanUnalignedRead[i] = 1542c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar checkSanitizerInterfaceFunction(M.getOrInsertFunction( 1552c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar UnalignedReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 156ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 157ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + 158ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines itostr(ByteSize)); 1592c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanUnalignedWrite[i] = 1602c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar checkSanitizerInterfaceFunction(M.getOrInsertFunction( 1612c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar UnalignedWriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 162ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines 163e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 164e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *PtrTy = Ty->getPointerTo(); 165e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> AtomicLoadName("__tsan_atomic" + itostr(BitSize) + 166e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "_load"); 1672c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanAtomicLoad[i] = checkSanitizerInterfaceFunction( 1682c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar M.getOrInsertFunction(AtomicLoadName, Ty, PtrTy, OrdTy, nullptr)); 169e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 170e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallString<32> AtomicStoreName("__tsan_atomic" + itostr(BitSize) + 171e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany "_store"); 1722c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanAtomicStore[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 1732c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy, nullptr)); 1749f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 1759f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov for (int op = AtomicRMWInst::FIRST_BINOP; 1769f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov op <= AtomicRMWInst::LAST_BINOP; ++op) { 177dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines TsanAtomicRMW[op][i] = nullptr; 178dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const char *NamePart = nullptr; 1799f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (op == AtomicRMWInst::Xchg) 1809f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_exchange"; 1819f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Add) 1829f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_add"; 1839f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Sub) 1849f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_sub"; 1859f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::And) 1869f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_and"; 1879f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Or) 1889f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_or"; 1899f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else if (op == AtomicRMWInst::Xor) 1909f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov NamePart = "_fetch_xor"; 191b10675ef14bea530551172547e2111bf707a408eDmitry Vyukov else if (op == AtomicRMWInst::Nand) 192b10675ef14bea530551172547e2111bf707a408eDmitry Vyukov NamePart = "_fetch_nand"; 1939f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov else 1949f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov continue; 1959f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 1962c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanAtomicRMW[op][i] = checkSanitizerInterfaceFunction( 1972c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar M.getOrInsertFunction(RMWName, Ty, PtrTy, Ty, OrdTy, nullptr)); 1989f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } 1999f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov 2009f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov SmallString<32> AtomicCASName("__tsan_atomic" + itostr(BitSize) + 2019f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov "_compare_exchange_val"); 2022c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanAtomicCAS[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 20337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, nullptr)); 20460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 2052c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanVptrUpdate = checkSanitizerInterfaceFunction( 2062c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar M.getOrInsertFunction("__tsan_vptr_update", IRB.getVoidTy(), 2072c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), nullptr)); 2082c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanVptrLoad = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 20937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_vptr_read", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr)); 2102c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanAtomicThreadFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 21137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, nullptr)); 2122c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar TsanAtomicSignalFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction( 21337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, nullptr)); 214f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany 2152c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar MemmoveFn = checkSanitizerInterfaceFunction( 2162c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 2172c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar IRB.getInt8PtrTy(), IntptrTy, nullptr)); 2182c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar MemcpyFn = checkSanitizerInterfaceFunction( 2192c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 2202c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar IRB.getInt8PtrTy(), IntptrTy, nullptr)); 2212c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar MemsetFn = checkSanitizerInterfaceFunction( 2222c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 2232c3e0051c31c3f5b2328b447eadf1cf9c4427442Pirama Arumuga Nainar IRB.getInt32Ty(), IntptrTy, nullptr)); 2248b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany} 2258b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 2268b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryanybool ThreadSanitizer::doInitialization(Module &M) { 2274c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL = M.getDataLayout(); 2288b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 2298b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany // Always insert a call to __tsan_init into the module's CTORs. 2308b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany IRBuilder<> IRB(M.getContext()); 23136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines IntptrTy = IRB.getIntPtrTy(DL); 2328b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany Value *TsanInit = M.getOrInsertFunction("__tsan_init", 23337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines IRB.getVoidTy(), nullptr); 2348b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany appendToGlobalCtors(M, cast<Function>(TsanInit), 0); 2358b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 23660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 23760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 23860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 239cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanystatic bool isVtableAccess(Instruction *I) { 2400b3d39235aaed8bc66ccffb3942bf7b5f185329cManman Ren if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 2410b3d39235aaed8bc66ccffb3942bf7b5f185329cManman Ren return Tag->isTBAAVtableAccess(); 242cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 243cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 244cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 245cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryanybool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 246cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // If this is a GEP, just analyze its pointer operand. 247cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 248cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Addr = GEP->getPointerOperand(); 249cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 250cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 251cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (GV->isConstant()) { 252cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from constant globals can not race with any writes. 2532d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromConstantGlobals++; 254cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 255cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 2561dfe9b52646ce56446ecd3548dbc81068369ef3fAlexey Samsonov } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 257cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (isVtableAccess(L)) { 258cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Reads from a vtable pointer can not race with any writes. 2592d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsFromVtable++; 260cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return true; 261cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 262cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 263cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany return false; 264cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany} 265cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany 2662076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Instrumenting some of the accesses may be proven redundant. 2672076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// Currently handled: 2682076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// - read-before-write (within same BB, no calls between) 269ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines// - not captured variables 2702076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 2712076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// We do not handle some of the patterns that should not survive 2722076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// after the classic compiler optimizations. 2732076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// E.g. two reads from the same temp should be eliminated by CSE, 2742076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// two writes should be eliminated by DSE, etc. 2752076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 2762076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'Local' is a vector of insns within the same BB (no calls between). 2772076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany// 'All' is a vector of insns that will be instrumented. 27837cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryanyvoid ThreadSanitizer::chooseInstructionsToInstrument( 2794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All, 2804c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL) { 2812076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallSet<Value*, 8> WriteTargets; 2822076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // Iterate from the end. 2832076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany for (SmallVectorImpl<Instruction*>::reverse_iterator It = Local.rbegin(), 2842076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany E = Local.rend(); It != E; ++It) { 2852076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany Instruction *I = *It; 2862076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 2872076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany WriteTargets.insert(Store->getPointerOperand()); 2882076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } else { 2892076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany LoadInst *Load = cast<LoadInst>(I); 290cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany Value *Addr = Load->getPointerOperand(); 291cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (WriteTargets.count(Addr)) { 2922076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany // We will write to this temp, so no reason to analyze the read. 2932d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumOmittedReadsBeforeWrite++; 2942076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany continue; 2952076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 296cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany if (addrPointsToConstantData(Addr)) { 297cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany // Addr points to some constant data -- it can not race with any writes. 298cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany continue; 299cff60c1409e36079b4bc6ecbda84565143bf00afKostya Serebryany } 3002076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 301ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *Addr = isa<StoreInst>(*I) 302ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ? cast<StoreInst>(I)->getPointerOperand() 303ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : cast<LoadInst>(I)->getPointerOperand(); 3044c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) && 305ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines !PointerMayBeCaptured(Addr, true, true)) { 306ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // The variable is addressable but not captured, so it cannot be 307ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // referenced from a different thread and participate in a data race 308ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines // (see llvm/Analysis/CaptureTracking.h for details). 309ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines NumOmittedNonCaptured++; 310ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines continue; 311ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines } 3122076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany All.push_back(I); 3132076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 3142076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany Local.clear(); 3152076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany} 3162076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany 317e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic bool isAtomic(Instruction *I) { 318e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) 319e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return LI->isAtomic() && LI->getSynchScope() == CrossThread; 32037cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (StoreInst *SI = dyn_cast<StoreInst>(I)) 321e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return SI->isAtomic() && SI->getSynchScope() == CrossThread; 32237cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicRMWInst>(I)) 323e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 32437cb9acac54fdebdb833dc7e2f312d8e3bb33002Kostya Serebryany if (isa<AtomicCmpXchgInst>(I)) 325e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 3269f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (isa<FenceInst>(I)) 3279f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return true; 328e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 329e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 330e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 33160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryanybool ThreadSanitizer::runOnFunction(Function &F) { 3328b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany initializeCallbacks(*F.getParent()); 33360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany SmallVector<Instruction*, 8> RetVec; 3342076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> AllLoadsAndStores; 3352076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany SmallVector<Instruction*, 8> LocalLoadsAndStores; 336e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany SmallVector<Instruction*, 8> AtomicAccesses; 337f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany SmallVector<Instruction*, 8> MemIntrinCalls; 33860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool Res = false; 33960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool HasCalls = false; 340c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 3414c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL = F.getParent()->getDataLayout(); 34260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 34360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Traverse all instructions, collect loads/stores/returns, check for calls. 344c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto &BB : F) { 345c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto &Inst : BB) { 346c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (isAtomic(&Inst)) 347c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines AtomicAccesses.push_back(&Inst); 348c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 349c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines LocalLoadsAndStores.push_back(&Inst); 350c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines else if (isa<ReturnInst>(Inst)) 351c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines RetVec.push_back(&Inst); 352c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { 353c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (isa<MemIntrinsic>(Inst)) 354c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines MemIntrinCalls.push_back(&Inst); 35560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany HasCalls = true; 3564c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 3574c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar DL); 3582076af0184bd38185a719ea07cb10a90e8565a89Kostya Serebryany } 35960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 3604c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 36160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 36260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 36360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // We have collected all loads and stores. 36460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // FIXME: many of these accesses do not need to be checked for races 36560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // (e.g. variables that do not escape, etc). 36660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 367c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Instrument memory accesses only if we want to report bugs in the function. 368c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (ClInstrumentMemoryAccesses && SanitizeFunction) 369c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto Inst : AllLoadsAndStores) { 3704c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Res |= instrumentLoadOrStore(Inst, DL); 37141d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 37260ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 373c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // Instrument atomic memory accesses in any case (they can be used to 374c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines // implement synchronization). 37541d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if (ClInstrumentAtomics) 376c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto Inst : AtomicAccesses) { 3774c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar Res |= instrumentAtomic(Inst, DL); 37841d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany } 379e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 380c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines if (ClInstrumentMemIntrinsics && SanitizeFunction) 381c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto Inst : MemIntrinCalls) { 382c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Res |= instrumentMemIntrinsic(Inst); 383f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany } 384f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany 38560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany // Instrument function entry/exit points if there were instrumented accesses. 38641d876cea3b5212e5ee8bc25c13c6eab54a4a93cKostya Serebryany if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 38760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 38860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *ReturnAddress = IRB.CreateCall( 38960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 39060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.getInt32(0)); 39160ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(TsanFuncEntry, ReturnAddress); 392c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines for (auto RetInst : RetVec) { 393c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines IRBuilder<> IRBRet(RetInst); 39460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBRet.CreateCall(TsanFuncExit); 39560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 39652eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Res = true; 39760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany } 39860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return Res; 39960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 40060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany 4014c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarbool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, 4024c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL) { 40360ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRBuilder<> IRB(I); 40460ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany bool IsWrite = isa<StoreInst>(*I); 40560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany Value *Addr = IsWrite 40660ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany ? cast<StoreInst>(I)->getPointerOperand() 40760ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany : cast<LoadInst>(I)->getPointerOperand(); 4084c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 409e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 41060ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return false; 41152eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany if (IsWrite && isVtableAccess(I)) { 4124a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany DEBUG(dbgs() << " VPTR : " << *I << "\n"); 41352eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 41436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // StoredValue may be a vector type if we are storing several vptrs at once. 41536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // In this case, just take the first element of the vector since this is 41636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines // enough to find vptr races. 41736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (isa<VectorType>(StoredValue->getType())) 41836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines StoredValue = IRB.CreateExtractElement( 41936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 42036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if (StoredValue->getType()->isIntegerTy()) 4214a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 4224a002ab9c3eb19d33641cf37945a2ed153ee6d77Kostya Serebryany // Call TsanVptrUpdate. 42352eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreateCall2(TsanVptrUpdate, 42452eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 42552eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())); 4262d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany NumInstrumentedVtableWrites++; 42752eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany return true; 42852eb699220aa00696b5c4a1a67141a8bcc8a4e68Kostya Serebryany } 429ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov if (!IsWrite && isVtableAccess(I)) { 430ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov IRB.CreateCall(TsanVptrLoad, 431ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 432ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov NumInstrumentedVtableReads++; 433ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov return true; 434ab78ac1a2d55cd81e42f4688fbaa7bd689e6291fDmitry Vyukov } 435ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const unsigned Alignment = IsWrite 436ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines ? cast<StoreInst>(I)->getAlignment() 437ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines : cast<LoadInst>(I)->getAlignment(); 438ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 4394c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 440ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines Value *OnAccessFunc = nullptr; 441ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) 442ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 443ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines else 444ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 44560ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 4462d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany if (IsWrite) NumInstrumentedWrites++; 4472d5fdf81dc483e42cfad7ffbfa7492a31bb5ffd2Kostya Serebryany else NumInstrumentedReads++; 44860ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany return true; 44960ebb1947faed42e493179e569c5db0c01d38a2aKostya Serebryany} 450e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 451e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryanystatic ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 452e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany uint32_t v = 0; 453e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany switch (ord) { 454ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines case NotAtomic: llvm_unreachable("unexpected atomic ordering!"); 455e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany case Unordered: // Fall-through. 456c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Monotonic: v = 0; break; 4579a33f9f9804126661e87edfe491288b81f5c51e6Dmitry Vyukov // case Consume: v = 1; break; // Not specified yet. 458c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Acquire: v = 2; break; 459c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case Release: v = 3; break; 460c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case AcquireRelease: v = 4; break; 461c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov case SequentiallyConsistent: v = 5; break; 462e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 463c2e9ca15fbb7ce2bc0bc79e44862d74cdd2802b9Dmitry Vyukov return IRB->getInt32(v); 464e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 465e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 466f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// If a memset intrinsic gets inlined by the code gen, we will miss races on it. 467f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// So, we either need to ensure the intrinsic is not inlined, or instrument it. 468f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// We do not instrument memset/memmove/memcpy intrinsics (too complicated), 469f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// instead we simply replace them with regular function calls, which are then 470f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// intercepted by the run-time. 471f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// Since tsan is running after everyone else, the calls should not be 472f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// replaced back with intrinsics. If that becomes wrong at some point, 473f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany// we will need to call e.g. __tsan_memset to avoid the intrinsics. 474f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryanybool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 475f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRBuilder<> IRB(I); 476f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 477f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreateCall3(MemsetFn, 478f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 479f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), 480f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)); 481f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany I->eraseFromParent(); 482f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 483f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreateCall3(isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 484f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 485f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), 486f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)); 487f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany I->eraseFromParent(); 488f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany } 489f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany return false; 490f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany} 491f464481db0c3b5404004b510921ca454803fd1d0Kostya Serebryany 4926702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 49336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines// standards. For background see C++11 standard. A slightly older, publicly 4946702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// available draft of the standard (not entirely up-to-date, but close enough 4956702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// for casual browsing) is available here: 49670af909f67565753cbd61886da2b0b7e191d32bfMatt Beaumont-Gay// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 4976702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// The following page contains more background information: 4986702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 4996702e53926c2ce04ab4c049930d24a9c42996941Dmitry Vyukov 5004c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarbool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 501e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRBuilder<> IRB(I); 502e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 503e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = LI->getPointerOperand(); 5044c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 505e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 506e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 507e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << Idx; 508e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 509e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 510b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 511e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 512e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, LI->getOrdering())}; 51337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CallInst *C = CallInst::Create(TsanAtomicLoad[Idx], Args); 514e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReplaceInstWithInst(I, C); 515e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 516e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 517e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Addr = SI->getPointerOperand(); 5184c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 519e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (Idx < 0) 520e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return false; 521e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t ByteSize = 1 << Idx; 522e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany const size_t BitSize = ByteSize * 8; 523e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 524b8bce928f4ffdf50eff69334f3e25b27848536b6Micah Villmow Type *PtrTy = Ty->getPointerTo(); 525e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 526e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany IRB.CreateIntCast(SI->getValueOperand(), Ty, false), 527e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany createOrdering(&IRB, SI->getOrdering())}; 52837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); 529e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany ReplaceInstWithInst(I, C); 5309f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 5319f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = RMWI->getPointerOperand(); 5324c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 5339f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 5349f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 5359f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 536dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines if (!F) 5379f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 5389f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t ByteSize = 1 << Idx; 5399f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t BitSize = ByteSize * 8; 5409f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 5419f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 5429f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 5439f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 5449f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov createOrdering(&IRB, RMWI->getOrdering())}; 54537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CallInst *C = CallInst::Create(F, Args); 5469f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 5479f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 5489f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Addr = CASI->getPointerOperand(); 5494c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar int Idx = getMemoryAccessFuncIndex(Addr, DL); 5509f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov if (Idx < 0) 5519f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov return false; 5529f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t ByteSize = 1 << Idx; 5539f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov const size_t BitSize = ByteSize * 8; 5549f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 5559f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Type *PtrTy = Ty->getPointerTo(); 5569f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 5579f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(CASI->getCompareOperand(), Ty, false), 5589f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false), 55936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines createOrdering(&IRB, CASI->getSuccessOrdering()), 56036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines createOrdering(&IRB, CASI->getFailureOrdering())}; 561c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 562c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Value *Success = IRB.CreateICmpEQ(C, CASI->getCompareOperand()); 563c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 564c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Value *Res = IRB.CreateInsertValue(UndefValue::get(CASI->getType()), C, 0); 565c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines Res = IRB.CreateInsertValue(Res, Success, 1); 566c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines 567c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines I->replaceAllUsesWith(Res); 568c6a4f5e819217e1e12c458aed8e7b122e23a3a58Stephen Hines I->eraseFromParent(); 5699f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 5709f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 5719f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov Function *F = FI->getSynchScope() == SingleThread ? 5729f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov TsanAtomicSignalFence : TsanAtomicThreadFence; 57337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines CallInst *C = CallInst::Create(F, Args); 5749f8a90b3ce0e248e3b68b056d4c840295facbc02Dmitry Vyukov ReplaceInstWithInst(I, C); 575e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 576e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return true; 577e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 578e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany 5794c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainarint ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, 5804c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar const DataLayout &DL) { 581e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigPtrTy = Addr->getType(); 582e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 583e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(OrigTy->isSized()); 5844c5e43da7792f75567b693105cc53e3f1992ad98Pirama Arumuga Nainar uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 585e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany if (TypeSize != 8 && TypeSize != 16 && 586e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 587e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany NumAccessesWithBadSize++; 588e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany // Ignore all unusual sizes. 589e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return -1; 590e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany } 591c6af2432c802d241c8fffbe0371c023e6c58844eMichael J. Spencer size_t Idx = countTrailingZeros(TypeSize / 8); 592e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany assert(Idx < kNumberOfAccessSizes); 593e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany return Idx; 594e507922779b0d0d9ed3c159306174b4e191d119aKostya Serebryany} 595