17ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Body of the hottest inner loop. 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// If we wrap this body into a function, compilers (both gcc and clang) 147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// produce sligtly less efficient code. 157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanydo { 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowProcessed); 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const unsigned kAccessSize = 1 << kAccessSizeLog; 195d71de26cedae3dafc17449fe0182045c0bd20e8Stephen Hines u64 *sp = &shadow_mem[idx]; 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany old = LoadShadow(sp); 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (old.IsZero()) { 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowZero); 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (store_word) 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreIfNotYetStored(sp, &store_word); 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // The above StoreIfNotYetStored could be done unconditionally 267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // and it even shows 4% gain on synthetic benchmarks (r4307). 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // is the memory access equal to the previous? 307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (Shadow::Addr0AndSizeAreEqual(cur, old)) { 317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowSameSize); 327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // same thread? 337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (Shadow::TidsAreEqual(old, cur)) { 347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowSameThread); 35334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) 367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreIfNotYetStored(sp, &store_word); 377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowAnotherThread); 407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (HappensBefore(old, thr)) { 417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreIfNotYetStored(sp, &store_word); 427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 44334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) 457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany goto RACE; 477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Do the memory access intersect? 4948cd12d0398914db195c1a25852b153f3345090dDmitry Vyukov if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) { 507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowIntersect); 517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (Shadow::TidsAreEqual(old, cur)) { 527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowSameThread); 537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowAnotherThread); 56334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) 577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 58334553ec45d8982df45a6f5e656e068142ecde3fDmitry Vyukov if (HappensBefore(old, thr)) 597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany goto RACE; 617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // The accesses do not intersect. 637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowNotIntersect); 647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} while (0); 66