17ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===// 27ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 37ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// The LLVM Compiler Infrastructure 47ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 57ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is distributed under the University of Illinois Open Source 67ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// License. See LICENSE.TXT for details. 77ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 87ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 97ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 107ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// This file is a part of ThreadSanitizer (TSan), a race detector. 117ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// 127ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// Body of the hottest inner loop. 137ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// If we wrap this body into a function, compilers (both gcc and clang) 147ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany// produce sligtly less efficient code. 157ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany//===----------------------------------------------------------------------===// 167ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryanydo { 177ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowProcessed); 187ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany const unsigned kAccessSize = 1 << kAccessSizeLog; 197ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany unsigned off = cur.ComputeSearchOffset(); 207ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany u64 *sp = &shadow_mem[(idx + off) % kShadowCnt]; 217ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany old = LoadShadow(sp); 227ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (old.IsZero()) { 237ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowZero); 247ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (store_word) 257ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreIfNotYetStored(sp, &store_word); 267ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // The above StoreIfNotYetStored could be done unconditionally 277ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // and it even shows 4% gain on synthetic benchmarks (r4307). 287ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 297ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 307ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // is the memory access equal to the previous? 317ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (Shadow::Addr0AndSizeAreEqual(cur, old)) { 327ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowSameSize); 337ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // same thread? 347ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (Shadow::TidsAreEqual(old, cur)) { 357ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowSameThread); 367ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (OldIsInSameSynchEpoch(old, thr)) { 377ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (OldIsRWStronger(old, kAccessIsWrite)) { 387ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // found a slot that holds effectively the same info 397ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // (that is, same tid, same sync epoch and same size) 407ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatMopSame); 417ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany return; 427ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 437ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreIfNotYetStored(sp, &store_word); 447ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 457ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 467ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (OldIsRWWeaker(old, kAccessIsWrite)) 477ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreIfNotYetStored(sp, &store_word); 487ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 497ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 507ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowAnotherThread); 517ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (HappensBefore(old, thr)) { 527ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StoreIfNotYetStored(sp, &store_word); 537ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 547ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 557ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (BothReads(old, kAccessIsWrite)) 567ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 577ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany goto RACE; 587ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 597ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 607ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // Do the memory access intersect? 617ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) { 627ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowIntersect); 637ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (Shadow::TidsAreEqual(old, cur)) { 647ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowSameThread); 657ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 667ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 677ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowAnotherThread); 687ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (HappensBefore(old, thr)) 697ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 707ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 717ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany if (BothReads(old, kAccessIsWrite)) 727ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 737ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany 747ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany goto RACE; 757ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany } 767ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany // The accesses do not intersect. 777ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany StatInc(thr, StatShadowNotIntersect); 787ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany break; 797ac41484ea322e0ea5774df681660269f5dc321eKostya Serebryany} while (0); 80