12d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===-- sanitizer_atomic_clang_other.h --------------------------*- C++ -*-===// 22d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 32d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// The LLVM Compiler Infrastructure 42d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 52d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// This file is distributed under the University of Illinois Open Source 62d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// License. See LICENSE.TXT for details. 72d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 82d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===----------------------------------------------------------------------===// 92d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// This file is a part of ThreadSanitizer/AddressSanitizer runtime. 112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// Not intended for direct inclusion. Include sanitizer_atomic.h. 122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===----------------------------------------------------------------------===// 142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef SANITIZER_ATOMIC_CLANG_OTHER_H 162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#define SANITIZER_ATOMIC_CLANG_OTHER_H 172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesnamespace __sanitizer { 192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE void proc_yield(int cnt) { 212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE typename T::Type atomic_load( 262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines const volatile T *a, memory_order mo) { 272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(mo & (memory_order_relaxed | memory_order_consume 282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines | memory_order_acquire | memory_order_seq_cst)); 292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(!((uptr)a % sizeof(*a))); 302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines typename T::Type v; 312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (sizeof(*a) < 8 || sizeof(void*) == 8) { 332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Assume that aligned loads are atomic. 342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (mo == memory_order_relaxed) { 352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else if (mo == memory_order_consume) { 372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Assume that processor respects data dependencies 382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // (and that compiler won't break them). 392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else if (mo == memory_order_acquire) { 432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { // seq_cst 472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // E.g. on POWER we need a hw fence even before the store. 482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { 532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // 64-bit load on 32-bit platform. 542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Gross, but simple and reliable. 552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Assume that it is not in read-only memory. 562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = __sync_fetch_and_add( 572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines const_cast<typename T::Type volatile *>(&a->val_dont_use), 0); 582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return v; 602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(mo & (memory_order_relaxed | memory_order_release 652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines | memory_order_seq_cst)); 662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(!((uptr)a % sizeof(*a))); 672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (sizeof(*a) < 8 || sizeof(void*) == 8) { 692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Assume that aligned loads are atomic. 702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (mo == memory_order_relaxed) { 712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a->val_dont_use = v; 722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else if (mo == memory_order_release) { 732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 742d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a->val_dont_use = v; 752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 762d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { // seq_cst 772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a->val_dont_use = v; 792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { 822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // 64-bit store on 32-bit platform. 832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Gross, but simple and reliable. 842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines typename T::Type cmp = a->val_dont_use; 852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines typename T::Type cur; 862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (;;) { 872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines cur = __sync_val_compare_and_swap(&a->val_dont_use, cmp, v); 882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (cmp == v) 892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines break; 902d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines cmp = cur; 912d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 952d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} // namespace __sanitizer 962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif // #ifndef SANITIZER_ATOMIC_CLANG_OTHER_H 98