12d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===-- sanitizer_atomic_clang_x86.h ----------------------------*- C++ -*-===// 22d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 32d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// The LLVM Compiler Infrastructure 42d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 52d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// This file is distributed under the University of Illinois Open Source 62d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// License. See LICENSE.TXT for details. 72d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 82d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===----------------------------------------------------------------------===// 92d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// This file is a part of ThreadSanitizer/AddressSanitizer runtime. 112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// Not intended for direct inclusion. Include sanitizer_atomic.h. 122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines// 132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines//===----------------------------------------------------------------------===// 142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifndef SANITIZER_ATOMIC_CLANG_X86_H 162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#define SANITIZER_ATOMIC_CLANG_X86_H 172d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 182d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinesnamespace __sanitizer { 192d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 202d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE void proc_yield(int cnt) { 212d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 222d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines for (int i = 0; i < cnt; i++) 232d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("pause"); 242d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 252d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 262d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 272d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 282d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE typename T::Type atomic_load( 292d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines const volatile T *a, memory_order mo) { 302d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(mo & (memory_order_relaxed | memory_order_consume 312d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines | memory_order_acquire | memory_order_seq_cst)); 322d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(!((uptr)a % sizeof(*a))); 332d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines typename T::Type v; 342d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 352d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (sizeof(*a) < 8 || sizeof(void*) == 8) { 362d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Assume that aligned loads are atomic. 372d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (mo == memory_order_relaxed) { 382d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 392d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else if (mo == memory_order_consume) { 402d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Assume that processor respects data dependencies 412d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // (and that compiler won't break them). 422d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 432d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 442d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 452d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else if (mo == memory_order_acquire) { 462d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 472d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 482d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // On x86 loads are implicitly acquire. 492d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 502d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { // seq_cst 512d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // On x86 plain MOV is enough for seq_cst store. 522d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 532d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines v = a->val_dont_use; 542d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 552d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 562d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { 572d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // 64-bit load on 32-bit platform. 582d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__( 592d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves 602d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "movq %%mm0, %0;" // (ptr could be read-only) 612d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "emms;" // Empty mmx state/Reset FP regs 622d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines : "=m" (v) 632d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines : "m" (a->val_dont_use) 642d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines : // mark the FP stack and mmx registers as clobbered 652d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)", 662d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifdef __MMX__ 672d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", 682d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif // #ifdef __MMX__ 692d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "memory"); 702d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 712d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines return v; 722d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 732d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 742d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hinestemplate<typename T> 752d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen HinesINLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { 762d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(mo & (memory_order_relaxed | memory_order_release 772d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines | memory_order_seq_cst)); 782d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines DCHECK(!((uptr)a % sizeof(*a))); 792d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 802d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (sizeof(*a) < 8 || sizeof(void*) == 8) { 812d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // Assume that aligned loads are atomic. 822d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (mo == memory_order_relaxed) { 832d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a->val_dont_use = v; 842d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else if (mo == memory_order_release) { 852d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // On x86 stores are implicitly release. 862d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 872d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a->val_dont_use = v; 882d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 892d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { // seq_cst 902d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // On x86 stores are implicitly release. 912d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__("" ::: "memory"); 922d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines a->val_dont_use = v; 932d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 942d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 952d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } else { 962d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines // 64-bit store on 32-bit platform. 972d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __asm__ __volatile__( 982d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves 992d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "movq %%mm0, %0;" 1002d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "emms;" // Empty mmx state/Reset FP regs 1012d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines : "=m" (a->val_dont_use) 1022d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines : "m" (v) 1032d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines : // mark the FP stack and mmx registers as clobbered 1042d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)", 1052d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#ifdef __MMX__ 1062d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", 1072d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif // #ifdef __MMX__ 1082d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines "memory"); 1092d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines if (mo == memory_order_seq_cst) 1102d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines __sync_synchronize(); 1112d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines } 1122d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} 1132d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1142d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines} // namespace __sanitizer 1152d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines 1162d1fdb26e458c4ddc04155c1d421bced3ba90cd0Stephen Hines#endif // #ifndef SANITIZER_ATOMIC_CLANG_X86_H 117