1c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. 2c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Use of this source code is governed by a BSD-style license that can be 3c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// found in the LICENSE file. 4c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 5c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// This file is an internal atomic implementation, use base/atomicops.h instead. 6c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 7c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#ifndef BASE_ATOMICOPS_INTERNALS_X86_GCC_H_ 8c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#define BASE_ATOMICOPS_INTERNALS_X86_GCC_H_ 93345a6884c488ff3a535c2c9acdd33d74b37e311Iain Merrick#pragma once 10c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 11c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// This struct is not part of the public API of this module; clients may not 12c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// use it. 13c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Features of this x86. Values may not be correct before main() is run, 14c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// but are set conservatively. 15c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottstruct AtomicOps_x86CPUFeatureStruct { 16c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence 17c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // after acquire compare-and-swap. 18c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott bool has_sse2; // Processor has SSE2. 19c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott}; 20c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottextern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures; 21c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 22c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") 23c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 24c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottnamespace base { 25c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottnamespace subtle { 26c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 27c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 32-bit low-level operations on any platform. 28c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 29c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, 30c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 old_value, 31c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 new_value) { 32c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 prev; 33c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lock; cmpxchgl %1,%2" 34c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "=a" (prev) 35c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "q" (new_value), "m" (*ptr), "0" (old_value) 36c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "memory"); 37c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return prev; 38c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 39c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 40c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, 41c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 new_value) { 42c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("xchgl %1,%0" // The lock prefix is implicit for xchg. 43c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "=r" (new_value) 44c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "m" (*ptr), "0" (new_value) 45c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "memory"); 46c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return new_value; // Now it's the previous value. 47c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 48c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 49c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, 50c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 increment) { 51c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 temp = increment; 52c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lock; xaddl %0,%1" 53c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "+r" (temp), "+m" (*ptr) 54c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : : "memory"); 55c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // temp now holds the old value of *ptr 56c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return temp + increment; 57c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 58c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 59c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, 60c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 increment) { 61c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 temp = increment; 62c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lock; xaddl %0,%1" 63c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "+r" (temp), "+m" (*ptr) 64c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : : "memory"); 65c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // temp now holds the old value of *ptr 66c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 67c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lfence" : : : "memory"); 68c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 69c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return temp + increment; 70c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 71c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 72c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, 73c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 old_value, 74c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 new_value) { 75c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 76c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 77c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lfence" : : : "memory"); 78c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 79c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return x; 80c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 81c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 82c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, 83c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 old_value, 84c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 new_value) { 85c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 86c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 87c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 88c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { 89c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *ptr = value; 90c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 91c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 92c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#if defined(__x86_64__) 93c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 94c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 64-bit implementations of memory barrier can be simpler, because it 95c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// "mfence" is guaranteed to exist. 96c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void MemoryBarrier() { 97c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("mfence" : : : "memory"); 98c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 99c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 100c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 101c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *ptr = value; 102c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott MemoryBarrier(); 103c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 104c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 105c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#else 106c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 107c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void MemoryBarrier() { 108c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (AtomicOps_Internalx86CPUFeatures.has_sse2) { 109c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("mfence" : : : "memory"); 110c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } else { // mfence is faster but not present on PIII 111c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 x = 0; 112c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII 113c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 114c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 115c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 116c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 117c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (AtomicOps_Internalx86CPUFeatures.has_sse2) { 118c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *ptr = value; 119c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("mfence" : : : "memory"); 120c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } else { 121c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott NoBarrier_AtomicExchange(ptr, value); 122c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // acts as a barrier on PIII 123c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 124c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 125c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#endif 126c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 127c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { 128c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott ATOMICOPS_COMPILER_BARRIER(); 129c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *ptr = value; // An x86 store acts as a release barrier. 130c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // See comments in Atomic64 version of Release_Store(), below. 131c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 132c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 133c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { 134c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return *ptr; 135c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 136c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 137c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { 138c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic32 value = *ptr; // An x86 load acts as a acquire barrier. 139c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // See comments in Atomic64 version of Release_Store(), below. 140c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott ATOMICOPS_COMPILER_BARRIER(); 141c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return value; 142c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 143c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 144c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic32 Release_Load(volatile const Atomic32* ptr) { 145c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott MemoryBarrier(); 146c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return *ptr; 147c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 148c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 149c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#if defined(__x86_64__) 150c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 151c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 64-bit low-level operations on 64-bit platform. 152c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 153c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 154c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 old_value, 155c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 new_value) { 156c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 prev; 157c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lock; cmpxchgq %1,%2" 158c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "=a" (prev) 159c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "q" (new_value), "m" (*ptr), "0" (old_value) 160c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "memory"); 161c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return prev; 162c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 163c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 164c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 165c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 new_value) { 166c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("xchgq %1,%0" // The lock prefix is implicit for xchg. 167c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "=r" (new_value) 168c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "m" (*ptr), "0" (new_value) 169c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "memory"); 170c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return new_value; // Now it's the previous value. 171c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 172c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 173c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 174c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 increment) { 175c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 temp = increment; 176c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lock; xaddq %0,%1" 177c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "+r" (temp), "+m" (*ptr) 178c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : : "memory"); 179c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // temp now contains the previous value of *ptr 180c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return temp + increment; 181c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 182c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 183c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 184c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 increment) { 185c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 temp = increment; 186c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lock; xaddq %0,%1" 187c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : "+r" (temp), "+m" (*ptr) 188c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott : : "memory"); 189c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // temp now contains the previous value of *ptr 190c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 191c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lfence" : : : "memory"); 192c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 193c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return temp + increment; 194c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 195c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 196c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 197c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *ptr = value; 198c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 199c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 200c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { 201c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *ptr = value; 202c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott MemoryBarrier(); 203c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 204c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 205c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { 206c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott ATOMICOPS_COMPILER_BARRIER(); 207c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 208c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *ptr = value; // An x86 store acts as a release barrier 209c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // for current AMD/Intel chips as of Jan 2008. 210c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // See also Acquire_Load(), below. 211c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 212c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // When new chips come out, check: 213c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // IA-32 Intel Architecture Software Developer's Manual, Volume 3: 214c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // System Programming Guide, Chatper 7: Multiple-processor management, 215c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // Section 7.2, Memory Ordering. 216c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // Last seen at: 217c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // http://developer.intel.com/design/pentium4/manuals/index_new.htm 218c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // 219c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // x86 stores/loads fail to act as barriers for a few instructions (clflush 220c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // maskmovdqu maskmovq movntdq movnti movntpd movntps movntq) but these are 221c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // not generated by the compiler, and are rare. Users of these instructions 222c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // need to know about cache behaviour in any case since all of these involve 223c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // either flushing cache lines or non-temporal cache hints. 224c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 225c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 226c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 227c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return *ptr; 228c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 229c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 230c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { 231c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 value = *ptr; // An x86 load acts as a acquire barrier, 232c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // for current AMD/Intel chips as of Jan 2008. 233c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // See also Release_Store(), above. 234c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott ATOMICOPS_COMPILER_BARRIER(); 235c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return value; 236c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 237c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 238c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 Release_Load(volatile const Atomic64* ptr) { 239c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott MemoryBarrier(); 240c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return *ptr; 241c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 242c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 243c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, 244c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 old_value, 245c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 new_value) { 246c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 247c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 248c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott __asm__ __volatile__("lfence" : : : "memory"); 249c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 250c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return x; 251c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 252c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 253c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottinline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, 254c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 old_value, 255c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott Atomic64 new_value) { 256c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 257c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 258c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 259c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#endif // defined(__x86_64__) 260c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 261c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} // namespace base::subtle 262c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} // namespace base 263c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 264c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#undef ATOMICOPS_COMPILER_BARRIER 265c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 266c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#endif // BASE_ATOMICOPS_INTERNALS_X86_GCC_H_ 267