1a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// Protocol Buffers - Google's data interchange format 2a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// Copyright 2012 Google Inc. All rights reserved. 3afb4b72037e3f13db208590fc782c4bc8e27f862Jeff Davidson// https://developers.google.com/protocol-buffers/ 4a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// 5a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// Redistribution and use in source and binary forms, with or without 6a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// modification, are permitted provided that the following conditions are 7a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// met: 8a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// 9a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// * Redistributions of source code must retain the above copyright 10a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// notice, this list of conditions and the following disclaimer. 11a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// * Redistributions in binary form must reproduce the above 12a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// copyright notice, this list of conditions and the following disclaimer 13a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// in the documentation and/or other materials provided with the 14a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// distribution. 15a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// * Neither the name of Google Inc. nor the names of its 16a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// contributors may be used to endorse or promote products derived from 17a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// this software without specific prior written permission. 18a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// 19a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 31a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// This file is an internal atomic implementation, use atomicops.h instead. 32a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 33a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#ifndef GOOGLE_PROTOBUF_ATOMICOPS_INTERNALS_X86_GCC_H_ 34a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#define GOOGLE_PROTOBUF_ATOMICOPS_INTERNALS_X86_GCC_H_ 35a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 36a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsonnamespace google { 37a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsonnamespace protobuf { 38a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsonnamespace internal { 39a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 40a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// This struct is not part of the public API of this module; clients may not 41a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// use it. 42a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// Features of this x86. Values may not be correct before main() is run, 43a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// but are set conservatively. 44a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsonstruct AtomicOps_x86CPUFeatureStruct { 45a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence 46a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // after acquire compare-and-swap. 47a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson bool has_sse2; // Processor has SSE2. 48a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson}; 49a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsonextern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures; 50a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 51a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") 52a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 53a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// 32-bit low-level operations on any platform. 54a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 55a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, 56a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 old_value, 57a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 new_value) { 58a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 prev; 59a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lock; cmpxchgl %1,%2" 60a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "=a" (prev) 61a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "q" (new_value), "m" (*ptr), "0" (old_value) 62a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "memory"); 63a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return prev; 64a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 65a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 66a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, 67a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 new_value) { 68a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("xchgl %1,%0" // The lock prefix is implicit for xchg. 69a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "=r" (new_value) 70a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "m" (*ptr), "0" (new_value) 71a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "memory"); 72a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return new_value; // Now it's the previous value. 73a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 74a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 75a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, 76a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 increment) { 77a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 temp = increment; 78a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lock; xaddl %0,%1" 79a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "+r" (temp), "+m" (*ptr) 80a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : : "memory"); 81a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // temp now holds the old value of *ptr 82a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return temp + increment; 83a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 84a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 85a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, 86a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 increment) { 87a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 temp = increment; 88a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lock; xaddl %0,%1" 89a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "+r" (temp), "+m" (*ptr) 90a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : : "memory"); 91a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // temp now holds the old value of *ptr 92a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 93a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lfence" : : : "memory"); 94a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } 95a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return temp + increment; 96a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 97a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 98a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, 99a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 old_value, 100a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 new_value) { 101a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 102a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 103a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lfence" : : : "memory"); 104a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } 105a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return x; 106a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 107a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 108a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, 109a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 old_value, 110a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 new_value) { 111a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 112a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 113a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 114a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { 115a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson *ptr = value; 116a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 117a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 118a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#if defined(__x86_64__) 119a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 120a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// 64-bit implementations of memory barrier can be simpler, because it 121a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// "mfence" is guaranteed to exist. 122a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void MemoryBarrier() { 123a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("mfence" : : : "memory"); 124a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 125a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 126a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 127a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson *ptr = value; 128a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson MemoryBarrier(); 129a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 130a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 131a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#else 132a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 133a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void MemoryBarrier() { 134a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson if (AtomicOps_Internalx86CPUFeatures.has_sse2) { 135a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("mfence" : : : "memory"); 136a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } else { // mfence is faster but not present on PIII 137a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 x = 0; 138a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII 139a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } 140a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 141a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 142a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 143a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson if (AtomicOps_Internalx86CPUFeatures.has_sse2) { 144a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson *ptr = value; 145a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("mfence" : : : "memory"); 146a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } else { 147a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson NoBarrier_AtomicExchange(ptr, value); 148a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // acts as a barrier on PIII 149a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } 150a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 151a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#endif 152a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 153a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { 154a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson ATOMICOPS_COMPILER_BARRIER(); 155a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson *ptr = value; // An x86 store acts as a release barrier. 156a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // See comments in Atomic64 version of Release_Store(), below. 157a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 158a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 159a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { 160a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return *ptr; 161a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 162a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 163a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { 164a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic32 value = *ptr; // An x86 load acts as a acquire barrier. 165a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // See comments in Atomic64 version of Release_Store(), below. 166a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson ATOMICOPS_COMPILER_BARRIER(); 167a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return value; 168a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 169a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 170a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic32 Release_Load(volatile const Atomic32* ptr) { 171a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson MemoryBarrier(); 172a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return *ptr; 173a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 174a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 175a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#if defined(__x86_64__) 176a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 177a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson// 64-bit low-level operations on 64-bit platform. 178a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 179a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 180a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 old_value, 181a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 new_value) { 182a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 prev; 183a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lock; cmpxchgq %1,%2" 184a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "=a" (prev) 185a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "q" (new_value), "m" (*ptr), "0" (old_value) 186a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "memory"); 187a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return prev; 188a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 189a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 190a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 191a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 new_value) { 192a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("xchgq %1,%0" // The lock prefix is implicit for xchg. 193a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "=r" (new_value) 194a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "m" (*ptr), "0" (new_value) 195a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "memory"); 196a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return new_value; // Now it's the previous value. 197a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 198a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 199a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 200a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 increment) { 201a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 temp = increment; 202a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lock; xaddq %0,%1" 203a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "+r" (temp), "+m" (*ptr) 204a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : : "memory"); 205a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // temp now contains the previous value of *ptr 206a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return temp + increment; 207a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 208a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 209a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 210a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 increment) { 211a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 temp = increment; 212a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lock; xaddq %0,%1" 213a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : "+r" (temp), "+m" (*ptr) 214a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson : : "memory"); 215a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // temp now contains the previous value of *ptr 216a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 217a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lfence" : : : "memory"); 218a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } 219a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return temp + increment; 220a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 221a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 222a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 223a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson *ptr = value; 224a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 225a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 226a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { 227a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson *ptr = value; 228a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson MemoryBarrier(); 229a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 230a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 231a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { 232a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson ATOMICOPS_COMPILER_BARRIER(); 233a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 234a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson *ptr = value; // An x86 store acts as a release barrier 235a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // for current AMD/Intel chips as of Jan 2008. 236a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // See also Acquire_Load(), below. 237a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 238a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // When new chips come out, check: 239a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // IA-32 Intel Architecture Software Developer's Manual, Volume 3: 240a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // System Programming Guide, Chatper 7: Multiple-processor management, 241a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // Section 7.2, Memory Ordering. 242a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // Last seen at: 243a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // http://developer.intel.com/design/pentium4/manuals/index_new.htm 244a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // 245a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // x86 stores/loads fail to act as barriers for a few instructions (clflush 246a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // maskmovdqu maskmovq movntdq movnti movntpd movntps movntq) but these are 247a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // not generated by the compiler, and are rare. Users of these instructions 248a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // need to know about cache behaviour in any case since all of these involve 249a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // either flushing cache lines or non-temporal cache hints. 250a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 251a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 252a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 253a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return *ptr; 254a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 255a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 256a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { 257a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 value = *ptr; // An x86 load acts as a acquire barrier, 258a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // for current AMD/Intel chips as of Jan 2008. 259a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson // See also Release_Store(), above. 260a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson ATOMICOPS_COMPILER_BARRIER(); 261a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return value; 262a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 263a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 264a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 Release_Load(volatile const Atomic64* ptr) { 265a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson MemoryBarrier(); 266a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return *ptr; 267a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 268a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 269a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, 270a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 old_value, 271a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 new_value) { 272a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 273a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 274a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson __asm__ __volatile__("lfence" : : : "memory"); 275a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson } 276a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return x; 277a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 278a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 279a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidsoninline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, 280a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 old_value, 281a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson Atomic64 new_value) { 282a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 283a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} 284a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 285a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#endif // defined(__x86_64__) 286a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 287a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} // namespace internal 288a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} // namespace protobuf 289a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson} // namespace google 290a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 291a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#undef ATOMICOPS_COMPILER_BARRIER 292a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson 293a3b2a6da25a76f17c73d31def3952feb0fd2296eJeff Davidson#endif // GOOGLE_PROTOBUF_ATOMICOPS_INTERNALS_X86_GCC_H_ 294