15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* Copyright (c) 2006, Google Inc. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * All rights reserved. 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * Redistribution and use in source and binary forms, with or without 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * modification, are permitted provided that the following conditions are 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * met: 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * * Redistributions of source code must retain the above copyright 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * notice, this list of conditions and the following disclaimer. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * * Redistributions in binary form must reproduce the above 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * copyright notice, this list of conditions and the following disclaimer 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * in the documentation and/or other materials provided with the 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * distribution. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * * Neither the name of Google Inc. nor the names of its 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * contributors may be used to endorse or promote products derived from 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * this software without specific prior written permission. 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * --- 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) * Author: Sanjay Ghemawat 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Implementation of atomic operations using Windows API 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// functions. This file should not be included directly. Clients 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// should instead include "base/atomicops.h". 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdio.h> 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdlib.h> 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/abort.h" 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/basictypes.h" // For COMPILE_ASSERT 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef int32 Atomic32; 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(_WIN64) 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace base { 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace subtle { 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef int64 Atomic64; 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 32-bit low-level operations on any platform 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" { 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We use windows intrinsics when we can (they seem to be supported 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// well on MSVC 8.0 and above). Unfortunately, in some 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// environments, <windows.h> and <intrin.h> have conflicting 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// declarations of some other intrinsics, breaking compilation: 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// http://connect.microsoft.com/VisualStudio/feedback/details/262047 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Therefore, we simply declare the relevant intrinsics ourself. 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// MinGW has a bug in the header files where it doesn't indicate the 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// first argument is volatile -- they're not up to date. See 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// http://readlist.com/lists/lists.sourceforge.net/mingw-users/0/3861.html 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// We have to const_cast away the volatile to avoid compiler warnings. 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO(csilvers): remove this once MinGW has updated MinGW/include/winbase.h 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(__MINGW32__) 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONG newval, LONG oldval) { 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedCompareExchange(const_cast<LONG*>(ptr), newval, oldval); 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchange(const_cast<LONG*>(ptr), newval); 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchangeAdd(const_cast<LONG*>(ptr), increment); 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Unfortunately, in some environments, <windows.h> and <intrin.h> 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// have conflicting declarations of some intrinsics, breaking 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// compilation. So we declare the intrinsics we need ourselves. See 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// http://connect.microsoft.com/VisualStudio/feedback/details/262047 895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// Don't declare the intrinsics if using Clang. Clang provides inline 915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)// definitions in its Intrin.h. 925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#ifndef __clang__ 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)LONG _InterlockedCompareExchange(volatile LONG* ptr, LONG newval, LONG oldval); 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#pragma intrinsic(_InterlockedCompareExchange) 955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)LONG _InterlockedExchange(volatile LONG* ptr, LONG newval); 975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#pragma intrinsic(_InterlockedExchange) 985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)LONG _InterlockedExchangeAdd(volatile LONG* ptr, LONG increment); 1005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#pragma intrinsic(_InterlockedExchangeAdd) 1015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)#endif 1025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONG newval, LONG oldval) { 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return _InterlockedCompareExchange(ptr, newval, oldval); 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return _InterlockedExchange(ptr, newval); 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return _InterlockedExchangeAdd(ptr, increment); 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONG newval, LONG oldval) { 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedCompareExchange(ptr, newval, oldval); 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchange(ptr, newval); 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchangeAdd(ptr, increment); 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // ifdef __MINGW32__ 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // extern "C" 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 old_value, 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONG result = FastInterlockedCompareExchange( 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<volatile LONG*>(ptr), 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<LONG>(new_value), 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<LONG>(old_value)); 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return static_cast<Atomic32>(result); 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONG result = FastInterlockedExchange( 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<volatile LONG*>(ptr), 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<LONG>(new_value)); 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return static_cast<Atomic32>(result); 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 increment) { 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return FastInterlockedExchangeAdd( 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<volatile LONG*>(ptr), 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<LONG>(increment)) + increment; 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 increment) { 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Barrier_AtomicIncrement(ptr, increment); 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace base::subtle 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace base 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// In msvc8/vs2005, winnt.h already contains a definition for 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// MemoryBarrier in the global namespace. Add it there for earlier 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// versions and forward to it from within the namespace. 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if !(defined(_MSC_VER) && _MSC_VER >= 1400) 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void MemoryBarrier() { 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 value = 0; 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::subtle::NoBarrier_AtomicExchange(&value, 0); 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // actually acts as a barrier in thisd implementation 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace base { 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace subtle { 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void MemoryBarrier() { 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ::MemoryBarrier(); 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 old_value, 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 old_value, 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 new_value) { 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ptr = value; 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NoBarrier_AtomicExchange(ptr, value); 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // acts as a barrier in this implementation 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ptr = value; // works w/o barrier for current Intel chips as of June 2005 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // See comments in Atomic64 version of Release_Store() below. 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return *ptr; 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic32 value = *ptr; 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic32 Release_Load(volatile const Atomic32* ptr) { 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return *ptr; 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 64-bit operations 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(_WIN64) || defined(__MINGW64__) 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 64-bit low-level operations on 64-bit platform. 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)COMPILE_ASSERT(sizeof(Atomic64) == sizeof(PVOID), atomic_word_is_atomic); 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// These are the intrinsics needed for 64-bit operations. Similar to the 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 32-bit case above. 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)extern "C" { 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if defined(__MINGW64__) 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PVOID newval, PVOID oldval) { 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedCompareExchangePointer(const_cast<PVOID*>(ptr), 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) newval, oldval); 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchangePointer(const_cast<PVOID*>(ptr), newval); 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONGLONG increment) { 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchangeAdd64(const_cast<LONGLONG*>(ptr), increment); 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Like above, we need to declare the intrinsics ourselves. 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)PVOID _InterlockedCompareExchangePointer(volatile PVOID* ptr, 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PVOID newval, PVOID oldval); 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#pragma intrinsic(_InterlockedCompareExchangePointer) 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PVOID newval, PVOID oldval) { 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return _InterlockedCompareExchangePointer(const_cast<PVOID*>(ptr), 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) newval, oldval); 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)PVOID _InterlockedExchangePointer(volatile PVOID* ptr, PVOID newval); 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#pragma intrinsic(_InterlockedExchangePointer) 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return _InterlockedExchangePointer(const_cast<PVOID*>(ptr), newval); 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)LONGLONG _InterlockedExchangeAdd64(volatile LONGLONG* ptr, LONGLONG increment); 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#pragma intrinsic(_InterlockedExchangeAdd64) 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONGLONG increment) { 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return _InterlockedExchangeAdd64(const_cast<LONGLONG*>(ptr), increment); 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PVOID newval, PVOID oldval) { 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedCompareExchangePointer(ptr, newval, oldval); 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchangePointer(ptr, newval); 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) LONGLONG increment) { 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ::InterlockedExchangeAdd64(ptr, increment); 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // ifdef __MINGW64__ 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // extern "C" 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PVOID result = FastInterlockedCompareExchangePointer( 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<volatile PVOID*>(ptr), 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<PVOID>(new_value), reinterpret_cast<PVOID>(old_value)); 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return reinterpret_cast<Atomic64>(result); 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PVOID result = FastInterlockedExchangePointer( 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<volatile PVOID*>(ptr), 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<PVOID>(new_value)); 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return reinterpret_cast<Atomic64>(result); 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return FastInterlockedExchangeAdd64( 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reinterpret_cast<volatile LONGLONG*>(ptr), 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static_cast<LONGLONG>(increment)) + increment; 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return Barrier_AtomicIncrement(ptr, increment); 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ptr = value; 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NoBarrier_AtomicExchange(ptr, value); 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // acts as a barrier in this implementation 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ptr = value; // works w/o barrier for current Intel chips as of June 2005 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // When new chips come out, check: 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // IA-32 Intel Architecture Software Developer's Manual, Volume 3: 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // System Programming Guide, Chatper 7: Multiple-processor management, 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Section 7.2, Memory Ordering. 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Last seen at: 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // http://developer.intel.com/design/pentium4/manuals/index_new.htm 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return *ptr; 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 value = *ptr; 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Release_Load(volatile const Atomic64* ptr) { 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return *ptr; 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else // defined(_WIN64) || defined(__MINGW64__) 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 64-bit low-level operations on 32-bit platform 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// TODO(vchen): The GNU assembly below must be converted to MSVC inline 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// assembly. Then the file should be renamed to ...-x86-msvc.h, probably. 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NotImplementedFatalError(const char *function_name) { 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fprintf(stderr, "64-bit %s() not implemented on this platform\n", 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) function_name); 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tcmalloc::Abort(); 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if 0 // Not implemented 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 prev; 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__("movl (%3), %%ebx\n\t" // Move 64-bit new_value into 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl 4(%3), %%ecx\n\t" // ecx:ebx 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "lock; cmpxchg8b %1\n\t" // If edx:eax (old_value) same 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=A" (prev) // as contents of ptr: 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "m" (*ptr), // ecx:ebx => ptr 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "0" (old_value), // else: 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "r" (&new_value) // old *ptr => edx:eax 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "memory", "%ebx", "%ecx"); 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return prev; 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_CompareAndSwap"); 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if 0 // Not implemented 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl (%2), %%ebx\n\t" // Move 64-bit new_value into 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl 4(%2), %%ecx\n\t" // ecx:ebx 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "0:\n\t" 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl %1, %%eax\n\t" // Read contents of ptr into 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl 4%1, %%edx\n\t" // edx:eax 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "lock; cmpxchg8b %1\n\t" // Attempt cmpxchg; if *ptr 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "jnz 0b\n\t" // is no longer edx:eax, loop 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=A" (new_value) 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "m" (*ptr), 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "r" (&new_value) 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "memory", "%ebx", "%ecx"); 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return new_value; // Now it's the previous value. 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_AtomicExchange"); 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if 0 // Not implemented 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 temp = increment; 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__( 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "0:\n\t" 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl (%3), %%ebx\n\t" // Move 64-bit increment into 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl 4(%3), %%ecx\n\t" // ecx:ebx 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl (%2), %%eax\n\t" // Read contents of ptr into 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "movl 4(%2), %%edx\n\t" // edx:eax 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "add %%eax, %%ebx\n\t" // sum => ecx:ebx 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "adc %%edx, %%ecx\n\t" // edx:eax still has old *ptr 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "lock; cmpxchg8b (%2)\n\t"// Attempt cmpxchg; if *ptr 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "jnz 0b\n\t" // is no longer edx:eax, loop 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "=A"(temp), "+m"(*ptr) 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "D" (ptr), "S" (&increment) 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : "memory", "%ebx", "%ecx"); 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // temp now contains the previous value of *ptr 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return temp + increment; 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_AtomicIncrement"); 4285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 4295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 4335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 increment) { 4345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if 0 // Not implemented 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_val = NoBarrier_AtomicIncrement(ptr, increment); 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm__ __volatile__("lfence" : : : "memory"); 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return new_val; 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("Barrier_AtomicIncrement"); 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if 0 // Not implemented 4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm { 4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) mov mm0, value; // Use mmx reg for 64-bit atomic moves 4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) mov ptr, mm0; 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) emms; // Empty mmx state to enable FP registers 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_Store"); 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { 4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NoBarrier_AtomicExchange(ptr, value); 4605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // acts as a barrier in this implementation 4615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { 4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NoBarrier_Store(ptr, value); 4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if 0 // Not implemented 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 value; 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __asm { 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) mov mm0, ptr; // Use mmx reg for 64-bit atomic moves 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) mov value, mm0; 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) emms; // Empty mmx state to enable FP registers 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#else 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NotImplementedFatalError("NoBarrier_Store"); 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 4795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { 4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 value = NoBarrier_Load(ptr); 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return value; 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Release_Load(volatile const Atomic64* ptr) { 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MemoryBarrier(); 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_Load(ptr); 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // defined(_WIN64) || defined(__MINGW64__) 4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, 4965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 4975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 4985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 4995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, 5025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 old_value, 5035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Atomic64 new_value) { 5045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 5055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace base::subtle 5085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace base 5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ 511